1 write to _specialTokens
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBpeTokenizer.cs (1)
80_specialTokens = new Dictionary<StringSpanOrdinalKey, int>();
5 references to _specialTokens
Microsoft.ML.Tokenizers (5)
Model\SentencePieceBpeTokenizer.cs (5)
85_specialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value); 287if (_specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 618if (idsCount < maxTokens && _specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 951if (idsCount < maxTokens && _specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1324if (_specialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))