1 write to _specialTokens
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
80_specialTokens = new Dictionary<StringSpanOrdinalKey, int>();
5 references to _specialTokens
Microsoft.ML.Tokenizers (5)
Model\SentencePieceTokenizer.cs (5)
85_specialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value); 288if (_specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 619if (idsCount < maxTokens && _specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 952if (idsCount < maxTokens && _specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1325if (_specialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))