2 writes to InternalSpecialTokens
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
41InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>(); 102InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
10 references to InternalSpecialTokens
Microsoft.ML.Tokenizers (10)
Model\SentencePieceBaseModel.cs (2)
46InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value); 107InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
Model\SentencePieceBpeModel.cs (4)
169if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 429if (idsCount < maxTokens && InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 706if (idsCount < maxTokens && InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 976if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))
Model\SentencePieceUnigramModel.cs (4)
285if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 718if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1089if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1356if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))