2 writes to InternalSpecialTokens
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
41InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>(); 88InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
10 references to InternalSpecialTokens
Microsoft.ML.Tokenizers (10)
Model\SentencePieceBaseModel.cs (2)
46InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value); 93InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
Model\SentencePieceBpeModel.cs (4)
123if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 383if (idsCount < maxTokens && InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 660if (idsCount < maxTokens && InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 930if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))
Model\SentencePieceUnigramModel.cs (4)
510if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 954if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1335if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1612if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))