1 write to InternalSpecialTokens
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
41InternalSpecialTokens = new Dictionary<StringSpanOrdinalKey, int>();
9 references to InternalSpecialTokens
Microsoft.ML.Tokenizers (9)
Model\SentencePieceBaseModel.cs (1)
46InternalSpecialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
Model\SentencePieceBpeModel.cs (4)
123if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 383if (idsCount < maxTokens && InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 660if (idsCount < maxTokens && InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 930if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))
Model\SentencePieceUnigramModel.cs (4)
235if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 668if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1039if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1306if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))