15 references to SymbolsCount
Microsoft.ML.Tokenizers (15)
Model\BpeTokenizer.cs (4)
1392if (word.SymbolsCount <= maxTokens) 1400return word.SymbolsCount; 1413if (word.SymbolsCount <= maxTokens) 1421return word.SymbolsCount;
Model\Word.cs (11)
199for (int i = 0; i < SymbolsCount; i++) 209int count = Math.Min(SymbolsCount, maxTokens); 224int count = Math.Min(SymbolsCount, maxTokens); 226for (int i = SymbolsCount - count; i < SymbolsCount; i++) 239int count = Math.Min(SymbolsCount, maxTokens); 253int count = Math.Min(SymbolsCount, maxTokens); 255for (int i = SymbolsCount - count; i < SymbolsCount; i++) 298for (int i = 0; i < SymbolsCount; i++) 307for (int i = 0; i < SymbolsCount; i++)