14 references to SymbolsCount
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (4)
965if (word.SymbolsCount <= maxTokens) 973return word.SymbolsCount; 986if (word.SymbolsCount <= maxTokens) 994return word.SymbolsCount;
Model\Word.cs (10)
199for (int i = 0; i < SymbolsCount; i++) 209int count = Math.Min(SymbolsCount, maxTokens); 224int count = Math.Min(SymbolsCount, maxTokens); 226for (int i = SymbolsCount - count; i < SymbolsCount; i++) 239int count = Math.Min(SymbolsCount, maxTokens); 253int count = Math.Min(SymbolsCount, maxTokens); 255for (int i = SymbolsCount - count; i < SymbolsCount; i++) 296for (int i = 0; i < SymbolsCount; i++)