15 references to SymbolsCount
Microsoft.ML.Tokenizers (15)
Model\BPETokenizer.cs (4)
1372if (word.SymbolsCount <= maxTokens) 1380return word.SymbolsCount; 1393if (word.SymbolsCount <= maxTokens) 1401return word.SymbolsCount;
Model\Word.cs (11)
199for (int i = 0; i < SymbolsCount; i++) 209int count = Math.Min(SymbolsCount, maxTokens); 224int count = Math.Min(SymbolsCount, maxTokens); 226for (int i = SymbolsCount - count; i < SymbolsCount; i++) 239int count = Math.Min(SymbolsCount, maxTokens); 253int count = Math.Min(SymbolsCount, maxTokens); 255for (int i = SymbolsCount - count; i < SymbolsCount; i++) 298for (int i = 0; i < SymbolsCount; i++) 307for (int i = 0; i < SymbolsCount; i++)