14 references to SymbolsCount
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (4)
965
if (word.
SymbolsCount
<= maxTokens)
973
return word.
SymbolsCount
;
986
if (word.
SymbolsCount
<= maxTokens)
994
return word.
SymbolsCount
;
Model\Word.cs (10)
199
for (int i = 0; i <
SymbolsCount
; i++)
209
int count = Math.Min(
SymbolsCount
, maxTokens);
224
int count = Math.Min(
SymbolsCount
, maxTokens);
226
for (int i =
SymbolsCount
- count; i <
SymbolsCount
; i++)
239
int count = Math.Min(
SymbolsCount
, maxTokens);
253
int count = Math.Min(
SymbolsCount
, maxTokens);
255
for (int i =
SymbolsCount
- count; i <
SymbolsCount
; i++)
296
for (int i = 0; i <
SymbolsCount
; i++)