14 references to SymbolsCount
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (4)
1015
if (word.
SymbolsCount
<= maxTokens)
1023
return word.
SymbolsCount
;
1036
if (word.
SymbolsCount
<= maxTokens)
1044
return word.
SymbolsCount
;
Model\Word.cs (10)
199
for (int i = 0; i <
SymbolsCount
; i++)
209
int count = Math.Min(
SymbolsCount
, maxTokens);
224
int count = Math.Min(
SymbolsCount
, maxTokens);
226
for (int i =
SymbolsCount
- count; i <
SymbolsCount
; i++)
239
int count = Math.Min(
SymbolsCount
, maxTokens);
253
int count = Math.Min(
SymbolsCount
, maxTokens);
255
for (int i =
SymbolsCount
- count; i <
SymbolsCount
; i++)
296
for (int i = 0; i <
SymbolsCount
; i++)