1 write to _maxByteId
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
56
_maxByteId
= i;
7 references to _maxByteId
Microsoft.ML.Tokenizers (7)
Model\SentencePieceTokenizer.cs (7)
60
_byteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value) ? value.Id :
_maxByteId
;
1559
if (enumerator.Current <=
_maxByteId
)
1575
if (enumerator.Current <=
_maxByteId
)
1614
if (enumerator.Current <=
_maxByteId
)
1808
if (enumerator.Current <=
_maxByteId
)
1828
if (enumerator.Current <=
_maxByteId
)
1897
if (enumerator.Current <=
_maxByteId
)