2 writes to ByteFallback
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
36ByteFallback = modelProto.TrainerSpec.ByteFallback; 83ByteFallback = byteFallback;
21 references to ByteFallback
Microsoft.ML.Tokenizers (21)
Model\SentencePieceBaseModel.cs (6)
212if (current <= MaxByteId && ByteFallback) 230if (current <= MaxByteId && ByteFallback) 262if (current <= MaxByteId && ByteFallback) 437if (current <= MaxByteId && ByteFallback) 459if (current <= MaxByteId && ByteFallback) 535if (current <= MaxByteId && ByteFallback)
Model\SentencePieceBpeModel.cs (4)
182if (id == UnknownId && ByteFallback) 460if (id == UnknownId && ByteFallback) 728if (id == UnknownId && ByteFallback) 1004if (id == UnknownId && ByteFallback)
Model\SentencePieceTokenizer.cs (1)
47public bool ByteFallback => _model.ByteFallback;
Model\SentencePieceUnigramModel.cs (10)
212if (ByteFallback) 675if (ByteFallback && unknownEncountered) 1081bool needToTrackUnknown = ByteFallback || maxTokenCount != int.MaxValue; 1130Debug.Assert(ByteFallback || (unknownTokensCount == 0 && unknownTokensTracking is null)); 1132if (ByteFallback && unknownTokensCount > 0) 1175if (ByteFallback) 1200if (unknownTokensCount > 0 && ByteFallback) 1440if ((!ByteFallback || !unknownEncountered) && (maxTokenCount == int.MaxValue || (tokenCount + ids.Length - idsIndex <= maxTokenCount))) 1466if (ByteFallback) 1687if (ByteFallback)