2 writes to ByteFallback
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
36ByteFallback = modelProto.TrainerSpec.ByteFallback; 97ByteFallback = options.ByteFallback;
19 references to ByteFallback
Microsoft.ML.Tokenizers (19)
Model\SentencePieceBaseModel.cs (6)
229if (current <= MaxByteId && ByteFallback) 247if (current <= MaxByteId && ByteFallback) 279if (current <= MaxByteId && ByteFallback) 454if (current <= MaxByteId && ByteFallback) 476if (current <= MaxByteId && ByteFallback) 552if (current <= MaxByteId && ByteFallback)
Model\SentencePieceBpeModel.cs (4)
228if (id == UnknownId && ByteFallback) 506if (id == UnknownId && ByteFallback) 774if (id == UnknownId && ByteFallback) 1050if (id == UnknownId && ByteFallback)
Model\SentencePieceTokenizer.cs (1)
51public bool ByteFallback => _model.ByteFallback;
Model\SentencePieceUnigramModel.cs (8)
450if (ByteFallback && unknownEncountered) 845bool needToTrackUnknown = ByteFallback || maxTokenCount != int.MaxValue; 896if (ByteFallback && unknownTokensCount > 0) 939if (ByteFallback) 964if (unknownTokensCount > 0 && ByteFallback) 1194if ((!ByteFallback || !unknownEncountered) && (maxTokenCount == int.MaxValue || (tokenCount + ids.Length - idsIndex <= maxTokenCount))) 1220if (ByteFallback) 1431if (ByteFallback)