1 write to ByteFallback
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
36ByteFallback = modelProto.TrainerSpec.ByteFallback;
19 references to ByteFallback
Microsoft.ML.Tokenizers (19)
Model\SentencePieceBaseModel.cs (6)
168if (current <= MaxByteId && ByteFallback) 186if (current <= MaxByteId && ByteFallback) 218if (current <= MaxByteId && ByteFallback) 393if (current <= MaxByteId && ByteFallback) 415if (current <= MaxByteId && ByteFallback) 491if (current <= MaxByteId && ByteFallback)
Model\SentencePieceBpeModel.cs (4)
182if (id == UnknownId && ByteFallback) 460if (id == UnknownId && ByteFallback) 728if (id == UnknownId && ByteFallback) 1004if (id == UnknownId && ByteFallback)
Model\SentencePieceTokenizer.cs (1)
41public bool ByteFallback => _model.ByteFallback;
Model\SentencePieceUnigramModel.cs (8)
400if (ByteFallback && unknownEncountered) 795bool needToTrackUnknown = ByteFallback || maxTokenCount != int.MaxValue; 846if (ByteFallback && unknownTokensCount > 0) 889if (ByteFallback) 914if (unknownTokensCount > 0 && ByteFallback) 1144if ((!ByteFallback || !unknownEncountered) && (maxTokenCount == int.MaxValue || (tokenCount + ids.Length - idsIndex <= maxTokenCount))) 1170if (ByteFallback) 1381if (ByteFallback)