1 write to ByteFallback
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
73
ByteFallback
= modelProto.TrainerSpec.ByteFallback;
5 references to ByteFallback
Microsoft.ML.Tokenizers (4)
Model\SentencePieceTokenizer.cs (4)
347
if (id == UnknownId &&
ByteFallback
)
696
if (id == UnknownId &&
ByteFallback
)
1020
if (id == UnknownId &&
ByteFallback
)
1400
if (id == UnknownId &&
ByteFallback
)
Microsoft.ML.Tokenizers.Tests (1)
LlamaTests.cs (1)
371
Assert.True(bpe.
ByteFallback
);