1 write to ByteFallback
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
73
ByteFallback
= modelProto.TrainerSpec.ByteFallback;
5 references to ByteFallback
Microsoft.ML.Tokenizers (4)
Model\SentencePieceTokenizer.cs (4)
346
if (id == UnknownId &&
ByteFallback
)
695
if (id == UnknownId &&
ByteFallback
)
1019
if (id == UnknownId &&
ByteFallback
)
1399
if (id == UnknownId &&
ByteFallback
)
Microsoft.ML.Tokenizers.Tests (1)
LlamaTests.cs (1)
371
Assert.True(bpe.
ByteFallback
);