1 write to EndOfSentenceToken
Microsoft.ML.Tokenizers.Tests (1)
UnigramTests.cs (1)
46
options.
EndOfSentenceToken
= "</s>";
7 references to EndOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\SentencePieceBaseModel.cs (3)
79
if (options.
EndOfSentenceToken
is null)
81
throw new ArgumentNullException(nameof(options.
EndOfSentenceToken
));
92
EndOfSentenceToken = options.
EndOfSentenceToken
;
Model\SentencePieceBpeModel.cs (2)
83
if (!_vocab.TryGetValue(options.
EndOfSentenceToken
, out (int Id, float Score, byte Type) endOfSentenceToken))
85
throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.
EndOfSentenceToken
}'.");
Model\SentencePieceUnigramModel.cs (2)
139
if (!_vocab.TryGetValue(options.
EndOfSentenceToken
, out int endOfSentenceToken))
141
throw new ArgumentException($"The vocabulary must include the end of sentence token '{options.
EndOfSentenceToken
}'.");