1 write to BeginningOfSentenceToken
Microsoft.ML.Tokenizers.Tests (1)
UnigramTests.cs (1)
45
options.
BeginningOfSentenceToken
= "<s>";
7 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (7)
Model\SentencePieceBaseModel.cs (3)
74
if (options.
BeginningOfSentenceToken
is null)
76
throw new ArgumentNullException(nameof(options.
BeginningOfSentenceToken
));
91
BeginningOfSentenceToken = options.
BeginningOfSentenceToken
;
Model\SentencePieceBpeModel.cs (2)
77
if (!_vocab.TryGetValue(options.
BeginningOfSentenceToken
, out (int Id, float Score, byte Type) beginOfSentenceToken))
79
throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.
BeginningOfSentenceToken
}'.");
Model\SentencePieceUnigramModel.cs (2)
133
if (!_vocab.TryGetValue(options.
BeginningOfSentenceToken
, out int beginOfSentenceToken))
135
throw new ArgumentException($"The vocabulary must include the beginning of sentence token '{options.
BeginningOfSentenceToken
}'.");