13 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers.Tests (13)
LlamaTests.cs (3)
366
Assert.Equal("<s>", bpe.
BeginningOfSentenceToken
);
665
Assert.Equal(new[] { tokenizer.
BeginningOfSentenceToken
, kvp.Key }, encodedTokens.Select(et => et.Value).ToArray());
838
string textWithSpecialTokens = $"{tokenizer.
BeginningOfSentenceToken
}{text}";
SentencePieceTests.cs (4)
80
Assert.Equal("<s>", tokenizer.
BeginningOfSentenceToken
);
203
Assert.Equal("<s>", tokenizer.
BeginningOfSentenceToken
);
236
Assert.Equal("[BOS]", tokenizer.
BeginningOfSentenceToken
);
310
Assert.Equal("[BOS]", tokenizer.
BeginningOfSentenceToken
);
UnigramTests.cs (6)
293
Assert.True(writableTokens[0].Value == tokenizer.
BeginningOfSentenceToken
);
353
string newString = $"{_unigramTokenizer.
BeginningOfSentenceToken
}{inputText}<pad>{inputText}{_unigramTokenizer.EndOfSentenceToken}";
366
expectedTokens[0] = _unigramTokenizerWithSpecialTokens.
BeginningOfSentenceToken
;
464
inputText = $"{_unigramTokenizerWithSpecialTokens.
BeginningOfSentenceToken
}{inputText}<pad>{inputText}{_unigramTokenizerWithSpecialTokens.EndOfSentenceToken}";
471
string expectedNormalized = $"{_unigramTokenizerWithSpecialTokens.
BeginningOfSentenceToken
}{normalizedText}<pad>{normalizedText}{_unigramTokenizerWithSpecialTokens.EndOfSentenceToken}";
560
Assert.Equal("<s>", _unigramTokenizer.
BeginningOfSentenceToken
);