13 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers.Tests (13)
LlamaTests.cs (3)
366
Assert.Equal("<s>", bpe.
BeginningOfSentenceToken
);
665
Assert.Equal(new[] { tokenizer.
BeginningOfSentenceToken
, kvp.Key }, encodedTokens.Select(et => et.Value).ToArray());
838
string textWithSpecialTokens = $"{tokenizer.
BeginningOfSentenceToken
}{text}";
UnigramTests.cs (10)
335
Assert.True(writableTokens[0].Value == tokenizer.
BeginningOfSentenceToken
);
452
string newString = $"{_unigramTokenizer.
BeginningOfSentenceToken
}{inputText}<pad>{inputText}{_unigramTokenizer.EndOfSentenceToken}";
465
expectedTokens[0] = _unigramTokenizerWithSpecialTokens.
BeginningOfSentenceToken
;
472
newString = $"{_unigramTokenizerFromJson.
BeginningOfSentenceToken
}{inputText}<pad>{inputText}{_unigramTokenizerFromJson.EndOfSentenceToken}";
485
expectedTokens[0] = _unigramTokenizerFromJson.
BeginningOfSentenceToken
;
664
inputText = $"{_unigramTokenizerWithSpecialTokens.
BeginningOfSentenceToken
}{inputText}<pad>{inputText}{_unigramTokenizerWithSpecialTokens.EndOfSentenceToken}";
671
string expectedNormalized = $"{_unigramTokenizerWithSpecialTokens.
BeginningOfSentenceToken
}{normalizedText}<pad>{normalizedText}{_unigramTokenizerWithSpecialTokens.EndOfSentenceToken}";
690
expectedNormalized = $"{_unigramTokenizerFromJson.
BeginningOfSentenceToken
}{normalizedText}<pad>{normalizedText}{_unigramTokenizerFromJson.EndOfSentenceToken}";
805
Assert.Equal("<s>", _unigramTokenizer.
BeginningOfSentenceToken
);
816
Assert.Equal("<s>", _unigramTokenizerFromJson.
BeginningOfSentenceToken
);