14 references to EndOfSentenceToken
Microsoft.ML.Tokenizers.Tests (14)
LlamaTests.cs (8)
278
Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Skip(1).Concat(new[] { bpe.
EndOfSentenceToken
}), bpeTokens.Select(token => token.Value));
288
Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Concat(new[] { bpe.
EndOfSentenceToken
}), bpeTokens.Select(token => token.Value));
368
Assert.Equal("</s>", bpe.
EndOfSentenceToken
);
386
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
, llamaTokenizer.Decode([llamaTokenizer.EndOfSentenceId], considerSpecialTokens: true));
387
Span<char> destinationBuffer = stackalloc char[llamaTokenizer.
EndOfSentenceToken
.Length];
389
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
.Length, charactersWritten);
390
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
, destinationBuffer.ToString());
533
expectedTokens1 = addEndOfSentence ? expectedTokens1.Concat(new[] { sentencePieceBpe.
EndOfSentenceToken
}).ToArray() : expectedTokens1;
UnigramTests.cs (6)
301
Assert.True(writableTokens[writableTokens.Count - 1].Value == tokenizer.
EndOfSentenceToken
);
353
string newString = $"{_unigramTokenizer.BeginningOfSentenceToken}{inputText}<pad>{inputText}{_unigramTokenizer.
EndOfSentenceToken
}";
370
expectedTokens[tokens.Length * 2 + 2] = _unigramTokenizerWithSpecialTokens.
EndOfSentenceToken
;
464
inputText = $"{_unigramTokenizerWithSpecialTokens.BeginningOfSentenceToken}{inputText}<pad>{inputText}{_unigramTokenizerWithSpecialTokens.
EndOfSentenceToken
}";
471
string expectedNormalized = $"{_unigramTokenizerWithSpecialTokens.BeginningOfSentenceToken}{normalizedText}<pad>{normalizedText}{_unigramTokenizerWithSpecialTokens.
EndOfSentenceToken
}";
562
Assert.Equal("</s>", _unigramTokenizer.
EndOfSentenceToken
);