18 references to EndOfSentenceToken
Microsoft.ML.Tokenizers.Tests (18)
LlamaTests.cs (8)
278
Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Skip(1).Concat(new[] { bpe.
EndOfSentenceToken
}), bpeTokens.Select(token => token.Value));
288
Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Concat(new[] { bpe.
EndOfSentenceToken
}), bpeTokens.Select(token => token.Value));
368
Assert.Equal("</s>", bpe.
EndOfSentenceToken
);
386
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
, llamaTokenizer.Decode([llamaTokenizer.EndOfSentenceId], considerSpecialTokens: true));
387
Span<char> destinationBuffer = stackalloc char[llamaTokenizer.
EndOfSentenceToken
.Length];
389
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
.Length, charactersWritten);
390
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
, destinationBuffer.ToString());
533
expectedTokens1 = addEndOfSentence ? expectedTokens1.Concat(new[] { sentencePieceBpe.
EndOfSentenceToken
}).ToArray() : expectedTokens1;
UnigramTests.cs (10)
343
Assert.True(writableTokens[writableTokens.Count - 1].Value == tokenizer.
EndOfSentenceToken
);
452
string newString = $"{_unigramTokenizer.BeginningOfSentenceToken}{inputText}<pad>{inputText}{_unigramTokenizer.
EndOfSentenceToken
}";
469
expectedTokens[tokens.Length * 2 + 2] = _unigramTokenizerWithSpecialTokens.
EndOfSentenceToken
;
472
newString = $"{_unigramTokenizerFromJson.BeginningOfSentenceToken}{inputText}<pad>{inputText}{_unigramTokenizerFromJson.
EndOfSentenceToken
}";
489
expectedTokens[tokens.Length * 2 + 2] = _unigramTokenizerFromJson.
EndOfSentenceToken
;
664
inputText = $"{_unigramTokenizerWithSpecialTokens.BeginningOfSentenceToken}{inputText}<pad>{inputText}{_unigramTokenizerWithSpecialTokens.
EndOfSentenceToken
}";
671
string expectedNormalized = $"{_unigramTokenizerWithSpecialTokens.BeginningOfSentenceToken}{normalizedText}<pad>{normalizedText}{_unigramTokenizerWithSpecialTokens.
EndOfSentenceToken
}";
690
expectedNormalized = $"{_unigramTokenizerFromJson.BeginningOfSentenceToken}{normalizedText}<pad>{normalizedText}{_unigramTokenizerFromJson.
EndOfSentenceToken
}";
807
Assert.Equal("</s>", _unigramTokenizer.
EndOfSentenceToken
);
818
Assert.Equal("</s>", _unigramTokenizerFromJson.
EndOfSentenceToken
);