6 references to DefaultSpecialToken
Microsoft.ML.Tokenizers.Tests (6)
CodeGenTests.cs (6)
974
Assert.Equal(codeGenTokenizer.EncodeToIds(
DefaultSpecialToken
)[0], codeGenTokenizer.BeginningOfSentenceId!.Value);
975
Assert.Equal(codeGenTokenizer.EncodeToIds(
DefaultSpecialToken
)[0], codeGenTokenizer.EndOfSentenceId!.Value);
976
Assert.Equal(codeGenTokenizer.EncodeToIds(
DefaultSpecialToken
)[0], codeGenTokenizer.UnknownTokenId!.Value);
978
Assert.Equal(
DefaultSpecialToken
, codeGenTokenizer.BeginningOfSentenceToken);
979
Assert.Equal(
DefaultSpecialToken
, codeGenTokenizer.EndOfSentenceToken);
980
Assert.Equal(
DefaultSpecialToken
, codeGenTokenizer.UnknownToken);