1 write to EndOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBpeTokenizer.cs (1)
65
EndOfSentenceToken
= modelProto.TrainerSpec.EosPiece ?? "</s>";
12 references to EndOfSentenceToken
Microsoft.ML.Tokenizers (4)
Model\SentencePieceBpeTokenizer.cs (4)
302
tokens.Add(new EncodedToken(EndOfSentenceId,
EndOfSentenceToken
, (text.Length, 0)));
367
tokens.Add(new EncodedToken(EndOfSentenceId,
EndOfSentenceToken
, (text.Length, 0)));
1751
sb.Append(tokenizer.
EndOfSentenceToken
);
1992
specialToken = tokenizer.
EndOfSentenceToken
;
Microsoft.ML.Tokenizers.Tests (8)
LlamaTests.cs (8)
278
Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Skip(1).Concat(new[] { bpe.
EndOfSentenceToken
}), bpeTokens.Select(token => token.Value));
288
Assert.Equal(isEmptyInput ? Array.Empty<string>() : tokens.Concat(new[] { bpe.
EndOfSentenceToken
}), bpeTokens.Select(token => token.Value));
368
Assert.Equal("</s>", bpe.
EndOfSentenceToken
);
386
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
, llamaTokenizer.Decode([llamaTokenizer.EndOfSentenceId], considerSpecialTokens: true));
387
Span<char> destinationBuffer = stackalloc char[llamaTokenizer.
EndOfSentenceToken
.Length];
389
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
.Length, charactersWritten);
390
Assert.Equal(llamaTokenizer.
EndOfSentenceToken
, destinationBuffer.ToString());
533
expectedTokens1 = addEndOfSentence ? expectedTokens1.Concat(new[] { sentencePieceBpe.
EndOfSentenceToken
}).ToArray() : expectedTokens1;