1 write to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
63BeginningOfSentenceToken = modelProto.TrainerSpec.BosPiece ?? "<s>";
7 references to BeginningOfSentenceToken
Microsoft.ML.Tokenizers (4)
Model\SentencePieceTokenizer.cs (4)
276tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 323tokens.Add(new EncodedToken(BeginningOfSentenceId, BeginningOfSentenceToken, new Range(0, 0))); 1748sb.Append(tokenizer.BeginningOfSentenceToken); 1989specialToken = tokenizer.BeginningOfSentenceToken;
Microsoft.ML.Tokenizers.Tests (3)
LlamaTests.cs (3)
366Assert.Equal("<s>", bpe.BeginningOfSentenceToken); 665Assert.Equal(new[] { tokenizer.BeginningOfSentenceToken, kvp.Key }, encodedTokens.Select(et => et.Value).ToArray()); 838string textWithSpecialTokens = $"{tokenizer.BeginningOfSentenceToken}{text}";