1 write to BeginningOfSentenceId
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
64
BeginningOfSentenceId
= modelProto.TrainerSpec.BosId <= 0 ? 1 : modelProto.TrainerSpec.BosId;
9 references to BeginningOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi3Tests.cs (1)
121
tokenizer.
BeginningOfSentenceId
.Should().Be(1);
Microsoft.ML.Tokenizers (6)
Model\SentencePieceTokenizer.cs (6)
275
tokens.Add(new EncodedToken(
BeginningOfSentenceId
, BeginningOfSentenceToken, new Range(0, 0)));
322
tokens.Add(new EncodedToken(
BeginningOfSentenceId
, BeginningOfSentenceToken, new Range(0, 0)));
602
accumulatedIds.Add(
BeginningOfSentenceId
);
666
accumulatedIds.Add(
BeginningOfSentenceId
);
1745
if (id == tokenizer.
BeginningOfSentenceId
)
1986
if (id == tokenizer.
BeginningOfSentenceId
)
Microsoft.ML.Tokenizers.Tests (2)
LlamaTests.cs (2)
365
Assert.Equal(1, bpe.
BeginningOfSentenceId
);
666
Assert.Equal(new[] { tokenizer.
BeginningOfSentenceId
, kvp.Value }, encodedTokens.Select(et => et.Id).ToArray());