12 references to BeginningOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi3Tests.cs (1)
121
tokenizer.
BeginningOfSentenceId
.Should().Be(1);
Microsoft.ML.Tokenizers.Tests (11)
LlamaTests.cs (2)
365
Assert.Equal(1, bpe.
BeginningOfSentenceId
);
666
Assert.Equal(new[] { tokenizer.
BeginningOfSentenceId
, kvp.Value }, encodedTokens.Select(et => et.Id).ToArray());
SentencePieceTests.cs (5)
83
Assert.Equal(1, tokenizer.
BeginningOfSentenceId
);
206
Assert.Equal(1, tokenizer.
BeginningOfSentenceId
);
239
Assert.Equal(6, tokenizer.
BeginningOfSentenceId
);
481
Assert.Equal(1, tokenizer.
BeginningOfSentenceId
);
618
Assert.Equal(0, tokenizer.
BeginningOfSentenceId
);
UnigramTests.cs (4)
292
Assert.True(writableTokens[0].Id == tokenizer.
BeginningOfSentenceId
);
358
expectedIds[0] = _unigramTokenizerWithSpecialTokens.
BeginningOfSentenceId
;
466
expectedIds[0] = _unigramTokenizerWithSpecialTokens.
BeginningOfSentenceId
;
561
Assert.Equal(1, _unigramTokenizer.
BeginningOfSentenceId
);