7 references to BeginningOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi3Tests.cs (1)
121
tokenizer.
BeginningOfSentenceId
.Should().Be(1);
Microsoft.ML.Tokenizers.Tests (6)
LlamaTests.cs (2)
365
Assert.Equal(1, bpe.
BeginningOfSentenceId
);
666
Assert.Equal(new[] { tokenizer.
BeginningOfSentenceId
, kvp.Value }, encodedTokens.Select(et => et.Id).ToArray());
UnigramTests.cs (4)
292
Assert.True(writableTokens[0].Id == tokenizer.
BeginningOfSentenceId
);
358
expectedIds[0] = _unigramTokenizerWithSpecialTokens.
BeginningOfSentenceId
;
466
expectedIds[0] = _unigramTokenizerWithSpecialTokens.
BeginningOfSentenceId
;
561
Assert.Equal(1, _unigramTokenizer.
BeginningOfSentenceId
);