12 references to BeginningOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi3Tests.cs (1)
121
tokenizer.
BeginningOfSentenceId
.Should().Be(1);
Microsoft.ML.Tokenizers.Tests (11)
LlamaTests.cs (2)
365
Assert.Equal(1, bpe.
BeginningOfSentenceId
);
666
Assert.Equal(new[] { tokenizer.
BeginningOfSentenceId
, kvp.Value }, encodedTokens.Select(et => et.Id).ToArray());
UnigramTests.cs (9)
334
Assert.True(writableTokens[0].Id == tokenizer.
BeginningOfSentenceId
);
380
else if (ids[i] == _unigramTokenizer.
BeginningOfSentenceId
)
382
shiftedIds[i] = _unigramTokenizerFromJson.
BeginningOfSentenceId
;
457
expectedIds[0] = _unigramTokenizerWithSpecialTokens.
BeginningOfSentenceId
;
477
expectedIds[0] = _unigramTokenizerFromJson.
BeginningOfSentenceId
;
666
expectedIds[0] = _unigramTokenizerWithSpecialTokens.
BeginningOfSentenceId
;
685
expectedIds[0] = _unigramTokenizerFromJson.
BeginningOfSentenceId
;
806
Assert.Equal(1, _unigramTokenizer.
BeginningOfSentenceId
);
817
Assert.Equal(0, _unigramTokenizerFromJson.
BeginningOfSentenceId
);