19 references to EndOfSentenceId
Microsoft.ML.GenAI.Phi.Tests (1)
Phi3Tests.cs (1)
122
tokenizer.
EndOfSentenceId
.Should().Be(2);
Microsoft.ML.Tokenizers.Tests (18)
LlamaTests.cs (9)
277
Assert.Equal(isEmptyInput ? Array.Empty<int>() : ids.Skip(1).Concat(new[] { bpe.
EndOfSentenceId
}), bpeTokens.Select(token => token.Id));
283
Assert.Equal(isEmptyInput ? Array.Empty<int>() : ids.Skip(1).Concat(new[] { bpe.
EndOfSentenceId
}), encodedIds);
287
Assert.Equal(isEmptyInput ? Array.Empty<int>() : ids.Concat(new[] { bpe.
EndOfSentenceId
}), bpeTokens.Select(token => token.Id));
293
Assert.Equal(isEmptyInput ? Array.Empty<int>() : ids.Concat(new[] { bpe.
EndOfSentenceId
}), encodedIds);
367
Assert.Equal(2, bpe.
EndOfSentenceId
);
386
Assert.Equal(llamaTokenizer.EndOfSentenceToken, llamaTokenizer.Decode([llamaTokenizer.
EndOfSentenceId
], considerSpecialTokens: true));
388
Assert.Equal(OperationStatus.Done, llamaTokenizer.Decode([llamaTokenizer.
EndOfSentenceId
], destinationBuffer, considerSpecialTokens: true, out int idsConsumed, out int charactersWritten));
539
expectedIds1 = addEndOfSentence ? expectedIds1.Concat(new[] { sentencePieceBpe.
EndOfSentenceId
}).ToArray() : expectedIds1;
573
expectedIds1 = addEndOfSentence ? expectedIds1.Concat(new[] { sentencePieceBpe.
EndOfSentenceId
}).ToArray() : expectedIds1;
SentencePieceTests.cs (5)
84
Assert.Equal(2, tokenizer.
EndOfSentenceId
);
207
Assert.Equal(2, tokenizer.
EndOfSentenceId
);
240
Assert.Equal(7, tokenizer.
EndOfSentenceId
);
482
Assert.Equal(2, tokenizer.
EndOfSentenceId
);
619
Assert.Equal(0, tokenizer.
EndOfSentenceId
);
UnigramTests.cs (4)
300
Assert.True(writableTokens[writableTokens.Count - 1].Id == tokenizer.
EndOfSentenceId
);
362
expectedIds[ids.Length * 2 + 2] = _unigramTokenizerWithSpecialTokens.
EndOfSentenceId
;
470
expectedIds[ids.Length * 2 + 2] = _unigramTokenizerWithSpecialTokens.
EndOfSentenceId
;
563
Assert.Equal(2, _unigramTokenizer.
EndOfSentenceId
);