3 references to EncodeToIds
Microsoft.ML.Tokenizers.Tests (3)
LlamaTests.cs (3)
272
IReadOnlyList<int> encodedIds = bpe.
EncodeToIds
(normalizedInput.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: false, considerNormalization: false);
282
encodedIds = bpe.
EncodeToIds
(normalizedInput.AsSpan(), addBeginningOfSentence: false, addEndOfSentence: true, considerNormalization: false);
292
encodedIds = bpe.
EncodeToIds
(normalizedInput.AsSpan(), addBeginningOfSentence: true, addEndOfSentence: true, considerNormalization: false);