5 references to Decode
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBpeTokenizer.cs (1)
1773=> Decode(ids, destination, considerSpecialTokens: false, out idsConsumed, out charsWritten);
Microsoft.ML.Tokenizers.Tests (4)
LlamaTests.cs (4)
862Assert.Equal(OperationStatus.DestinationTooSmall, tokenizer.Decode(ids, destinationBuffer.AsSpan().Slice(0, i), considerSpecialTokens: true, out idsConsumed, out charactersWritten)); 867Assert.Equal(OperationStatus.Done, tokenizer.Decode(ids, destinationBuffer.AsSpan(), considerSpecialTokens: true, out idsConsumed, out charactersWritten)); 901Assert.Equal(OperationStatus.DestinationTooSmall, tokenizerWithSuffix.Decode(ids, destinationBuffer.AsSpan().Slice(0, i), considerSpecialTokens: true, out idsConsumed, out charactersWritten)); 911Assert.Equal(OperationStatus.Done, tokenizerWithSuffix.Decode(ids, destinationBuffer.AsSpan(), considerSpecialTokens: true, out idsConsumed, out charactersWritten));