7 references to Decode
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
888=> Decode(ids, destination, considerSpecialTokens: true, out idsConsumed, out charsWritten);
Microsoft.ML.Tokenizers.Tests (6)
BpeTests.cs (6)
341status = bpe.Decode(ids, destinationBuffer.AsSpan().Slice(0, i), considerSpecialTokens, out idsConsumed, out charactersWritten); 351status = bpe.Decode(ids, destinationBuffer.AsSpan(), considerSpecialTokens, out idsConsumed, out charactersWritten); 849Assert.Equal(OperationStatus.Done, tokenizer.Decode(ids, destinationBuffer.AsSpan(), considerSpecialTokens: true, out int idsConsumed, out int charsWritten)); 852Assert.Equal(OperationStatus.Done, tokenizer.Decode(ids, destinationBuffer.AsSpan(), considerSpecialTokens: false, out idsConsumed, out charsWritten)); 855Assert.Equal(OperationStatus.DestinationTooSmall, tokenizer.Decode(ids, destinationBuffer.AsSpan(0, text.Length - 1), considerSpecialTokens: false, out idsConsumed, out charsWritten)); 858Assert.Equal(OperationStatus.DestinationTooSmall, tokenizer.Decode(ids, destinationBuffer.AsSpan(0, text.Length), considerSpecialTokens: true, out idsConsumed, out charsWritten));