6 references to Decode
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
773public override string Decode(IEnumerable<int> ids) => Decode(ids, considerSpecialTokens: true);
Microsoft.ML.Tokenizers.Tests (5)
BpeTests.cs (5)
311Assert.Equal(decodedTokensWithoutUnknownToken, bpe.Decode(encodingIds, considerSpecialTokens: false)); 846Assert.Equal(text, tokenizer.Decode(ids, considerSpecialTokens: false)); 885Assert.Equal(text, tokenizer.Decode(ids, considerSpecialTokens: false)); 922Assert.Equal("Helloworld!", bpeTokenizer.Decode(ids, considerSpecialTokens: false)); 945Assert.Equal("Helloworld!", bpeTokenizer.Decode(ids, considerSpecialTokens: false));