6 references to Decode
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
773
public override string Decode(IEnumerable<int> ids) =>
Decode
(ids, considerSpecialTokens: true);
Microsoft.ML.Tokenizers.Tests (5)
BpeTests.cs (5)
311
Assert.Equal(decodedTokensWithoutUnknownToken, bpe.
Decode
(encodingIds, considerSpecialTokens: false));
846
Assert.Equal(text, tokenizer.
Decode
(ids, considerSpecialTokens: false));
885
Assert.Equal(text, tokenizer.
Decode
(ids, considerSpecialTokens: false));
922
Assert.Equal("Helloworld!", bpeTokenizer.
Decode
(ids, considerSpecialTokens: false));
945
Assert.Equal("Helloworld!", bpeTokenizer.
Decode
(ids, considerSpecialTokens: false));