4 references to Decode
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
768
public override string Decode(IEnumerable<int> ids) =>
Decode
(ids, considerSpecialTokens: true);
Microsoft.ML.Tokenizers.Tests (3)
BpeTests.cs (3)
298
Assert.Equal(decodedTokensWithoutUnknownToken, bpe.
Decode
(encodingIds, considerSpecialTokens: false));
833
Assert.Equal(text, tokenizer.
Decode
(ids, considerSpecialTokens: false));
872
Assert.Equal(text, tokenizer.
Decode
(ids, considerSpecialTokens: false));