Base:
method
Decode
Microsoft.ML.Tokenizers.Tokenizer.Decode(System.Collections.Generic.IEnumerable<System.Int32>)
4 references to Decode
Microsoft.ML.Tokenizers.Tests (4)
BpeTests.cs (4)
845
Assert.Equal(beginningOfSentenceToken + text, tokenizer.
Decode
(ids));
884
Assert.Equal(beginningOfSentenceToken + textWithSpecialTokens, tokenizer.
Decode
(ids));
921
Assert.Equal("<|sos|>Hellounkworld!unk<|eos|>", bpeTokenizer.
Decode
(ids));
944
Assert.Equal("<|sos|>Helloworld!<|eos|>", bpeTokenizer.
Decode
(ids));
