Base:
method
Decode
Microsoft.ML.Tokenizers.Tokenizer.Decode(System.Collections.Generic.IEnumerable<System.Int32>)
10 references to Decode
Microsoft.ML.Tokenizers.Tests (10)
LlamaTests.cs (2)
837Assert.Equal(decodedWithNoSpecialTokens, tokenizer.Decode(expectedIds)); 876Assert.Equal(decodedWithNoSpecialTokens, tokenizerWithSuffix.Decode(expectedIdsWithSuffix));
SentencePieceTests.cs (5)
49Assert.Equal("Hello", tokenizer.Decode(tokens.Select(t => t.Id))); 63Assert.Equal("Hello", tokenizer.Decode(tokens.Select(t => t.Id))); 133string decoded = tokenizer.Decode(tokens.Select(t => t.Id)); 165Assert.Equal(input, bpe.Decode(bpeTokens.Select(t => t.Id))); 166Assert.Equal(input, unigram.Decode(unigramTokens.Select(t => t.Id)));
UnigramTests.cs (3)
1026Assert.Equal(text, tokenizer.Decode(ids)); 1056Assert.Equal(text, tokenizer.Decode(ids)); 1683Assert.Equal("<0x00>a", tokenizer.Decode(new[] { 1, 2 }));