7 references to Decode
Microsoft.ML.Tokenizers.Tests (7)
LlamaTests.cs (3)
386
Assert.Equal(llamaTokenizer.EndOfSentenceToken, llamaTokenizer.
Decode
([llamaTokenizer.EndOfSentenceId], considerSpecialTokens: true));
839
Assert.Equal(textWithSpecialTokens, tokenizer.
Decode
(expectedIds, considerSpecialTokens: true));
877
Assert.Equal(textWithSpecialTokens, tokenizerWithSuffix.
Decode
(expectedIdsWithSuffix, considerSpecialTokens: true));
UnigramTests.cs (4)
537
string result = tokenizer.
Decode
(ids, considerSpecialTokens: false);
594
string decoded = tokenizer.
Decode
(ids, considerSpecialTokens: false);
1299
string decoded = tokenizer.
Decode
(new[] { 0 }, considerSpecialTokens: true);
1783
Assert.Equal("a", tokenizer.
Decode
(withSuffix, considerSpecialTokens: false));