8 references to Decode
Microsoft.ML.GenAI.Core (2)
Pipeline\CausalLMPipeline.cs (2)
275
SentencePieceTokenizer bpeTokenizer => bpeTokenizer.
Decode
(tokenIds.Concat(tokenIds), considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"),
281
SentencePieceTokenizer bpeTokenizer => bpeTokenizer.
Decode
(tokenIds, considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"),
Microsoft.ML.GenAI.Mistral.Tests (1)
Mistral_7B_Instruct_V0_3Tests.cs (1)
129
var decodeToString = tokenizer.
Decode
(tokenizeIds, considerSpecialTokens: true);
Microsoft.ML.GenAI.Phi.Tests (1)
Phi3Tests.cs (1)
143
var decodeToString = tokenizer.
Decode
(tokenizeIds, considerSpecialTokens: true);
Microsoft.ML.Tokenizers.Tests (4)
LlamaTests.cs (3)
386
Assert.Equal(llamaTokenizer.EndOfSentenceToken, llamaTokenizer.
Decode
([llamaTokenizer.EndOfSentenceId], considerSpecialTokens: true));
839
Assert.Equal(textWithSpecialTokens, tokenizer.
Decode
(expectedIds, considerSpecialTokens: true));
877
Assert.Equal(textWithSpecialTokens, tokenizerWithSuffix.
Decode
(expectedIdsWithSuffix, considerSpecialTokens: true));
UnigramTests.cs (1)
537
string result = tokenizer.
Decode
(ids, considerSpecialTokens: false);