8 references to Decode
Microsoft.ML.GenAI.Core (2)
Pipeline\CausalLMPipeline.cs (2)
275SentencePieceTokenizer bpeTokenizer => bpeTokenizer.Decode(tokenIds.Concat(tokenIds), considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"), 281SentencePieceTokenizer bpeTokenizer => bpeTokenizer.Decode(tokenIds, considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"),
Microsoft.ML.GenAI.Mistral.Tests (1)
Mistral_7B_Instruct_V0_3Tests.cs (1)
129var decodeToString = tokenizer.Decode(tokenizeIds, considerSpecialTokens: true);
Microsoft.ML.GenAI.Phi.Tests (1)
Phi3Tests.cs (1)
143var decodeToString = tokenizer.Decode(tokenizeIds, considerSpecialTokens: true);
Microsoft.ML.Tokenizers.Tests (4)
LlamaTests.cs (3)
386Assert.Equal(llamaTokenizer.EndOfSentenceToken, llamaTokenizer.Decode([llamaTokenizer.EndOfSentenceId], considerSpecialTokens: true)); 839Assert.Equal(textWithSpecialTokens, tokenizer.Decode(expectedIds, considerSpecialTokens: true)); 877Assert.Equal(textWithSpecialTokens, tokenizerWithSuffix.Decode(expectedIdsWithSuffix, considerSpecialTokens: true));
UnigramTests.cs (1)
537string result = tokenizer.Decode(ids, considerSpecialTokens: false);