6 overrides of Decode
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
773public override string Decode(IEnumerable<int> ids) => Decode(ids, considerSpecialTokens: true);
Model\CodeGenTokenizer.cs (1)
1237public override string Decode(IEnumerable<int> ids) => Decode(ids, hasPrefixSpace: AddPrefixSpace, considerSpecialTokens: false);
Model\EnglishRobertaTokenizer.cs (1)
764public override string Decode(IEnumerable<int> ids)
Model\SentencePieceTokenizer.cs (1)
401public override string Decode(IEnumerable<int> ids) => _model.Decode(ids, considerSpecialTokens: false);
Model\TiktokenTokenizer.cs (1)
795public override string Decode(IEnumerable<int> ids)
Model\WordPieceTokenizer.cs (1)
671public override string Decode(IEnumerable<int> ids) => Decode(ids, skipSpecialTokens: false);
33 references to Decode
Microsoft.ML.GenAI.Core (2)
Pipeline\CausalLMPipeline.cs (2)
276_ => this.Tokenizer.Decode(tokenIds.Concat(tokenIds)) ?? throw new InvalidOperationException("Failed to decode token ids"), 282_ => this.Tokenizer.Decode(tokenIds) ?? throw new InvalidOperationException("Failed to decode token ids"),
Microsoft.ML.Tokenizers.Tests (30)
BpeTests.cs (1)
310Assert.Equal(decodedTokens, tokenizer.Decode(encodingIds));
CodeGenTests.cs (2)
254Assert.Equal(text, tokenizer.Decode(ids)); 257Assert.Equal(text, tokenizer.Decode(ids));
EnglishRobertaTests.cs (1)
280Assert.Equal(expectedDecodedString, tokenizer.Decode(encodingIds));
LlamaTests.cs (5)
248Assert.Equal(input, llamaTokenizer.Decode(ids)); 270Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 280Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 290Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 347Assert.Throws<ArgumentNullException>(() => llamaTokenizer.Decode(null!));
TiktokenTests.cs (17)
148Assert.Equal(text, tokenizer.Decode(encoded)!); 201Assert.Equal(text, GPT4.Decode(encoded)); 232Assert.Equal(text, gpt4Tokenizer.Decode(encoded)); 244Assert.Equal(text, GPT4.Decode(encoded)); 279Assert.Equal(text, GPT4.Decode(encoded)); 344Assert.Equal(text, GPT2.Decode(encoded)); 363Assert.Equal(text, P50kBase.Decode(encoded)); 382Assert.Equal(text, P50kEdit.Decode(encoded)); 401Assert.Equal(text, R50kBase.Decode(encoded)); 796Assert.Equal(text, Phi4.Decode(encoded)); 865string decoded = GPT4.Decode(ids); 871string mixedDecoded = GPT4.Decode(mixedIds); 877string boundaryDecoded = GPT4.Decode(boundaryIds); 883string belowDecoded = GPT4.Decode(belowIds); 889string aboveDecoded = GPT4.Decode(aboveIds); 908string decodedRepeated = GPT4.Decode(idsRepeated); 914string decodedMixed = GPT4.Decode(idsMixed);
TokenizerTests.cs (4)
20Assert.Equal("", tokenizer.Decode([])); 22Assert.Equal("hello", tokenizer.Decode([7, 4, 11, 11, 14])); 26tokenizer.Decode(Enumerable.Repeat("abcdefghijklmnopqrstuvwxyz", 100).SelectMany(s => s.Select(c => c - 'a')))); 28Assert.Throws<InvalidOperationException>(() => tokenizer.Decode([26, 27, 28, 29]));
Microsoft.ML.TorchSharp (1)
Roberta\QATrainer.cs (1)
927outputCache.PredictedAnswersBuffer[index] = new ReadOnlyMemory<char>(_parent.Tokenizer.Decode(_parent.Tokenizer.RobertaModel().ConvertOccurrenceRanksToIds(contextIds).ToArray().AsSpan(predictStart - questionLength - 2, predictEnd - predictStart).ToArray()).Trim().ToCharArray());