6 overrides of Decode
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
616public override string Decode(IEnumerable<int> ids) => Decode(ids, considerSpecialTokens: true);
Model\CodeGenTokenizer.cs (1)
1237public override string Decode(IEnumerable<int> ids) => Decode(ids, hasPrefixSpace: AddPrefixSpace, considerSpecialTokens: false);
Model\EnglishRobertaTokenizer.cs (1)
764public override string Decode(IEnumerable<int> ids)
Model\SentencePieceTokenizer.cs (1)
411public override string Decode(IEnumerable<int> ids) => _model.Decode(ids, considerSpecialTokens: false);
Model\TiktokenTokenizer.cs (1)
795public override string Decode(IEnumerable<int> ids)
Model\WordPieceTokenizer.cs (1)
671public override string Decode(IEnumerable<int> ids) => Decode(ids, skipSpecialTokens: false);
23 references to Decode
Microsoft.ML.Tokenizers.Tests (22)
BpeTests.cs (1)
266Assert.Equal(decodedTokens, tokenizer.Decode(encodingIds));
CodeGenTests.cs (2)
254Assert.Equal(text, tokenizer.Decode(ids)); 257Assert.Equal(text, tokenizer.Decode(ids));
EnglishRobertaTests.cs (1)
280Assert.Equal(expectedDecodedString, tokenizer.Decode(encodingIds));
LlamaTests.cs (5)
248Assert.Equal(input, llamaTokenizer.Decode(ids)); 270Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 280Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 290Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 347Assert.Throws<ArgumentNullException>(() => llamaTokenizer.Decode(null!));
TiktokenTests.cs (12)
143Assert.Equal(text, tokenizer.Decode(encoded)!); 196Assert.Equal(text, GPT4.Decode(encoded)); 227Assert.Equal(text, gpt4Tokenizer.Decode(encoded)); 239Assert.Equal(text, GPT4.Decode(encoded)); 274Assert.Equal(text, GPT4.Decode(encoded)); 300Assert.Equal(text, GPT4o.Decode(encoded)); 308Assert.Equal(text, GPT4o.Decode(encoded)); 336Assert.Equal(text, GPT2.Decode(encoded)); 355Assert.Equal(text, P50kBase.Decode(encoded)); 374Assert.Equal(text, P50kEdit.Decode(encoded)); 393Assert.Equal(text, R50kBase.Decode(encoded)); 752Assert.Equal(text, Phi4.Decode(encoded));
TokenizerTests.cs (1)
26tokenizer.Decode(Enumerable.Repeat("abcdefghijklmnopqrstuvwxyz", 100).SelectMany(s => s.Select(c => c - 'a'))));
Microsoft.ML.TorchSharp (1)
Roberta\QATrainer.cs (1)
927outputCache.PredictedAnswersBuffer[index] = new ReadOnlyMemory<char>(_parent.Tokenizer.Decode(_parent.Tokenizer.RobertaModel().ConvertOccurrenceRanksToIds(contextIds).ToArray().AsSpan(predictStart - questionLength - 2, predictEnd - predictStart).ToArray()).Trim().ToCharArray());