6 overrides of Decode
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
616public override string Decode(IEnumerable<int> ids) => Decode(ids, considerSpecialTokens: true);
Model\CodeGenTokenizer.cs (1)
1234public override string Decode(IEnumerable<int> ids) => Decode(ids, hasPrefixSpace: AddPrefixSpace, considerSpecialTokens: false);
Model\EnglishRobertaTokenizer.cs (1)
764public override string Decode(IEnumerable<int> ids)
Model\SentencePieceTokenizer.cs (1)
1530public override string Decode(IEnumerable<int> ids)
Model\TiktokenTokenizer.cs (1)
795public override string Decode(IEnumerable<int> ids)
Model\WordPieceTokenizer.cs (1)
671public override string Decode(IEnumerable<int> ids) => Decode(ids, skipSpecialTokens: false);
22 references to Decode
Microsoft.ML.Tokenizers.Tests (21)
BpeTests.cs (1)
266Assert.Equal(decodedTokens, tokenizer.Decode(encodingIds));
CodeGenTests.cs (2)
254Assert.Equal(text, tokenizer.Decode(ids)); 257Assert.Equal(text, tokenizer.Decode(ids));
EnglishRobertaTests.cs (1)
280Assert.Equal(expectedDecodedString, tokenizer.Decode(encodingIds));
LlamaTests.cs (5)
248Assert.Equal(input, llamaTokenizer.Decode(ids)); 270Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 280Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 290Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 347Assert.Throws<ArgumentNullException>(() => llamaTokenizer.Decode(null!));
TiktokenTests.cs (11)
140Assert.Equal(text, tokenizer.Decode(encoded)!); 193Assert.Equal(text, GPT4.Decode(encoded)); 224Assert.Equal(text, gpt4Tokenizer.Decode(encoded)); 236Assert.Equal(text, GPT4.Decode(encoded)); 271Assert.Equal(text, GPT4.Decode(encoded)); 297Assert.Equal(text, GPT4o.Decode(encoded)); 305Assert.Equal(text, GPT4o.Decode(encoded)); 333Assert.Equal(text, GPT2.Decode(encoded)); 352Assert.Equal(text, P50kBase.Decode(encoded)); 371Assert.Equal(text, P50kEdit.Decode(encoded)); 390Assert.Equal(text, R50kBase.Decode(encoded));
TokenizerTests.cs (1)
26tokenizer.Decode(Enumerable.Repeat("abcdefghijklmnopqrstuvwxyz", 100).SelectMany(s => s.Select(c => c - 'a'))));
Microsoft.ML.TorchSharp (1)
Roberta\QATrainer.cs (1)
927outputCache.PredictedAnswersBuffer[index] = new ReadOnlyMemory<char>(_parent.Tokenizer.Decode(_parent.Tokenizer.RobertaModel().ConvertOccurrenceRanksToIds(contextIds).ToArray().AsSpan(predictStart - questionLength - 2, predictEnd - predictStart).ToArray()).Trim().ToCharArray());