5 overrides of Decode
Microsoft.ML.Tokenizers (5)
Model\BPETokenizer.cs (1)
571public override string? Decode(IEnumerable<int> ids) => Decode(ids, considerSpecialTokens: true);
Model\CodeGenTokenizer.cs (1)
1232public override string? Decode(IEnumerable<int> ids) => Decode(ids, hasPrefixSpace: AddPrefixSpace, considerSpecialTokens: false);
Model\EnglishRobertaTokenizer.cs (1)
753public override string? Decode(IEnumerable<int> ids)
Model\SentencePieceBpeTokenizer.cs (1)
1529public override string? Decode(IEnumerable<int> ids)
Model\TiktokenTokenizer.cs (1)
794public override string? Decode(IEnumerable<int> ids)
22 references to Decode
Microsoft.ML.Tokenizers.Tests (21)
BpeTests.cs (1)
266Assert.Equal(decodedTokens, tokenizer.Decode(encodingIds));
CodeGenTests.cs (2)
254Assert.Equal(text, tokenizer.Decode(ids)); 257Assert.Equal(text, tokenizer.Decode(ids));
EnglishRobertaTests.cs (1)
280Assert.Equal(expectedDecodedString, tokenizer.Decode(encodingIds));
LlamaTests.cs (5)
248Assert.Equal(input, llamaTokenizer.Decode(ids)); 270Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 280Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 290Assert.Equal(input, llamaTokenizer.Decode(extractedIds)); 347Assert.Throws<ArgumentNullException>(() => llamaTokenizer.Decode(null!));
TitokenTests.cs (11)
139Assert.Equal(text, tokenizer.Decode(encoded)!); 192Assert.Equal(text, GPT4.Decode(encoded)); 223Assert.Equal(text, gpt4Tokenizer.Decode(encoded)); 235Assert.Equal(text, GPT4.Decode(encoded)); 270Assert.Equal(text, GPT4.Decode(encoded)); 296Assert.Equal(text, GPT4o.Decode(encoded)); 304Assert.Equal(text, GPT4o.Decode(encoded)); 332Assert.Equal(text, GPT2.Decode(encoded)); 351Assert.Equal(text, P50kBase.Decode(encoded)); 370Assert.Equal(text, P50kEdit.Decode(encoded)); 389Assert.Equal(text, R50kBase.Decode(encoded));
TokenizerTests.cs (1)
26tokenizer.Decode(Enumerable.Repeat("abcdefghijklmnopqrstuvwxyz", 100).SelectMany(s => s.Select(c => c - 'a'))));
Microsoft.ML.TorchSharp (1)
Roberta\QATrainer.cs (1)
927outputCache.PredictedAnswersBuffer[index] = new ReadOnlyMemory<char>(_parent.Tokenizer.Decode(_parent.Tokenizer.RobertaModel().ConvertOccurrenceRanksToIds(contextIds).ToArray().AsSpan(predictStart - questionLength - 2, predictEnd - predictStart).ToArray()).Trim().ToCharArray());