5 overrides of Decode
Microsoft.ML.Tokenizers (5)
Model\BPETokenizer.cs (1)
571
public override string?
Decode
(IEnumerable<int> ids) => Decode(ids, considerSpecialTokens: true);
Model\CodeGenTokenizer.cs (1)
1232
public override string?
Decode
(IEnumerable<int> ids) => Decode(ids, hasPrefixSpace: AddPrefixSpace, considerSpecialTokens: false);
Model\EnglishRobertaTokenizer.cs (1)
753
public override string?
Decode
(IEnumerable<int> ids)
Model\SentencePieceBpeTokenizer.cs (1)
1529
public override string?
Decode
(IEnumerable<int> ids)
Model\TiktokenTokenizer.cs (1)
794
public override string?
Decode
(IEnumerable<int> ids)
22 references to Decode
Microsoft.ML.Tokenizers.Tests (21)
BpeTests.cs (1)
266
Assert.Equal(decodedTokens, tokenizer.
Decode
(encodingIds));
CodeGenTests.cs (2)
254
Assert.Equal(text, tokenizer.
Decode
(ids));
257
Assert.Equal(text, tokenizer.
Decode
(ids));
EnglishRobertaTests.cs (1)
280
Assert.Equal(expectedDecodedString, tokenizer.
Decode
(encodingIds));
LlamaTests.cs (5)
248
Assert.Equal(input, llamaTokenizer.
Decode
(ids));
270
Assert.Equal(input, llamaTokenizer.
Decode
(extractedIds));
280
Assert.Equal(input, llamaTokenizer.
Decode
(extractedIds));
290
Assert.Equal(input, llamaTokenizer.
Decode
(extractedIds));
347
Assert.Throws<ArgumentNullException>(() => llamaTokenizer.
Decode
(null!));
TitokenTests.cs (11)
139
Assert.Equal(text, tokenizer.
Decode
(encoded)!);
192
Assert.Equal(text, GPT4.
Decode
(encoded));
223
Assert.Equal(text, gpt4Tokenizer.
Decode
(encoded));
235
Assert.Equal(text, GPT4.
Decode
(encoded));
270
Assert.Equal(text, GPT4.
Decode
(encoded));
296
Assert.Equal(text, GPT4o.
Decode
(encoded));
304
Assert.Equal(text, GPT4o.
Decode
(encoded));
332
Assert.Equal(text, GPT2.
Decode
(encoded));
351
Assert.Equal(text, P50kBase.
Decode
(encoded));
370
Assert.Equal(text, P50kEdit.
Decode
(encoded));
389
Assert.Equal(text, R50kBase.
Decode
(encoded));
TokenizerTests.cs (1)
26
tokenizer.
Decode
(Enumerable.Repeat("abcdefghijklmnopqrstuvwxyz", 100).SelectMany(s => s.Select(c => c - 'a'))));
Microsoft.ML.TorchSharp (1)
Roberta\QATrainer.cs (1)
927
outputCache.PredictedAnswersBuffer[index] = new ReadOnlyMemory<char>(_parent.Tokenizer.
Decode
(_parent.Tokenizer.RobertaModel().ConvertOccurrenceRanksToIds(contextIds).ToArray().AsSpan(predictStart - questionLength - 2, predictEnd - predictStart).ToArray()).Trim().ToCharArray());