5 references to Decode
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
1234
public override string Decode(IEnumerable<int> ids) =>
Decode
(ids, hasPrefixSpace: AddPrefixSpace, considerSpecialTokens: false);
Microsoft.ML.Tokenizers.Tests (4)
CodeGenTests.cs (4)
286
Assert.Equal(text, codeGenTokenizer.
Decode
(ids, hasPrefixSpace: true, considerSpecialTokens: false));
289
Assert.Equal(text, codeGenTokenizer.
Decode
(ids, hasPrefixSpace: true, considerSpecialTokens: false));
295
Assert.Equal(targetText, codeGenTokenizer.
Decode
(ids, hasPrefixSpace: true, considerSpecialTokens: true));
298
Assert.Equal(targetText, codeGenTokenizer.
Decode
(ids, hasPrefixSpace: true, considerSpecialTokens: true));