5 references to Decode
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
726
public override string Decode(IEnumerable<int> ids) =>
Decode
(ids, skipSpecialTokens: false);
Microsoft.ML.Tokenizers.Tests (4)
BertTokenizerTests.cs (4)
57
Assert.Equal("hello, how are you?", tokenizer.
Decode
(ids, skipSpecialTokens: true));
124
Assert.Equal(", are you?", tokenizer.
Decode
(ids, skipSpecialTokens: true));
236
Assert.Equal("叟 驷 叢 驸!", bertTokenizer.
Decode
(bertTokenizer.EncodeToIds(text), skipSpecialTokens: true));
254
Assert.Equal("叟驷 叢驸!", bertTokenizer.
Decode
(bertTokenizer.EncodeToIds(text), skipSpecialTokens: true));