6 references to Decode
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
671public override string Decode(IEnumerable<int> ids) => Decode(ids, skipSpecialTokens: false);
Microsoft.ML.Tokenizers.Tests (5)
BertTokenizerTests.cs (5)
74Assert.Equal("hello, how are you?", tokenizer.Decode(ids, skipSpecialTokens: true)); 146Assert.Equal("hello, how are you?", tokenizer.Decode(ids, skipSpecialTokens: true)); 214Assert.Equal(", are you?", tokenizer.Decode(ids, skipSpecialTokens: true)); 326Assert.Equal("叟 驷 叢 驸!", bertTokenizer.Decode(bertTokenizer.EncodeToIds(text), skipSpecialTokens: true)); 344Assert.Equal("叟驷 叢驸!", bertTokenizer.Decode(bertTokenizer.EncodeToIds(text), skipSpecialTokens: true));