Base:
method
Decode
Microsoft.ML.Tokenizers.Tokenizer.Decode(System.Collections.Generic.IEnumerable<System.Int32>)
8 references to Decode
Microsoft.ML.Tokenizers.Tests (8)
BertTokenizerTests.cs (5)
56Assert.Equal("[CLS] hello, how are you? [SEP]", tokenizer.Decode(ids)); 59tokens = tokenizer.EncodeToTokens(tokenizer.Decode(ids), out normalizedText); 123Assert.Equal("[CLS] [UNK], [UNK] are you? [SEP]", tokenizer.Decode(ids)); 235Assert.Equal("[CLS] 叟 驷 叢 驸! [SEP]", bertTokenizer.Decode(bertTokenizer.EncodeToIds(text))); 253Assert.Equal("[CLS] 叟驷 叢驸! [SEP]", bertTokenizer.Decode(bertTokenizer.EncodeToIds(text)));
WordPieceTests.cs (3)
96Assert.Equal(text, tokenizer.Decode(ids)); 169Assert.Equal("[UNK] running", tokenizer.Decode(ids)); 213Assert.Equal("[UNK] unwanted [SEP] [CLS] running [CLS]", tokenizer.Decode(ids));