Base:
method
Decode
Microsoft.ML.Tokenizers.Tokenizer.Decode(System.Collections.Generic.IEnumerable<System.Int32>)
10 references to Decode
Microsoft.ML.Tokenizers.Tests (10)
BertTokenizerTests.cs (7)
73Assert.Equal("[CLS] hello, how are you [SPECIAL]? [SEP]", tokenizer.Decode(ids)); 76tokens = tokenizer.EncodeToTokens(tokenizer.Decode(ids), out normalizedText); 145Assert.Equal("[CLS] hello, how are you? [SEP]", tokenizer.Decode(ids)); 148tokens = tokenizer.EncodeToTokens(tokenizer.Decode(ids), out normalizedText); 213Assert.Equal("[CLS] [UNK], [UNK] are you? [SEP]", tokenizer.Decode(ids)); 325Assert.Equal("[CLS] 叟 驷 叢 驸! [SEP]", bertTokenizer.Decode(bertTokenizer.EncodeToIds(text))); 343Assert.Equal("[CLS] 叟驷 叢驸! [SEP]", bertTokenizer.Decode(bertTokenizer.EncodeToIds(text)));
WordPieceTests.cs (3)
96Assert.Equal(text, tokenizer.Decode(ids)); 169Assert.Equal("[UNK] running", tokenizer.Decode(ids)); 213Assert.Equal("[UNK] unwanted [SEP] [CLS] running [CLS]", tokenizer.Decode(ids));