15 references to EncodeToIds
Microsoft.ML.Tokenizers.Tests (15)
BertTokenizerTests.cs (15)
70
var ids = tokenizer.
EncodeToIds
(text);
92
ids = tokenizer.
EncodeToIds
(normalizedText!);
142
var ids = tokenizer.
EncodeToIds
(text);
163
ids = tokenizer.
EncodeToIds
(normalizedText!);
210
var ids = tokenizer.
EncodeToIds
(text);
324
IReadOnlyList<int> ids = bertTokenizer.
EncodeToIds
(text);
325
Assert.Equal("[CLS] 叟 驷 叢 驸! [SEP]", bertTokenizer.Decode(bertTokenizer.
EncodeToIds
(text)));
326
Assert.Equal("叟 驷 叢 驸!", bertTokenizer.Decode(bertTokenizer.
EncodeToIds
(text), skipSpecialTokens: true));
342
ids = bertTokenizer.
EncodeToIds
(text);
343
Assert.Equal("[CLS] 叟驷 叢驸! [SEP]", bertTokenizer.Decode(bertTokenizer.
EncodeToIds
(text)));
344
Assert.Equal("叟驷 叢驸!", bertTokenizer.Decode(bertTokenizer.
EncodeToIds
(text), skipSpecialTokens: true));
368
var ids1 = bertTokenizer.
EncodeToIds
(text1);
371
var ids2 = bertTokenizer.
EncodeToIds
(text2);
468
var ids1 = bertTokenizer.
EncodeToIds
(text1);
471
var ids2 = bertTokenizer.
EncodeToIds
(text2);