13 references to EncodeToIds
Microsoft.ML.Tokenizers.Tests (13)
BertTokenizerTests.cs (13)
53
var ids = tokenizer.
EncodeToIds
(text);
74
ids = tokenizer.
EncodeToIds
(normalizedText!);
121
var ids = tokenizer.
EncodeToIds
(text);
235
IReadOnlyList<int> ids = bertTokenizer.
EncodeToIds
(text);
236
Assert.Equal("[CLS] 叟 驷 叢 驸! [SEP]", bertTokenizer.Decode(bertTokenizer.
EncodeToIds
(text)));
237
Assert.Equal("叟 驷 叢 驸!", bertTokenizer.Decode(bertTokenizer.
EncodeToIds
(text), skipSpecialTokens: true));
253
ids = bertTokenizer.
EncodeToIds
(text);
254
Assert.Equal("[CLS] 叟驷 叢驸! [SEP]", bertTokenizer.Decode(bertTokenizer.
EncodeToIds
(text)));
255
Assert.Equal("叟驷 叢驸!", bertTokenizer.Decode(bertTokenizer.
EncodeToIds
(text), skipSpecialTokens: true));
279
var ids1 = bertTokenizer.
EncodeToIds
(text1);
282
var ids2 = bertTokenizer.
EncodeToIds
(text2);
379
var ids1 = bertTokenizer.
EncodeToIds
(text1);
382
var ids2 = bertTokenizer.
EncodeToIds
(text2);