29 references to EncodeToTokens
Microsoft.ML.Tokenizers.Tests (26)
BpeTests.cs (3)
257
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(sentence, out _);
376
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(text, out _);
429
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(text, out _);
CodeGenTests.cs (5)
252
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(text, out _);
347
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(text, out _);
549
IReadOnlyList<EncodedToken> encoding = codeGenTokenizer.
EncodeToTokens
(text, out _);
682
encoding = codeGenTokenizer.
EncodeToTokens
(text, out _);
815
encoding = codeGenTokenizer.
EncodeToTokens
(text, out _);
EnglishRobertaTests.cs (4)
181
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(text, out _);
249
encoding = tokenizer.
EncodeToTokens
((string)p[0], out _);
256
encoding = tokenizer.
EncodeToTokens
((string)p[0], out _);
263
encoding = tokenizer.
EncodeToTokens
((string)p[0], out _);
LlamaTests.cs (4)
244
IReadOnlyList<EncodedToken> result = llamaTokenizer.
EncodeToTokens
(input, out _);
338
Assert.Equal([], llamaTokenizer.
EncodeToTokens
((string)null!, out _));
500
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(text, out _);
664
encodedTokens = tokenizer.
EncodeToTokens
(kvp.Key, out normalizedString);
NormalizerTests.cs (1)
65
IReadOnlyList<EncodedToken> tokens = tokenizer.
EncodeToTokens
(text, out string? normalizedString);
PreTokenizerTests.cs (1)
59
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(text, out _);
TitokenTests.cs (8)
142
IReadOnlyList<EncodedToken> result = tokenizer.
EncodeToTokens
(text, out string? normalizedString);
195
IReadOnlyList<EncodedToken> result = GPT4.
EncodeToTokens
(text, out string? normalizedString);
238
IReadOnlyList<EncodedToken> result = GPT4.
EncodeToTokens
(text, out string? normalizedString);
257
IReadOnlyList<EncodedToken> result = GPT4.
EncodeToTokens
(text, out string? normalizedString);
273
IReadOnlyList<EncodedToken> result = GPT4.
EncodeToTokens
(text, out string? normalizedString);
307
IReadOnlyList<EncodedToken> result = GPT4o.
EncodeToTokens
(text, out string? normalizedString);
564
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(text, out _);
679
IReadOnlyList<EncodedToken> result = GPT4.
EncodeToTokens
(text, out _);
Microsoft.ML.TorchSharp (3)
NasBert\NerTrainer.cs (2)
170
IReadOnlyList<EncodedToken> encoding = Tokenizer.
EncodeToTokens
(sentence, out string normalizedString);
380
IReadOnlyList<EncodedToken> encoding = tokenizer.
EncodeToTokens
(sentence, out string normalizedString);
Roberta\QATrainer.cs (1)
404
var contextTokens = Tokenizer.
EncodeToTokens
(contextString, out string normalized);