24 instantiations of EncodeResults
Microsoft.ML.Tokenizers (23)
Model\BPETokenizer.cs (4)
317
return new
EncodeResults
<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 };
346
return new
EncodeResults
<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = charsConsumed };
366
return new
EncodeResults
<int> { Tokens = [], NormalizedText = null, CharsConsumed = 0 };
402
return new
EncodeResults
<int> { Tokens = ids, NormalizedText = normalizedText, CharsConsumed = charsConsumed };
Model\CodeGenTokenizer.cs (3)
328
return new
EncodeResults
<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 };
402
return new
EncodeResults
<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = textSpanToEncode.Length };
490
return new
EncodeResults
<int>
Model\EnglishRobertaTokenizer.cs (5)
318
return new
EncodeResults
<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 };
343
return new
EncodeResults
<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = charsConsumed };
347
return new
EncodeResults
<EncodedToken> { Tokens = EncodeInternal(textSpanToEncode), NormalizedText = normalizedText, CharsConsumed = charsConsumed };
419
return new
EncodeResults
<int> { Tokens = [], NormalizedText = null, CharsConsumed = 0 };
455
return new
EncodeResults
<int> { Tokens = ids, NormalizedText = normalizedText, CharsConsumed = textLength };
Model\SentencePieceTokenizer.cs (2)
198
return new
EncodeResults
<EncodedToken>
455
return new
EncodeResults
<int>
Model\TiktokenTokenizer.cs (4)
262
return new
EncodeResults
<EncodedToken> { NormalizedText = null, Tokens = [], CharsConsumed = 0 };
290
return new
EncodeResults
<EncodedToken> { NormalizedText = normalizedText, Tokens = tokens, CharsConsumed = charsConsumed };
372
return new
EncodeResults
<int> { NormalizedText = null, Tokens = [], CharsConsumed = 0 };
407
return new
EncodeResults
<int> { NormalizedText = normalizedText, Tokens = ids, CharsConsumed = charsConsumed };
Model\WordPieceTokenizer.cs (4)
276
return new
EncodeResults
<EncodedToken> { NormalizedText = null, Tokens = [], CharsConsumed = 0 };
304
return new
EncodeResults
<EncodedToken> { NormalizedText = normalizedText, Tokens = tokens, CharsConsumed = charsConsumed };
399
return new
EncodeResults
<int> { NormalizedText = null, Tokens = [], CharsConsumed = 0 };
435
return new
EncodeResults
<int> { NormalizedText = normalizedText, Tokens = ids, CharsConsumed = charsConsumed };
Tokenizer.cs (1)
52
return new
EncodeResults
<int>
Microsoft.ML.Tokenizers.Tests (1)
TokenizerTests.cs (1)
119
return new
EncodeResults
<EncodedToken> { Tokens = tokens, CharsConsumed = count };
25 references to EncodeResults
Microsoft.ML.Tokenizers (24)
Model\BPETokenizer.cs (2)
313
protected override
EncodeResults
<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
356
protected override
EncodeResults
<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\CodeGenTokenizer.cs (5)
285
protected override
EncodeResults
<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
301
EncodeResults
<EncodedToken> result = EncodeToTokens(text, ReadOnlySpan<char>.Empty, addPrefixSpace, addBeginningOfSentence, addEndOfSentence, considerPreTokenization, considerNormalization);
319
EncodeResults
<EncodedToken> result = EncodeToTokens(null, text, addPrefixSpace, addBeginningOfSentence, addEndOfSentence, considerPreTokenization, considerNormalization);
324
private
EncodeResults
<EncodedToken> EncodeToTokens(string? text, scoped ReadOnlySpan<char> textSpan, bool addPrefixSpace, bool addBos, bool addEos, bool considerPreTokenization, bool considerNormalization)
488
protected override
EncodeResults
<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\EnglishRobertaTokenizer.cs (3)
314
protected override
EncodeResults
<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
407
protected override
EncodeResults
<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
410
private
EncodeResults
<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, bool considerPreTokenization, bool considerNormalization, int maxTokenCount = int.MaxValue)
Model\SentencePieceTokenizer.cs (2)
196
protected override
EncodeResults
<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
453
protected override
EncodeResults
<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\TiktokenTokenizer.cs (2)
258
protected override
EncodeResults
<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
362
protected override
EncodeResults
<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\WordPieceTokenizer.cs (2)
272
protected override
EncodeResults
<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
389
protected override
EncodeResults
<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Tokenizer.cs (8)
42
protected virtual
EncodeResults
<int> EncodeToIds(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
44
EncodeResults
<EncodedToken> results = EncodeToTokens(text, textSpan, settings);
92
EncodeResults
<int> result = EncodeToIds(text, text.AsSpan(),
118
EncodeResults
<int> result = EncodeToIds(null, text,
138
protected abstract
EncodeResults
<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings);
150
EncodeResults
<EncodedToken> result = EncodeToTokens(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization });
166
EncodeResults
<EncodedToken> result = EncodeToTokens(null, text, new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization });
235
EncodeResults
<EncodedToken> tokens = EncodeToTokens(text, textSpan, settings);
Microsoft.ML.Tokenizers.Tests (1)
TokenizerTests.cs (1)
105
protected override
EncodeResults
<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)