7 overrides of EncodeToTokens
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
313protected override EncodeResults<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\CodeGenTokenizer.cs (1)
285protected override EncodeResults<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\EnglishRobertaTokenizer.cs (1)
314protected override EncodeResults<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\SentencePieceTokenizer.cs (1)
196protected override EncodeResults<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\TiktokenTokenizer.cs (1)
258protected override EncodeResults<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\WordPieceTokenizer.cs (1)
272protected override EncodeResults<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Microsoft.ML.Tokenizers.Tests (1)
TokenizerTests.cs (1)
105protected override EncodeResults<EncodedToken> EncodeToTokens(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
8 references to EncodeToTokens
Microsoft.ML.Tokenizers (8)
Tokenizer.cs (8)
40/// By default, it uses <see cref="EncodeToTokens(string?, ReadOnlySpan{char}, EncodeSettings)"/>. 44EncodeResults<EncodedToken> results = EncodeToTokens(text, textSpan, settings); 150EncodeResults<EncodedToken> result = EncodeToTokens(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }); 166EncodeResults<EncodedToken> result = EncodeToTokens(null, text, new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }); 181/// By default, it uses <see cref="EncodeToTokens(string?, ReadOnlySpan{char}, EncodeSettings)"/>. 184=> EncodeToTokens(text, textSpan, settings).Tokens.Count; 224/// By default, it uses <see cref="EncodeToTokens(string?, ReadOnlySpan{char}, EncodeSettings)"/>. 235EncodeResults<EncodedToken> tokens = EncodeToTokens(text, textSpan, settings);