47 references to EncodeToTokens
Microsoft.ML.Tokenizers.Tests (44)
BertTokenizerTests.cs (11)
55var tokens = tokenizer.EncodeToTokens(text, out string? normalizedText); 76tokens = tokenizer.EncodeToTokens(tokenizer.Decode(ids), out normalizedText); 128var tokens = tokenizer.EncodeToTokens(text, out string? normalizedText); 148tokens = tokenizer.EncodeToTokens(tokenizer.Decode(ids), out normalizedText); 196var tokens = tokenizer.EncodeToTokens(text, out string? normalizedText); 238var tokens = bertTokenizer.EncodeToTokens(text, out string? normalizedText); 253tokens = bertTokenizer.EncodeToTokens(text, out normalizedText); 268tokens = bertTokenizer.EncodeToTokens(text, out normalizedText); 282tokens = bertTokenizer.EncodeToTokens(text, out normalizedText); 313var tokens = bertTokenizer.EncodeToTokens(text, out string? normalizedText); 330tokens = bertTokenizer.EncodeToTokens(text, out normalizedText);
BpeTests.cs (4)
257IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(sentence, out _); 376IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(text, out _); 429IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(text, out _); 507IReadOnlyList<EncodedToken> tokens = bpeTokenizer.EncodeToTokens(input, out _);
CodeGenTests.cs (5)
252IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(text, out _); 347IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(text, out _); 549IReadOnlyList<EncodedToken> encoding = codeGenTokenizer.EncodeToTokens(text, out _); 682encoding = codeGenTokenizer.EncodeToTokens(text, out _); 815encoding = codeGenTokenizer.EncodeToTokens(text, out _);
EnglishRobertaTests.cs (4)
181IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(text, out _); 249encoding = tokenizer.EncodeToTokens((string)p[0], out _); 256encoding = tokenizer.EncodeToTokens((string)p[0], out _); 263encoding = tokenizer.EncodeToTokens((string)p[0], out _);
LlamaTests.cs (4)
244IReadOnlyList<EncodedToken> result = llamaTokenizer.EncodeToTokens(input, out _); 338Assert.Equal([], llamaTokenizer.EncodeToTokens((string)null!, out _)); 500IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(text, out _); 664encodedTokens = tokenizer.EncodeToTokens(kvp.Key, out normalizedText);
NormalizerTests.cs (1)
65IReadOnlyList<EncodedToken> tokens = tokenizer.EncodeToTokens(text, out normalizedText);
PreTokenizerTests.cs (1)
66IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(text, out _);
TiktokenTests.cs (8)
146IReadOnlyList<EncodedToken> result = tokenizer.EncodeToTokens(text, out string? normalizedText); 199IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 242IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 261IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 277IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out string? normalizedText); 311IReadOnlyList<EncodedToken> result = GPT4o.EncodeToTokens(text, out string? normalizedText); 582IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(text, out _); 697IReadOnlyList<EncodedToken> result = GPT4.EncodeToTokens(text, out _);
UnigramTests.cs (2)
404IReadOnlyList<EncodedToken> result = _unigramTokenizer.EncodeToTokens(inputText, out string? normalized); 408result = _unigramTokenizerFromJson.EncodeToTokens(inputText, out normalized);
WordPieceTests.cs (4)
62IReadOnlyList<EncodedToken> tokens = tokenizer.EncodeToTokens("", out _); 75tokens = tokenizer.EncodeToTokens(text, out _); 156IReadOnlyList<EncodedToken> tokens = tokenizer.EncodeToTokens(text, out _); 194IReadOnlyList<EncodedToken> tokens = tokenizer.EncodeToTokens(text, out _);
Microsoft.ML.TorchSharp (3)
NasBert\NerTrainer.cs (2)
170IReadOnlyList<EncodedToken> encoding = Tokenizer.EncodeToTokens(sentence, out string normalizedText); 380IReadOnlyList<EncodedToken> encoding = tokenizer.EncodeToTokens(sentence, out string normalizedText);
Roberta\QATrainer.cs (1)
404var contextTokens = Tokenizer.EncodeToTokens(contextString, out string normalized);