6 overrides of PreTokenizer
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
300
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
Model\CodeGenTokenizer.cs (1)
249
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
Model\EnglishRobertaTokenizer.cs (1)
255
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
Model\SentencePieceTokenizer.cs (1)
159
public override PreTokenizer?
PreTokenizer
=> null;
Model\TiktokenTokenizer.cs (1)
127
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
Model\WordPieceTokenizer.cs (1)
243
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
11 references to PreTokenizer
Microsoft.ML.Tokenizers.Tests (10)
EnglishRobertaTests.cs (1)
239
Assert.True(tokenizer.
PreTokenizer
is RobertaPreTokenizer);
LlamaTests.cs (1)
498
Assert.Null(tokenizer.
PreTokenizer
);
TiktokenTests.cs (8)
59
Tokenizer tokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, GPT4.
PreTokenizer
, null, specialTokens);
64
tokenizer = TiktokenTokenizer.Create(stream, GPT4.
PreTokenizer
, null, specialTokens);
68
tokenizer = await TiktokenTokenizer.CreateAsync(tokenizerDataFileName, GPT4.
PreTokenizer
, normalizer: null, specialTokens);
73
tokenizer = await TiktokenTokenizer.CreateAsync(stream, GPT4.
PreTokenizer
, normalizer: null, specialTokens);
117
TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, tokenizer.
PreTokenizer
, null, tiktoken.SpecialTokens);
446
Assert.NotNull(tokenizer.
PreTokenizer
);
459
Assert.NotNull(tokenizer.
PreTokenizer
);
513
Assert.NotNull(tokenizer.
PreTokenizer
);
Microsoft.ML.TorchSharp (1)
NasBert\NerTrainer.cs (1)
379
var pre = tokenizer.
PreTokenizer
.PreTokenize(sentence);