6 overrides of PreTokenizer
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
300
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
Model\CodeGenTokenizer.cs (1)
252
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
Model\EnglishRobertaTokenizer.cs (1)
255
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
Model\SentencePieceTokenizer.cs (1)
111
public override PreTokenizer?
PreTokenizer
=> null;
Model\TiktokenTokenizer.cs (1)
127
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
Model\WordPieceTokenizer.cs (1)
243
public override PreTokenizer?
PreTokenizer
=> _preTokenizer;
11 references to PreTokenizer
Microsoft.ML.Tokenizers.Tests (10)
EnglishRobertaTests.cs (1)
239
Assert.True(tokenizer.
PreTokenizer
is RobertaPreTokenizer);
LlamaTests.cs (1)
498
Assert.Null(tokenizer.
PreTokenizer
);
TiktokenTests.cs (8)
62
Tokenizer tokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, GPT4.
PreTokenizer
, null, specialTokens);
67
tokenizer = TiktokenTokenizer.Create(stream, GPT4.
PreTokenizer
, null, specialTokens);
71
tokenizer = await TiktokenTokenizer.CreateAsync(tokenizerDataFileName, GPT4.
PreTokenizer
, normalizer: null, specialTokens);
76
tokenizer = await TiktokenTokenizer.CreateAsync(stream, GPT4.
PreTokenizer
, normalizer: null, specialTokens);
120
TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, tokenizer.
PreTokenizer
, null, tiktoken.SpecialTokens);
454
Assert.NotNull(tokenizer.
PreTokenizer
);
467
Assert.NotNull(tokenizer.
PreTokenizer
);
523
Assert.NotNull(tokenizer.
PreTokenizer
);
Microsoft.ML.TorchSharp (1)
NasBert\NerTrainer.cs (1)
379
var pre = tokenizer.
PreTokenizer
.PreTokenize(sentence);