3 references to Create
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
143
=>
Create
(vocabStream, mergesStream, preTokenizer: PreTokenizer.CreateWordOrNonWord(), normalizer: null, specialTokens: null, unknownToken: null, continuingSubwordPrefix: null, endOfWordSuffix: null, fuseUnknownTokens: false);
Microsoft.ML.Tokenizers.Tests (2)
BpeTests.cs (2)
503
var bpeTokenizer = BpeTokenizer.
Create
(vocabStream, mergesStream, PreTokenizer.CreateWordOrNonWord(specialTokens), normalizer: null, specialTokens: specialTokens, unknownToken: "<|endoftext|>");
558
return BpeTokenizer.
Create
(