1 type derived from CodeGenTokenizer
Microsoft.ML.Tokenizers (1)
Model\Phi2Tokenizer.cs (1)
15
public sealed class Phi2Tokenizer :
CodeGenTokenizer
1 instantiation of CodeGenTokenizer
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
1876
return new
CodeGenTokenizer
(
18 references to CodeGenTokenizer
Microsoft.ML.Tokenizers (5)
Model\CodeGenTokenizer.cs (3)
1859
public static
CodeGenTokenizer
Create(
1879
new RegexPreTokenizer(TiktokenTokenizer.P50kBaseRegex(),
CodeGenTokenizer
.CodeGenSpecialTokens),
1881
CodeGenTokenizer
.CodeGenSpecialTokens,
Model\Phi2Tokenizer.cs (2)
117
vocabStream, mergesStream, new RegexPreTokenizer(TiktokenTokenizer.P50kBaseRegex(),
CodeGenTokenizer
.CodeGenSpecialTokens), normalizer: null,
118
CodeGenTokenizer
.CodeGenSpecialTokens, addPrefixSpace: addPrefixSpace, addBeginningOfSentence: addBeginOfSentence, addEndOfSentence: addEndOfSentence);
Microsoft.ML.Tokenizers.Tests (13)
CodeGenTests.cs (13)
31
return
CodeGenTokenizer
.Create(vocabStream, mergesStream, addPrefixSpace, bos, eos);
42
return
CodeGenTokenizer
.Create(vocabStream, mergesStream);
250
CodeGenTokenizer
codeGenTokenizer = (tokenizer as
CodeGenTokenizer
)!;
302
private void TestDecodingWithSpan(
CodeGenTokenizer
tokenizer, int[] ids, bool hasPrefixSpace, bool considerSpecialTokens, string expectedDecoded)
341
CodeGenTokenizer
codeGenTokenizer = (tokenizer as
CodeGenTokenizer
)!;
547
CodeGenTokenizer
codeGenTokenizer = (_codegen350MMonoTokenizerWithBeginningOfSentence as
CodeGenTokenizer
)!;
680
codeGenTokenizer = (_codegen350MMonoTokenizerWithEndOfSentence as
CodeGenTokenizer
)!;
813
codeGenTokenizer = (_codegen350MMonoTokenizerWithBeginningAndEndOfSentence as
CodeGenTokenizer
)!;
969
CodeGenTokenizer
codeGenTokenizer = (_codegen350MMonoTokenizer as
CodeGenTokenizer
)!;