7 references to CreateForEncoding
Microsoft.ML.Tokenizers.Tests (7)
TitokenTests.cs (7)
454
Tokenizer tokenizer = TiktokenTokenizer.
CreateForEncoding
(encodingName);
485
Assert.Throws<ArgumentNullException>(() => TiktokenTokenizer.
CreateForEncoding
(null!));
486
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("r50k_base_"));
487
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("p50k_base_"));
488
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("p50k_edit_"));
489
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("cl100k_base_"));
490
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("o200k_base_"));