8 references to CreateForEncoding
Microsoft.ML.Tokenizers.Tests (8)
TiktokenTests.cs (8)
503
Tokenizer tokenizer = TiktokenTokenizer.
CreateForEncoding
(encodingName);
535
Assert.Throws<ArgumentNullException>(() => TiktokenTokenizer.
CreateForEncoding
(null!));
536
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("r50k_base_"));
537
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("p50k_base_"));
538
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("p50k_edit_"));
539
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("cl100k_base_"));
540
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("o200k_base_"));
541
Assert.Throws<ArgumentException>(() => TiktokenTokenizer.
CreateForEncoding
("o200k_harmony_"));