24 references to Create
Microsoft.ML.Tokenizers.Tests (24)
SentencePieceTests.cs (22)
20Assert.ThrowsAny<ArgumentException>(() => SentencePieceTokenizer.Create(null!)); 27Assert.ThrowsAny<ArgumentException>(() => SentencePieceTokenizer.Create(empty)); 36Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 45SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream); 59SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(ms); 72SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream); 94SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream); 109SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream, 143SentencePieceTokenizer bpe = SentencePieceTokenizer.Create(bpeStream, 148SentencePieceTokenizer unigram = SentencePieceTokenizer.Create(unigramStream, 175Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 185Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 495SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream); 505Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 518Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 535Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 551Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 563Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 574Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 768Assert.Throws<ArgumentException>(() => SentencePieceTokenizer.Create(ms)); 788SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(ms); 804return SentencePieceTokenizer.Create(ms, addBos, addEos);
UnigramTests.cs (2)
27return SentencePieceTokenizer.Create(remoteStream); 59return SentencePieceTokenizer.Create(remoteStream, specialTokens: