17public sealed class LlamaTokenizer : SentencePieceTokenizer

464return new SentencePieceTokenizer(modelProto, addBeginningOfSentence, addEndOfSentence, specialTokens); 523return new SentencePieceTokenizer(model); 757return new SentencePieceTokenizer(model);

275SentencePieceTokenizer bpeTokenizer => bpeTokenizer.Decode(tokenIds.Concat(tokenIds), considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"), 281SentencePieceTokenizer bpeTokenizer => bpeTokenizer.Decode(tokenIds, considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"),

451public static SentencePieceTokenizer Create( 468/// Creates a Unigram <see cref="SentencePieceTokenizer"/> from an in-memory vocabulary of (piece, score) pairs. 487/// <returns>A new <see cref="SentencePieceTokenizer"/> instance.</returns> 498public static SentencePieceTokenizer Create( 527/// Creates a Unigram <see cref="SentencePieceTokenizer"/> by parsing a Hugging Face <c>tokenizer.json</c> 534/// <returns>A new <see cref="SentencePieceTokenizer"/> instance.</returns> 562public static SentencePieceTokenizer CreateFromTokenizerJson(

69PropertyInfo? propertyInfo = typeof(SentencePieceTokenizer).GetProperty("TreatWhitespaceAsSuffix", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); 511SentencePieceTokenizer sentencePieceBpe = (tokenizer as SentencePieceTokenizer)!; 565SentencePieceTokenizer sentencePieceBpe = (tokenizer as SentencePieceTokenizer)!;

20Assert.ThrowsAny<ArgumentException>(() => SentencePieceTokenizer.Create(null!)); 27Assert.ThrowsAny<ArgumentException>(() => SentencePieceTokenizer.Create(empty)); 36Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 45SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream); 59SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(ms); 72SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream); 94SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream); 109SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream, 143SentencePieceTokenizer bpe = SentencePieceTokenizer.Create(bpeStream, 148SentencePieceTokenizer unigram = SentencePieceTokenizer.Create(unigramStream, 175Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 185Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 197SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 231SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 253SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 268SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 286SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 307SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model.ToArray()); 326SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 362SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model.ToArray()); 379SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 393SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model.ToArray()); 411SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 427SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 441SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 453SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 478SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 495SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(stream); 505Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 518Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 535Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 551Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 563Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 574Assert.ThrowsAny<Exception>(() => SentencePieceTokenizer.Create(ms)); 587SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model.ToArray()); 602SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 617SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 639SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 656SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 671SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 684SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model.ToArray()); 707SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model.ToArray()); 730SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model.ToArray()); 749SentencePieceTokenizer tokenizer = CreateFromSyntheticModel(model); 768Assert.Throws<ArgumentException>(() => SentencePieceTokenizer.Create(ms)); 788SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create(ms); 796private static SentencePieceTokenizer CreateFromSyntheticModel( 804return SentencePieceTokenizer.Create(ms, addBos, addEos);

155if (tokenizer is SentencePieceTokenizer) 173if (tokenizer is SentencePieceTokenizer)

20private static SentencePieceTokenizer _unigramTokenizer = CreateUnigramTokenizer(); 21private static SentencePieceTokenizer _unigramTokenizerWithSpecialTokens = CreateUnigramTokenizerWithSpecialTokens(); 23private static SentencePieceTokenizer CreateUnigramTokenizer() 27return SentencePieceTokenizer.Create(remoteStream); 55private static SentencePieceTokenizer CreateUnigramTokenizerWithSpecialTokens() 59return SentencePieceTokenizer.Create(remoteStream, specialTokens: 283SentencePieceTokenizer tokenizer, 535private static void DecodeWithTokenizerTest(SentencePieceTokenizer tokenizer, string decodedString, int[] ids) 581SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create( 602SentencePieceTokenizer.Create((IEnumerable<(string Piece, float Score)>)null!, unkId: 0)); 610SentencePieceTokenizer.Create(vocab, unkId: 5)); 617SentencePieceTokenizer jsonTokenizer = SentencePieceTokenizer.CreateFromTokenizerJson( 646SentencePieceTokenizer.CreateFromTokenizerJson(null!)); 655SentencePieceTokenizer jsonTokenizer = SentencePieceTokenizer.CreateFromTokenizerJson( 684SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create( 703SentencePieceTokenizer.Create(vocab, unkId: 0, addBeginningOfSentence: true)); 730SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false); 781SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 808SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 831SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 839SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 1020SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 1050SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 1073SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false)); 1089SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1109SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1125SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1141SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false)); 1157SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false)); 1256SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false)); 1275SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson( 1292SentencePieceTokenizer tokenizer = SentencePieceTokenizer.Create( 1319SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 1339SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1358SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1377SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1397SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1423SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1443SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1463SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1483SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1512SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1541SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1561SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1581SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1601SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1620SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false); 1640SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1660SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1680SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false); 1701SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1715SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false)); 1734SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson( 1772SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 1807SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 1836SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false)); 1858SentencePieceTokenizer tokenizer = SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false); 1888SentencePieceTokenizer.CreateFromTokenizerJson(stream, addBeginningOfSentence: false, addEndOfSentence: false));