26_model = modelProto.TrainerSpec.ModelType switch 36_model = model;

42public IReadOnlyDictionary<string, int>? SpecialTokens => _model.SpecialTokens; 47public bool ByteFallback => _model.ByteFallback; 52public bool AddDummyPrefix => _model.AddDummyPrefix; 57public bool EscapeWhiteSpaces => _model.EscapeWhiteSpaces; 62public bool TreatWhitespaceAsSuffix { get => _model.TreatWhitespaceAsSuffix; private set => _model.TreatWhitespaceAsSuffix = value; } 67public bool AddBeginningOfSentence => _model.AddBeginningOfSentence; 72public bool AddEndOfSentence => _model.AddEndOfSentence; 77public string BeginningOfSentenceToken => _model.BeginningOfSentenceToken; 82public string EndOfSentenceToken => _model.EndOfSentenceToken; 87public string UnknownToken => _model.UnknownToken; 92public int BeginningOfSentenceId => _model.BeginningOfSentenceId; 97public int EndOfSentenceId => _model.EndOfSentenceId; 102public int UnknownId => _model.UnknownId; 112public override Normalizer? Normalizer => _model.Normalizer; 117public IReadOnlyDictionary<string, int> Vocabulary => _model.Vocabulary; 129Tokens = _model.EncodeToTokens(text, textSpan, out string? normalizedText, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization), 146=> _model.EncodeToTokens(text, Span<char>.Empty, out normalizedText, addBeginningOfSentence, addEndOfSentence, considerNormalization); 159=> _model.EncodeToTokens(null, text, out normalizedText, addBeginningOfSentence, addEndOfSentence, considerNormalization); 173Tokens = _model.EncodeToIds(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization, out string? normalizedText, out int charsConsumed, settings.MaxTokenCount), 189=> _model.EncodeToIds(text, Span<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _); 201=> _model.EncodeToIds(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _); 216=> _model.EncodeToIds(text, Span<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount); 231=> _model.EncodeToIds(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount); 241=> _model.CountTokens(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization, out _, out _, settings.MaxTokenCount); 253=> _model.CountTokens(text, ReadOnlySpan<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _, int.MaxValue); 265=> _model.CountTokens(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _, int.MaxValue); 280=> _model.CountTokens(text, ReadOnlySpan<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount); 295=> _model.CountTokens(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount); 317return _model.GetIndexByTokenCountFromEnd(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.MaxTokenCount, settings.ConsiderNormalization, out normalizedText, out tokenCount); 320tokenCount = _model.CountTokens(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization, out normalizedText, out int charsConsumed, settings.MaxTokenCount); 342tokenCount = _model.CountTokens(text, ReadOnlySpan<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out int charsConsumed, maxTokenCount); 364tokenCount = _model.CountTokens(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out int charsConsumed, maxTokenCount); 383=> _model.GetIndexByTokenCountFromEnd(text, ReadOnlySpan<char>.Empty, addBeginningOfSentence, addEndOfSentence, maxTokenCount, considerNormalization, out normalizedText, out tokenCount); 400=> _model.GetIndexByTokenCountFromEnd(null, text, addBeginningOfSentence, addEndOfSentence, maxTokenCount, considerNormalization, out normalizedText, out tokenCount); 407public override string Decode(IEnumerable<int> ids) => _model.Decode(ids, considerSpecialTokens: false); 415public string Decode(IEnumerable<int> ids, bool considerSpecialTokens) => _model.Decode(ids, considerSpecialTokens); 426=> _model.Decode(ids, destination, considerSpecialTokens: false, out idsConsumed, out charsWritten); 438=> _model.Decode(ids, destination, considerSpecialTokens, out idsConsumed, out charsWritten);