1 reference to
Microsoft.ML.Tokenizers (1)
Model\LlamaTokenizer.cs (1)
19
internal LlamaTokenizer(ModelProto modelProto, bool addBos, bool addEos, IReadOnlyDictionary<string, int>? addedTokens = null) :
base
(modelProto, addBos, addEos, addedTokens)