2 writes to SpecialTokens
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
37
SpecialTokens
= specialTokens;
98
SpecialTokens
= options.SpecialTokens;
6 references to SpecialTokens
Microsoft.ML.Tokenizers (6)
Model\SentencePieceBaseModel.cs (5)
100
if (
SpecialTokens
is not null &&
SpecialTokens
.Count > 0)
105
foreach (var item in
SpecialTokens
)
112
SpecialTokensRegex = new Regex(string.Join("|",
SpecialTokens
.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled);
120
SpecialTokens
);
Model\SentencePieceTokenizer.cs (1)
46
public IReadOnlyDictionary<string, int>? SpecialTokens => _model.
SpecialTokens
;