1 write to SpecialTokens
Microsoft.ML.Tokenizers.Tests (1)
BpeTests.cs (1)
916
bpeOptions.
SpecialTokens
= GetSpecialTokens(root);
2 references to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
185
options.
SpecialTokens
,
Microsoft.ML.Tokenizers.Tests (1)
BpeTests.cs (1)
923
bpeOptions.PreTokenizer = new CompositePreTokenizer(preTokenizers, bpeOptions.
SpecialTokens
);