3 writes to SpecialTokens
Microsoft.ML.Tokenizers.Tests (3)
BpeTests.cs (3)
898SpecialTokens = new Dictionary<string, int> // SpecialTokens not part of the original vocab.json 928SpecialTokens = options.SpecialTokens, 989bpeOptions.SpecialTokens = GetSpecialTokens(root);
7 references to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
190options.SpecialTokens,
Microsoft.ML.Tokenizers.Tests (6)
BpeTests.cs (6)
915Assert.Equal(options.SpecialTokens["<|sos|>"], ids[0]); 916Assert.Equal(options.SpecialTokens["<|eos|>"], ids[^1]); 928SpecialTokens = options.SpecialTokens, 938Assert.Equal(options.SpecialTokens["<|sos|>"], ids[0]); 939Assert.Equal(options.SpecialTokens["<|eos|>"], ids[^1]); 996bpeOptions.PreTokenizer = new CompositePreTokenizer(preTokenizers, bpeOptions.SpecialTokens);