3 writes to SpecialTokens
Microsoft.ML.Tokenizers.Tests (3)
BpeTests.cs (3)
942SpecialTokens = new Dictionary<string, int> // SpecialTokens not part of the original vocab.json 972SpecialTokens = options.SpecialTokens, 1033bpeOptions.SpecialTokens = GetSpecialTokens(root);
7 references to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\BpeTokenizer.cs (1)
190options.SpecialTokens,
Microsoft.ML.Tokenizers.Tests (6)
BpeTests.cs (6)
959Assert.Equal(options.SpecialTokens["<|sos|>"], ids[0]); 960Assert.Equal(options.SpecialTokens["<|eos|>"], ids[^1]); 972SpecialTokens = options.SpecialTokens, 982Assert.Equal(options.SpecialTokens["<|sos|>"], ids[0]); 983Assert.Equal(options.SpecialTokens["<|eos|>"], ids[^1]); 1040bpeOptions.PreTokenizer = new CompositePreTokenizer(preTokenizers, bpeOptions.SpecialTokens);