3 writes to Merges
Microsoft.ML.Tokenizers.Tests (3)
BpeTests.cs (3)
272
Merges
= merges?.Select(kvp => $"{kvp.Item1} {kvp.Item2}"),
392
Merges
= File.ReadAllLines(mergesFile).Skip(1).ToArray() // Skip the first line which is the header "#version".
917
bpeOptions.
Merges
= GetMerges(modelElement);
2 references to Merges
Microsoft.ML.Tokenizers (2)
Model\BPETokenizer.cs (2)
160
if (options.
Merges
is not null)
164
foreach (string merge in options.
Merges
)