8 instantiations of Vec
Microsoft.ML.Tokenizers (8)
Model\BPETokenizer.cs (3)
162merges = new Vec<(string, string)>(1000); 1134return new Vec<(string, string)>(); 1140Vec<(string, string)> merges = new(1000);
Model\Word.cs (5)
17public Word() => _symbols = new Vec<Symbol>(); 25_symbols = new Vec<Symbol>(capacity); 51Vec<(Pair<int>, int)> changes = new(); 105Vec<Merge> skip = new Vec<Merge>(priorityQueue.Count); 265Vec<int> chars = new Vec<int>();
15 references to Vec
Microsoft.ML.Tokenizers (15)
Model\BPETokenizer.cs (9)
130(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult(); 159Vec<(string, string)> merges = default; 237(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult(); 273(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = await ReadModelDataAsync(vocabStream, mergesStream, useAsync: true).ConfigureAwait(false); 295Vec<(string, string)> merges, 1095internal static async ValueTask<(Dictionary<StringSpanOrdinalKey, int>?, Vec<(string, string)>)> ReadModelDataAsync(Stream vocab, Stream? merges, bool useAsync, CancellationToken cancellationToken = default) 1101var m = useAsync ? 1130internal static async ValueTask<Vec<(string, string)>> ConvertMergesToHashmapAsync(Stream? mergesStream, bool useAsync = false, CancellationToken cancellationToken = default) 1140Vec<(string, string)> merges = new(1000);
Model\Word.cs (6)
15private Vec<Symbol> _symbols; 49public Vec<(Pair<int>, int)> Merge(int c1, int c2, int replacement) 51Vec<(Pair<int>, int)> changes = new(); 105Vec<Merge> skip = new Vec<Merge>(priorityQueue.Count); 263public Vec<int> GetChars() 265Vec<int> chars = new Vec<int>();