3 instantiations of Merge
Microsoft.ML.Tokenizers (3)
Model\Word.cs (3)
111priorityQueue.Enqueue(new Merge(i, value.m1, value.m2)); 169priorityQueue.Enqueue(new Merge(current.Prev, value.m1, value.m2)); 181priorityQueue.Enqueue(new Merge(top.Pos, value.m1, value.m2));
18 references to Merge
Microsoft.ML.Tokenizers (18)
Model\BPETokenizer.cs (9)
455PriorityQueue<Merge>? priorityQueue = null; 515PriorityQueue<Merge>? priorityQueue = null; 575PriorityQueue<Merge>? priorityQueue = null; 657PriorityQueue<Merge>? priorityQueue = null; 710PriorityQueue<Merge>? priorityQueue = null; 1186internal Word MergeWord(ReadOnlySpan<char> w, ref PriorityQueue<Merge>? priorityQueue) 1303internal void EncodeWithCache(ReadOnlySpan<char> text, List<EncodedToken> tokens, int offset, ref PriorityQueue<Merge>? priorityQueue) 1412private int EncodeToIdsWithCache(ReadOnlySpan<char> text, List<int>? accumulatedIds, int maxTokens, out int charsConsumed, ref PriorityQueue<Merge>? priorityQueue) 1492internal int EncodeToIdsFromEndWithCache(ReadOnlySpan<char> text, IList<int>? accumulatedIds, int maxTokens, out int textIndex, ref PriorityQueue<Merge>? priorityQueue)
Model\Merge.cs (4)
12internal struct Merge : IEquatable<Merge>, IComparable<Merge> 25public int CompareTo(Merge other) 43public bool Equals(Merge other) => Pos == other.Pos && Rank == other.Rank;
Model\Word.cs (5)
100public void MergeAll(Dictionary<Pair<int>, (int, int)> merges, float? dropout, ref PriorityQueue<Merge>? priorityQueue) 102priorityQueue ??= new PriorityQueue<Merge>(_symbols.Count); 105Vec<Merge> skip = new Vec<Merge>(priorityQueue.Count); 117Merge top = priorityQueue.Dequeue();