3 instantiations of Merge
Microsoft.ML.Tokenizers (3)
18 references to Merge
Microsoft.ML.Tokenizers (18)
Model\BPETokenizer.cs (9)
460PriorityQueue<Merge>? priorityQueue = null;
520PriorityQueue<Merge>? priorityQueue = null;
580PriorityQueue<Merge>? priorityQueue = null;
662PriorityQueue<Merge>? priorityQueue = null;
715PriorityQueue<Merge>? priorityQueue = null;
1190internal Word MergeWord(ReadOnlySpan<char> w, ref PriorityQueue<Merge>? priorityQueue)
1307internal void EncodeWithCache(ReadOnlySpan<char> text, List<EncodedToken> tokens, int offset, ref PriorityQueue<Merge>? priorityQueue)
1416private int EncodeToIdsWithCache(ReadOnlySpan<char> text, List<int>? accumulatedIds, int maxTokens, out int charsConsumed, ref PriorityQueue<Merge>? priorityQueue)
1496internal int EncodeToIdsFromEndWithCache(ReadOnlySpan<char> text, IList<int>? accumulatedIds, int maxTokens, out int textIndex, ref PriorityQueue<Merge>? priorityQueue)