2 instantiations of PriorityQueue
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
1152PriorityQueue<SymbolPair> agenda = new(symbolIndex);
Model\Word.cs (1)
102priorityQueue ??= new PriorityQueue<Merge>(_symbols.Count);
19 references to PriorityQueue
Microsoft.ML.Tokenizers (19)
Model\BPETokenizer.cs (9)
455PriorityQueue<Merge>? priorityQueue = null; 515PriorityQueue<Merge>? priorityQueue = null; 575PriorityQueue<Merge>? priorityQueue = null; 657PriorityQueue<Merge>? priorityQueue = null; 710PriorityQueue<Merge>? priorityQueue = null; 1186internal Word MergeWord(ReadOnlySpan<char> w, ref PriorityQueue<Merge>? priorityQueue) 1303internal void EncodeWithCache(ReadOnlySpan<char> text, List<EncodedToken> tokens, int offset, ref PriorityQueue<Merge>? priorityQueue) 1412private int EncodeToIdsWithCache(ReadOnlySpan<char> text, List<int>? accumulatedIds, int maxTokens, out int charsConsumed, ref PriorityQueue<Merge>? priorityQueue) 1492internal int EncodeToIdsFromEndWithCache(ReadOnlySpan<char> text, IList<int>? accumulatedIds, int maxTokens, out int textIndex, ref PriorityQueue<Merge>? priorityQueue)
Model\CodeGenTokenizer.cs (8)
385PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 425private void EncodeInternal(string? text, scoped ReadOnlySpan<char> textSpan, List<EncodedToken> tokens, bool addPrefixSpace, int offset, PriorityQueue<SymbolPair> agenda) 625PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 837PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 974PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 1104private int EncodeToIdsInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int charsConsumed, int maxTokens) 1168private int EncodeToIdsFromEndInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int textIndex, int maxTokens) 1598private List<EncodedToken> EncodeToTokens(Span<char> text, Span<int> mapping, ReadOnlySpan<char> originalText, PriorityQueue<SymbolPair> agenda)
Model\SentencePieceBpeModel.cs (1)
1152PriorityQueue<SymbolPair> agenda = new(symbolIndex);
Model\Word.cs (1)
100public void MergeAll(Dictionary<Pair<int>, (int, int)> merges, float? dropout, ref PriorityQueue<Merge>? priorityQueue)