2 instantiations of PriorityQueue
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
1198PriorityQueue<SymbolPair> agenda = new(symbolIndex);
Model\Word.cs (1)
102priorityQueue ??= new PriorityQueue<Merge>(_symbols.Count);
19 references to PriorityQueue
Microsoft.ML.Tokenizers (19)
Model\BPETokenizer.cs (9)
332PriorityQueue<Merge>? priorityQueue = null; 381PriorityQueue<Merge>? priorityQueue = null; 436PriorityQueue<Merge>? priorityQueue = null; 513PriorityQueue<Merge>? priorityQueue = null; 561PriorityQueue<Merge>? priorityQueue = null; 864internal Word MergeWord(ReadOnlySpan<char> w, ref PriorityQueue<Merge>? priorityQueue) 981internal void EncodeWithCache(ReadOnlySpan<char> text, List<EncodedToken> tokens, int offset, ref PriorityQueue<Merge>? priorityQueue) 1055private int EncodeToIdsWithCache(ReadOnlySpan<char> text, List<int>? accumulatedIds, int maxTokens, out int charsConsumed, ref PriorityQueue<Merge>? priorityQueue) 1088internal int EncodeToIdsFromEndWithCache(ReadOnlySpan<char> text, IList<int>? accumulatedIds, int maxTokens, out int textIndex, ref PriorityQueue<Merge>? priorityQueue)
Model\CodeGenTokenizer.cs (8)
385PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 425private void EncodeInternal(string? text, scoped ReadOnlySpan<char> textSpan, List<EncodedToken> tokens, bool addPrefixSpace, int offset, PriorityQueue<SymbolPair> agenda) 625PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 837PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 974PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 1104private int EncodeToIdsInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int charsConsumed, int maxTokens) 1168private int EncodeToIdsFromEndInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int textIndex, int maxTokens) 1619private List<EncodedToken> EncodeToTokens(Span<char> text, Span<int> mapping, ReadOnlySpan<char> originalText, PriorityQueue<SymbolPair> agenda)
Model\SentencePieceBpeModel.cs (1)
1198PriorityQueue<SymbolPair> agenda = new(symbolIndex);
Model\Word.cs (1)
100public void MergeAll(Dictionary<Pair<int>, (int, int)> merges, float? dropout, ref PriorityQueue<Merge>? priorityQueue)