2 instantiations of PriorityQueue
Microsoft.ML.Tokenizers (2)
Model\SentencePieceTokenizer.cs (1)
2195PriorityQueue<SymbolPair> agenda = new(symbolIndex);
Model\Word.cs (1)
102priorityQueue ??= new PriorityQueue<Merge>(_symbols.Count);
19 references to PriorityQueue
Microsoft.ML.Tokenizers (19)
Model\BPETokenizer.cs (9)
332PriorityQueue<Merge>? priorityQueue = null; 381PriorityQueue<Merge>? priorityQueue = null; 436PriorityQueue<Merge>? priorityQueue = null; 513PriorityQueue<Merge>? priorityQueue = null; 561PriorityQueue<Merge>? priorityQueue = null; 864internal Word MergeWord(ReadOnlySpan<char> w, ref PriorityQueue<Merge>? priorityQueue) 981internal void EncodeWithCache(ReadOnlySpan<char> text, List<EncodedToken> tokens, int offset, ref PriorityQueue<Merge>? priorityQueue) 1055private int EncodeToIdsWithCache(ReadOnlySpan<char> text, List<int>? accumulatedIds, int maxTokens, out int charsConsumed, ref PriorityQueue<Merge>? priorityQueue) 1088internal int EncodeToIdsFromEndWithCache(ReadOnlySpan<char> text, IList<int>? accumulatedIds, int maxTokens, out int textIndex, ref PriorityQueue<Merge>? priorityQueue)
Model\CodeGenTokenizer.cs (8)
382PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 422private void EncodeInternal(string? text, scoped ReadOnlySpan<char> textSpan, List<EncodedToken> tokens, bool addPrefixSpace, int offset, PriorityQueue<SymbolPair> agenda) 622PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 834PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 971PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 1101private int EncodeToIdsInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int charsConsumed, int maxTokens) 1165private int EncodeToIdsFromEndInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int textIndex, int maxTokens) 1616private List<EncodedToken> EncodeToTokens(Span<char> text, Span<int> mapping, ReadOnlySpan<char> originalText, PriorityQueue<SymbolPair> agenda)
Model\SentencePieceTokenizer.cs (1)
2195PriorityQueue<SymbolPair> agenda = new(symbolIndex);
Model\Word.cs (1)
100public void MergeAll(Dictionary<Pair<int>, (int, int)> merges, float? dropout, ref PriorityQueue<Merge>? priorityQueue)