2 instantiations of PriorityQueue
Microsoft.ML.Tokenizers (2)
Model\SentencePieceTokenizer.cs (1)
2195
PriorityQueue<SymbolPair> agenda =
new
(symbolIndex);
Model\Word.cs (1)
102
priorityQueue ??= new
PriorityQueue
<Merge>(_symbols.Count);
19 references to PriorityQueue
Microsoft.ML.Tokenizers (19)
Model\BPETokenizer.cs (9)
332
PriorityQueue
<Merge>? priorityQueue = null;
381
PriorityQueue
<Merge>? priorityQueue = null;
436
PriorityQueue
<Merge>? priorityQueue = null;
513
PriorityQueue
<Merge>? priorityQueue = null;
561
PriorityQueue
<Merge>? priorityQueue = null;
864
internal Word MergeWord(ReadOnlySpan<char> w, ref
PriorityQueue
<Merge>? priorityQueue)
981
internal void EncodeWithCache(ReadOnlySpan<char> text, List<EncodedToken> tokens, int offset, ref
PriorityQueue
<Merge>? priorityQueue)
1055
private int EncodeToIdsWithCache(ReadOnlySpan<char> text, List<int>? accumulatedIds, int maxTokens, out int charsConsumed, ref
PriorityQueue
<Merge>? priorityQueue)
1088
internal int EncodeToIdsFromEndWithCache(ReadOnlySpan<char> text, IList<int>? accumulatedIds, int maxTokens, out int textIndex, ref
PriorityQueue
<Merge>? priorityQueue)
Model\CodeGenTokenizer.cs (8)
382
PriorityQueue
<SymbolPair> agenda = new(textSpanToEncode.Length);
422
private void EncodeInternal(string? text, scoped ReadOnlySpan<char> textSpan, List<EncodedToken> tokens, bool addPrefixSpace, int offset,
PriorityQueue
<SymbolPair> agenda)
622
PriorityQueue
<SymbolPair> agenda = new(textSpanToEncode.Length);
834
PriorityQueue
<SymbolPair> agenda = new(textSpanToEncode.Length);
971
PriorityQueue
<SymbolPair> agenda = new(textSpanToEncode.Length);
1101
private int EncodeToIdsInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds,
PriorityQueue
<SymbolPair> agenda, out int charsConsumed, int maxTokens)
1165
private int EncodeToIdsFromEndInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds,
PriorityQueue
<SymbolPair> agenda, out int textIndex, int maxTokens)
1616
private List<EncodedToken> EncodeToTokens(Span<char> text, Span<int> mapping, ReadOnlySpan<char> originalText,
PriorityQueue
<SymbolPair> agenda)
Model\SentencePieceTokenizer.cs (1)
2195
PriorityQueue
<SymbolPair> agenda = new(symbolIndex);
Model\Word.cs (1)
100
public void MergeAll(Dictionary<Pair<int>, (int, int)> merges, float? dropout, ref
PriorityQueue
<Merge>? priorityQueue)