1 instantiation of SymbolPair
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
1714SymbolPair pair = new(left, right, rank, symbols[left].pieceSpan.Length + symbols[right].pieceSpan.Length);
14 references to SymbolPair
Microsoft.ML.Tokenizers (14)
Model\CodeGenTokenizer.cs (14)
382PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 421private void EncodeInternal(string? text, scoped ReadOnlySpan<char> textSpan, List<EncodedToken> tokens, bool addPrefixSpace, int offset, PriorityQueue<SymbolPair> agenda) 620PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 832PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 969PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 1099private int EncodeToIdsInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int charsConsumed, int maxTokens) 1163private int EncodeToIdsFromEndInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int textIndex, int maxTokens) 1613private List<EncodedToken> EncodeToTokens(Span<char> text, Span<int> mapping, ReadOnlySpan<char> originalText, PriorityQueue<SymbolPair> agenda) 1649SymbolPair top = agenda.Dequeue(); 1714SymbolPair pair = new(left, right, rank, symbols[left].pieceSpan.Length + symbols[right].pieceSpan.Length); 1819private struct SymbolPair : IEquatable<SymbolPair>, IComparable<SymbolPair> 1834public int CompareTo(SymbolPair other) 1854public bool Equals(SymbolPair other) => Left == other.Left && Score == other.Score;