1 instantiation of SymbolPair
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
1716SymbolPair pair = new(left, right, rank, symbols[left].pieceSpan.Length + symbols[right].pieceSpan.Length);
14 references to SymbolPair
Microsoft.ML.Tokenizers (14)
Model\CodeGenTokenizer.cs (14)
382PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 422private void EncodeInternal(string? text, scoped ReadOnlySpan<char> textSpan, List<EncodedToken> tokens, bool addPrefixSpace, int offset, PriorityQueue<SymbolPair> agenda) 622PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 834PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 971PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 1101private int EncodeToIdsInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int charsConsumed, int maxTokens) 1165private int EncodeToIdsFromEndInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int textIndex, int maxTokens) 1616private List<EncodedToken> EncodeToTokens(Span<char> text, Span<int> mapping, ReadOnlySpan<char> originalText, PriorityQueue<SymbolPair> agenda) 1652SymbolPair top = agenda.Dequeue(); 1716SymbolPair pair = new(left, right, rank, symbols[left].pieceSpan.Length + symbols[right].pieceSpan.Length); 1820private struct SymbolPair : IEquatable<SymbolPair>, IComparable<SymbolPair> 1835public int CompareTo(SymbolPair other) 1855public bool Equals(SymbolPair other) => Left == other.Left && Score == other.Score;