1 instantiation of SymbolPair
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
1698SymbolPair pair = new(left, right, rank, symbols[left].pieceSpan.Length + symbols[right].pieceSpan.Length);
14 references to SymbolPair
Microsoft.ML.Tokenizers (14)
Model\CodeGenTokenizer.cs (14)
385PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 425private void EncodeInternal(string? text, scoped ReadOnlySpan<char> textSpan, List<EncodedToken> tokens, bool addPrefixSpace, int offset, PriorityQueue<SymbolPair> agenda) 625PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 837PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 974PriorityQueue<SymbolPair> agenda = new(textSpanToEncode.Length); 1104private int EncodeToIdsInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int charsConsumed, int maxTokens) 1168private int EncodeToIdsFromEndInternal(string? text, scoped ReadOnlySpan<char> textSpan, IList<int>? accumulatedIds, PriorityQueue<SymbolPair> agenda, out int textIndex, int maxTokens) 1598private List<EncodedToken> EncodeToTokens(Span<char> text, Span<int> mapping, ReadOnlySpan<char> originalText, PriorityQueue<SymbolPair> agenda) 1634SymbolPair top = agenda.Dequeue(); 1698SymbolPair pair = new(left, right, rank, symbols[left].pieceSpan.Length + symbols[right].pieceSpan.Length); 1802private struct SymbolPair : IEquatable<SymbolPair>, IComparable<SymbolPair> 1817public int CompareTo(SymbolPair other) 1837public bool Equals(SymbolPair other) => Left == other.Left && Score == other.Score;