20 references to StringSpanOrdinalKey
Microsoft.ML.Tokenizers (20)
Model\CodeGenTokenizer.cs (2)
134_addedTokens = addedTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key)); 1625return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)].Id, tokenValue, (mapping[0], 1)) };
Model\EnglishRobertaTokenizer.cs (2)
950return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)], tokenValue, (indexMapping[0], 1)) }; 1039tokens.Add(new EncodedToken(_vocab[new StringSpanOrdinalKey(w)], w, (indexMapping[index], w.Length)));
Model\SentencePieceBpeTokenizer.cs (2)
51_vocab.Add(new StringSpanOrdinalKey(piece.Piece), (i, piece.Score, (byte)piece.Type)); 85_specialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
Model\TiktokenTokenizer.cs (1)
246vocab[new StringSpanOrdinalKey(decodedToken)] = (rank, decodedToken);
Utils\LruCache.cs (4)
51if (_cache.TryGetValue(new StringSpanOrdinalKey(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached)) 101if (_cache.TryGetValue(new StringSpanOrdinalKey(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached)) 113_cache.Remove(new StringSpanOrdinalKey(nodeToEvict!.Value.Key)); 117_cache[new StringSpanOrdinalKey(key)] = node;
Utils\StringSpanOrdinalKey.cs (9)
61_left = new StringSpanOrdinalKey(data1); 62_right = new StringSpanOrdinalKey(data2); 92return _map.TryGetValue(new StringSpanOrdinalKey(key), out value!); 111_map.Remove(new StringSpanOrdinalKey(key)); 121_map[new StringSpanOrdinalKey(k)] = v; 134new StringSpanOrdinalKey(reader.GetString()!); 139public override StringSpanOrdinalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new StringSpanOrdinalKey(reader.GetString()!); 162dictionary.Add(new StringSpanOrdinalKey(key!), (value, key!)); 185map.TryGetValue(new StringSpanOrdinalKey(key), out value!);