20 references to StringSpanOrdinalKey
Microsoft.ML.Tokenizers (20)
Model\CodeGenTokenizer.cs (2)
134
_addedTokens = addedTokens.ToDictionary(kvp => new
StringSpanOrdinalKey
(kvp.Key), kvp => (kvp.Value, kvp.Key));
1625
return new List<EncodedToken> { new EncodedToken(_vocab[new
StringSpanOrdinalKey
(tokenValue)].Id, tokenValue, (mapping[0], 1)) };
Model\EnglishRobertaTokenizer.cs (2)
950
return new List<EncodedToken> { new EncodedToken(_vocab[new
StringSpanOrdinalKey
(tokenValue)], tokenValue, (indexMapping[0], 1)) };
1039
tokens.Add(new EncodedToken(_vocab[new
StringSpanOrdinalKey
(w)], w, (indexMapping[index], w.Length)));
Model\SentencePieceBpeTokenizer.cs (2)
51
_vocab.Add(new
StringSpanOrdinalKey
(piece.Piece), (i, piece.Score, (byte)piece.Type));
85
_specialTokens.Add(new
StringSpanOrdinalKey
(item.Key), item.Value);
Model\TiktokenTokenizer.cs (1)
246
vocab[new
StringSpanOrdinalKey
(decodedToken)] = (rank, decodedToken);
Utils\LruCache.cs (4)
51
if (_cache.TryGetValue(new
StringSpanOrdinalKey
(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached))
101
if (_cache.TryGetValue(new
StringSpanOrdinalKey
(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached))
113
_cache.Remove(new
StringSpanOrdinalKey
(nodeToEvict!.Value.Key));
117
_cache[new
StringSpanOrdinalKey
(key)] = node;
Utils\StringSpanOrdinalKey.cs (9)
61
_left = new
StringSpanOrdinalKey
(data1);
62
_right = new
StringSpanOrdinalKey
(data2);
92
return _map.TryGetValue(new
StringSpanOrdinalKey
(key), out value!);
111
_map.Remove(new
StringSpanOrdinalKey
(key));
121
_map[new
StringSpanOrdinalKey
(k)] = v;
134
new
StringSpanOrdinalKey
(reader.GetString()!);
139
public override StringSpanOrdinalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new
StringSpanOrdinalKey
(reader.GetString()!);
162
dictionary.Add(new
StringSpanOrdinalKey
(key!), (value, key!));
185
map.TryGetValue(new
StringSpanOrdinalKey
(key), out value!);