29 references to StringSpanOrdinalKey
Microsoft.ML.Tokenizers (29)
Model\BertTokenizer.cs (7)
39
SeparatorTokenId = vocab[new
StringSpanOrdinalKey
(options.SeparatorToken)];
42
PaddingTokenId = vocab[new
StringSpanOrdinalKey
(options.PaddingToken)];
45
ClassificationTokenId = vocab[new
StringSpanOrdinalKey
(options.ClassificationToken)];
48
MaskingTokenId = vocab[new
StringSpanOrdinalKey
(options.MaskingToken)];
778
if (!vocab.TryGetValue(new
StringSpanOrdinalKey
(kvp.Key), out int id) || id != kvp.Value)
816
if (token is null || !vocab.TryGetValue(new
StringSpanOrdinalKey
(token), out int id))
835
vocab[new
StringSpanOrdinalKey
(normalizedToken)] = id;
Model\BPETokenizer.cs (1)
259
_specialTokens = specialTokens.ToDictionary(kvp => new
StringSpanOrdinalKey
(kvp.Key), kvp => (kvp.Value, kvp.Key));
Model\CodeGenTokenizer.cs (2)
134
_specialTokens = specialTokens.ToDictionary(kvp => new
StringSpanOrdinalKey
(kvp.Key), kvp => (kvp.Value, kvp.Key));
1628
return new List<EncodedToken> { new EncodedToken(_vocab[new
StringSpanOrdinalKey
(tokenValue)].Id, tokenValue, new Range(mapping[0], mapping[0] + 1)) };
Model\EnglishRobertaTokenizer.cs (2)
961
return new List<EncodedToken> { new EncodedToken(_vocab[new
StringSpanOrdinalKey
(tokenValue)], tokenValue, new Range(indexMapping[0], indexMapping[0] + 1)) };
1050
tokens.Add(new EncodedToken(_vocab[new
StringSpanOrdinalKey
(w)], w, new Range(indexMapping[index], indexMapping[index] + w.Length)));
Model\SentencePieceTokenizer.cs (2)
51
_vocab.Add(new
StringSpanOrdinalKey
(piece.Piece), (i, piece.Score, (byte)piece.Type));
85
_specialTokens.Add(new
StringSpanOrdinalKey
(item.Key), item.Value);
Model\TiktokenTokenizer.cs (1)
247
vocab[new
StringSpanOrdinalKey
(decodedToken)] = (rank, decodedToken);
Model\WordPieceTokenizer.cs (1)
110
vocab.Add(new
StringSpanOrdinalKey
(line), lineNumber);
Utils\LruCache.cs (4)
51
if (_cache.TryGetValue(new
StringSpanOrdinalKey
(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached))
101
if (_cache.TryGetValue(new
StringSpanOrdinalKey
(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached))
113
_cache.Remove(new
StringSpanOrdinalKey
(nodeToEvict!.Value.Key));
117
_cache[new
StringSpanOrdinalKey
(key)] = node;
Utils\StringSpanOrdinalKey.cs (9)
62
_left = new
StringSpanOrdinalKey
(data1);
63
_right = new
StringSpanOrdinalKey
(data2);
93
return _map.TryGetValue(new
StringSpanOrdinalKey
(key), out value!);
112
_map.Remove(new
StringSpanOrdinalKey
(key));
122
_map[new
StringSpanOrdinalKey
(k)] = v;
137
new
StringSpanOrdinalKey
(reader.GetString()!);
142
public override StringSpanOrdinalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new
StringSpanOrdinalKey
(reader.GetString()!);
163
dictionary.Add(new
StringSpanOrdinalKey
(key!), (value, key!));
186
map.TryGetValue(new
StringSpanOrdinalKey
(key), out value!);