25 instantiations of StringSpanOrdinalKey
Microsoft.ML.Tokenizers (25)
Model\CodeGenTokenizer.cs (2)
134
_addedTokens = addedTokens.ToDictionary(kvp => new
StringSpanOrdinalKey
(kvp.Key), kvp => (kvp.Value, kvp.Key));
1625
return new List<EncodedToken> { new EncodedToken(_vocab[new
StringSpanOrdinalKey
(tokenValue)].Id, tokenValue, (mapping[0], 1)) };
Model\EnglishRobertaTokenizer.cs (2)
950
return new List<EncodedToken> { new EncodedToken(_vocab[new
StringSpanOrdinalKey
(tokenValue)], tokenValue, (indexMapping[0], 1)) };
1039
tokens.Add(new EncodedToken(_vocab[new
StringSpanOrdinalKey
(w)], w, (indexMapping[index], w.Length)));
Model\SentencePieceBpeTokenizer.cs (2)
51
_vocab.Add(new
StringSpanOrdinalKey
(piece.Piece), (i, piece.Score, (byte)piece.Type));
85
_specialTokens.Add(new
StringSpanOrdinalKey
(item.Key), item.Value);
Model\TiktokenTokenizer.cs (1)
246
vocab[new
StringSpanOrdinalKey
(decodedToken)] = (rank, decodedToken);
Utils\LruCache.cs (5)
51
if (_cache.TryGetValue(new
StringSpanOrdinalKey
(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached))
78
if (_cache.TryGetValue(new
StringSpanOrdinalKey
(ptr, key.Length), out LinkedListNode<KeyValuePair<string, TValue>>? cached))
101
if (_cache.TryGetValue(new
StringSpanOrdinalKey
(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached))
113
_cache.Remove(new
StringSpanOrdinalKey
(nodeToEvict!.Value.Key));
117
_cache[new
StringSpanOrdinalKey
(key)] = node;
Utils\StringSpanOrdinalKey.cs (13)
55
_left = new
StringSpanOrdinalKey
(ptr1, length1);
56
_right = new
StringSpanOrdinalKey
(ptr2, length2);
61
_left = new
StringSpanOrdinalKey
(data1);
62
_right = new
StringSpanOrdinalKey
(data2);
92
return _map.TryGetValue(new
StringSpanOrdinalKey
(key), out value!);
102
return _map.TryGetValue(new
StringSpanOrdinalKey
(ptr, key.Length), out value!);
111
_map.Remove(new
StringSpanOrdinalKey
(key));
121
_map[new
StringSpanOrdinalKey
(k)] = v;
134
new
StringSpanOrdinalKey
(reader.GetString()!);
139
public override StringSpanOrdinalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new
StringSpanOrdinalKey
(reader.GetString()!);
162
dictionary.Add(new
StringSpanOrdinalKey
(key!), (value, key!));
180
return map.TryGetValue(new
StringSpanOrdinalKey
(ptr, key.Length), out value!);
185
map.TryGetValue(new
StringSpanOrdinalKey
(key), out value!);
58 references to StringSpanOrdinalKey
Microsoft.ML.Tokenizers (58)
Model\BPETokenizer.cs (13)
114
(Dictionary<
StringSpanOrdinalKey
, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult();
153
(Dictionary<
StringSpanOrdinalKey
, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult();
184
(Dictionary<
StringSpanOrdinalKey
, int>? vocab, Vec<(string, string)> merges) result = await ReadModelDataAsync(vocabStream, mergesStream, useAsync: true).ConfigureAwait(false);
200
private BpeTokenizer(Dictionary<
StringSpanOrdinalKey
, int>? vocab, Vec<(string, string)> merges, PreTokenizer? preTokenizer, Normalizer? normalizer, string? unknownToken, string? continuingSubwordPrefix, string? endOfWordSuffix, bool fuseUnknownTokens)
208
_vocab = vocab ?? new Dictionary<
StringSpanOrdinalKey
, int>();
213
foreach (KeyValuePair<
StringSpanOrdinalKey
, int> kvp in _vocab)
728
internal static async ValueTask<(Dictionary<
StringSpanOrdinalKey
, int>?, Vec<(string, string)>)> ReadModelDataAsync(Stream vocab, Stream? merges, bool useAsync, CancellationToken cancellationToken = default)
732
Dictionary<
StringSpanOrdinalKey
, int>? dic = useAsync ?
733
await JsonSerializer.DeserializeAsync<Dictionary<
StringSpanOrdinalKey
, int>>(vocab, options, cancellationToken).ConfigureAwait(false) as Dictionary<
StringSpanOrdinalKey
, int> :
734
JsonSerializer.Deserialize<Dictionary<
StringSpanOrdinalKey
, int>>(vocab, options) as Dictionary<
StringSpanOrdinalKey
, int>;
743
private readonly Dictionary<
StringSpanOrdinalKey
, int> _vocab;
Model\CodeGenTokenizer.cs (6)
25
private readonly Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)> _vocab;
28
private readonly Dictionary<
StringSpanOrdinalKey
, (int, string)>? _addedTokens;
1763
private static Dictionary<
StringSpanOrdinalKey
, (int, string)> GetVocabulary(Stream vocabularyStream)
1765
Dictionary<
StringSpanOrdinalKey
, (int, string)>? vocab;
1769
vocab = JsonSerializer.Deserialize<Dictionary<
StringSpanOrdinalKey
, (int, string)>>(vocabularyStream, options) as Dictionary<
StringSpanOrdinalKey
, (int, string)>;
Model\EnglishRobertaTokenizer.cs (7)
23
private readonly Dictionary<
StringSpanOrdinalKey
, int> _vocab;
25
private readonly SortedDictionary<int,
StringSpanOrdinalKey
> _vocabReverse;
167
private static Dictionary<
StringSpanOrdinalKey
, int> GetVocabulary(Stream vocabularyStream)
169
Dictionary<
StringSpanOrdinalKey
, int>? vocab;
173
vocab = JsonSerializer.Deserialize<Dictionary<
StringSpanOrdinalKey
, int>>(vocabularyStream, options) as Dictionary<
StringSpanOrdinalKey
, int>;
267
if (_vocabReverse.TryGetValue(id, out
var
value))
Model\SentencePieceBpeTokenizer.cs (3)
28
private readonly Dictionary<
StringSpanOrdinalKey
, (int Id, float Score, byte Type)> _vocab = new();
36
private readonly Dictionary<
StringSpanOrdinalKey
, int>? _specialTokens;
80
_specialTokens = new Dictionary<
StringSpanOrdinalKey
, int>();
Model\TiktokenTokenizer.cs (8)
29
private readonly Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)> _vocab;
78
Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)> vocab,
156
internal static async ValueTask<(Dictionary<ReadOnlyMemory<byte>, int>, Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)>, Dictionary<int, ReadOnlyMemory<byte>>)> LoadTiktokenBpeAsync(
160
Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)> vocab;
182
vocab = new Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)>(suggestedCapacity);
1186
private static readonly ConcurrentDictionary<string, (Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder)> _tiktokenCache = new(StringComparer.OrdinalIgnoreCase);
1210
out (Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder) cache))
1287
(Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<
StringSpanOrdinalKey
, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder) =
Utils\LruCache.cs (1)
17
private readonly Dictionary<
StringSpanOrdinalKey
, LinkedListNode<KeyValuePair<string, TValue>>> _cache = new();
Utils\StringSpanOrdinalKey.cs (20)
18
internal readonly unsafe struct StringSpanOrdinalKey : IEquatable<
StringSpanOrdinalKey
>
40
obj is
StringSpanOrdinalKey
wrapper && Equals(wrapper);
42
public bool Equals(
StringSpanOrdinalKey
other) =>
50
private readonly
StringSpanOrdinalKey
_left;
51
private readonly
StringSpanOrdinalKey
_right;
76
private readonly Dictionary<
StringSpanOrdinalKey
, TValue> _map;
85
_map = new Dictionary<
StringSpanOrdinalKey
, TValue>(capacity);
128
/// Custom JSON converter for <see cref="
StringSpanOrdinalKey
"/>.
130
internal sealed class StringSpanOrdinalKeyConverter : JsonConverter<
StringSpanOrdinalKey
>
133
public override
StringSpanOrdinalKey
ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) =>
136
public override void WriteAsPropertyName(Utf8JsonWriter writer,
StringSpanOrdinalKey
value, JsonSerializerOptions options) =>
139
public override
StringSpanOrdinalKey
Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new StringSpanOrdinalKey(reader.GetString()!);
140
public override void Write(Utf8JsonWriter writer,
StringSpanOrdinalKey
value, JsonSerializerOptions options) => writer.WriteStringValue(value.Data!);
143
internal class StringSpanOrdinalKeyCustomConverter : JsonConverter<Dictionary<
StringSpanOrdinalKey
, (int, string)>>
147
public override Dictionary<
StringSpanOrdinalKey
, (int, string)> Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
149
var dictionary = new Dictionary<
StringSpanOrdinalKey
, (int, string)>();
168
public override void Write(Utf8JsonWriter writer, Dictionary<
StringSpanOrdinalKey
, (int, string)> value, JsonSerializerOptions options) => throw new NotImplementedException();
172
/// Extension methods for <see cref="
StringSpanOrdinalKey
"/>.
176
public static unsafe bool TryGetValue<TValue>(this Dictionary<
StringSpanOrdinalKey
, TValue> map, ReadOnlySpan<char> key, out TValue value)
184
public static bool TryGetValue<TValue>(this Dictionary<
StringSpanOrdinalKey
, TValue> map, string key, out TValue value) =>