25 instantiations of StringSpanOrdinalKey
Microsoft.ML.Tokenizers (25)
Model\CodeGenTokenizer.cs (2)
134_addedTokens = addedTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key)); 1625return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)].Id, tokenValue, (mapping[0], 1)) };
Model\EnglishRobertaTokenizer.cs (2)
950return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)], tokenValue, (indexMapping[0], 1)) }; 1039tokens.Add(new EncodedToken(_vocab[new StringSpanOrdinalKey(w)], w, (indexMapping[index], w.Length)));
Model\SentencePieceBpeTokenizer.cs (2)
51_vocab.Add(new StringSpanOrdinalKey(piece.Piece), (i, piece.Score, (byte)piece.Type)); 85_specialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
Model\TiktokenTokenizer.cs (1)
246vocab[new StringSpanOrdinalKey(decodedToken)] = (rank, decodedToken);
Utils\LruCache.cs (5)
51if (_cache.TryGetValue(new StringSpanOrdinalKey(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached)) 78if (_cache.TryGetValue(new StringSpanOrdinalKey(ptr, key.Length), out LinkedListNode<KeyValuePair<string, TValue>>? cached)) 101if (_cache.TryGetValue(new StringSpanOrdinalKey(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached)) 113_cache.Remove(new StringSpanOrdinalKey(nodeToEvict!.Value.Key)); 117_cache[new StringSpanOrdinalKey(key)] = node;
Utils\StringSpanOrdinalKey.cs (13)
55_left = new StringSpanOrdinalKey(ptr1, length1); 56_right = new StringSpanOrdinalKey(ptr2, length2); 61_left = new StringSpanOrdinalKey(data1); 62_right = new StringSpanOrdinalKey(data2); 92return _map.TryGetValue(new StringSpanOrdinalKey(key), out value!); 102return _map.TryGetValue(new StringSpanOrdinalKey(ptr, key.Length), out value!); 111_map.Remove(new StringSpanOrdinalKey(key)); 121_map[new StringSpanOrdinalKey(k)] = v; 134new StringSpanOrdinalKey(reader.GetString()!); 139public override StringSpanOrdinalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new StringSpanOrdinalKey(reader.GetString()!); 162dictionary.Add(new StringSpanOrdinalKey(key!), (value, key!)); 180return map.TryGetValue(new StringSpanOrdinalKey(ptr, key.Length), out value!); 185map.TryGetValue(new StringSpanOrdinalKey(key), out value!);
58 references to StringSpanOrdinalKey
Microsoft.ML.Tokenizers (58)
Model\BPETokenizer.cs (13)
114(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult(); 153(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult(); 184(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = await ReadModelDataAsync(vocabStream, mergesStream, useAsync: true).ConfigureAwait(false); 200private BpeTokenizer(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges, PreTokenizer? preTokenizer, Normalizer? normalizer, string? unknownToken, string? continuingSubwordPrefix, string? endOfWordSuffix, bool fuseUnknownTokens) 208_vocab = vocab ?? new Dictionary<StringSpanOrdinalKey, int>(); 213foreach (KeyValuePair<StringSpanOrdinalKey, int> kvp in _vocab) 728internal static async ValueTask<(Dictionary<StringSpanOrdinalKey, int>?, Vec<(string, string)>)> ReadModelDataAsync(Stream vocab, Stream? merges, bool useAsync, CancellationToken cancellationToken = default) 732Dictionary<StringSpanOrdinalKey, int>? dic = useAsync ? 733await JsonSerializer.DeserializeAsync<Dictionary<StringSpanOrdinalKey, int>>(vocab, options, cancellationToken).ConfigureAwait(false) as Dictionary<StringSpanOrdinalKey, int> : 734JsonSerializer.Deserialize<Dictionary<StringSpanOrdinalKey, int>>(vocab, options) as Dictionary<StringSpanOrdinalKey, int>; 743private readonly Dictionary<StringSpanOrdinalKey, int> _vocab;
Model\CodeGenTokenizer.cs (6)
25private readonly Dictionary<StringSpanOrdinalKey, (int Id, string Token)> _vocab; 28private readonly Dictionary<StringSpanOrdinalKey, (int, string)>? _addedTokens; 1763private static Dictionary<StringSpanOrdinalKey, (int, string)> GetVocabulary(Stream vocabularyStream) 1765Dictionary<StringSpanOrdinalKey, (int, string)>? vocab; 1769vocab = JsonSerializer.Deserialize<Dictionary<StringSpanOrdinalKey, (int, string)>>(vocabularyStream, options) as Dictionary<StringSpanOrdinalKey, (int, string)>;
Model\EnglishRobertaTokenizer.cs (7)
23private readonly Dictionary<StringSpanOrdinalKey, int> _vocab; 25private readonly SortedDictionary<int, StringSpanOrdinalKey> _vocabReverse; 167private static Dictionary<StringSpanOrdinalKey, int> GetVocabulary(Stream vocabularyStream) 169Dictionary<StringSpanOrdinalKey, int>? vocab; 173vocab = JsonSerializer.Deserialize<Dictionary<StringSpanOrdinalKey, int>>(vocabularyStream, options) as Dictionary<StringSpanOrdinalKey, int>; 267if (_vocabReverse.TryGetValue(id, out var value))
Model\SentencePieceBpeTokenizer.cs (3)
28private readonly Dictionary<StringSpanOrdinalKey, (int Id, float Score, byte Type)> _vocab = new(); 36private readonly Dictionary<StringSpanOrdinalKey, int>? _specialTokens; 80_specialTokens = new Dictionary<StringSpanOrdinalKey, int>();
Model\TiktokenTokenizer.cs (8)
29private readonly Dictionary<StringSpanOrdinalKey, (int Id, string Token)> _vocab; 78Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab, 156internal static async ValueTask<(Dictionary<ReadOnlyMemory<byte>, int>, Dictionary<StringSpanOrdinalKey, (int Id, string Token)>, Dictionary<int, ReadOnlyMemory<byte>>)> LoadTiktokenBpeAsync( 160Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab; 182vocab = new Dictionary<StringSpanOrdinalKey, (int Id, string Token)>(suggestedCapacity); 1186private static readonly ConcurrentDictionary<string, (Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder)> _tiktokenCache = new(StringComparer.OrdinalIgnoreCase); 1210out (Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder) cache)) 1287(Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder) =
Utils\LruCache.cs (1)
17private readonly Dictionary<StringSpanOrdinalKey, LinkedListNode<KeyValuePair<string, TValue>>> _cache = new();
Utils\StringSpanOrdinalKey.cs (20)
18internal readonly unsafe struct StringSpanOrdinalKey : IEquatable<StringSpanOrdinalKey> 40obj is StringSpanOrdinalKey wrapper && Equals(wrapper); 42public bool Equals(StringSpanOrdinalKey other) => 50private readonly StringSpanOrdinalKey _left; 51private readonly StringSpanOrdinalKey _right; 76private readonly Dictionary<StringSpanOrdinalKey, TValue> _map; 85_map = new Dictionary<StringSpanOrdinalKey, TValue>(capacity); 128/// Custom JSON converter for <see cref="StringSpanOrdinalKey"/>. 130internal sealed class StringSpanOrdinalKeyConverter : JsonConverter<StringSpanOrdinalKey> 133public override StringSpanOrdinalKey ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => 136public override void WriteAsPropertyName(Utf8JsonWriter writer, StringSpanOrdinalKey value, JsonSerializerOptions options) => 139public override StringSpanOrdinalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new StringSpanOrdinalKey(reader.GetString()!); 140public override void Write(Utf8JsonWriter writer, StringSpanOrdinalKey value, JsonSerializerOptions options) => writer.WriteStringValue(value.Data!); 143internal class StringSpanOrdinalKeyCustomConverter : JsonConverter<Dictionary<StringSpanOrdinalKey, (int, string)>> 147public override Dictionary<StringSpanOrdinalKey, (int, string)> Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) 149var dictionary = new Dictionary<StringSpanOrdinalKey, (int, string)>(); 168public override void Write(Utf8JsonWriter writer, Dictionary<StringSpanOrdinalKey, (int, string)> value, JsonSerializerOptions options) => throw new NotImplementedException(); 172/// Extension methods for <see cref="StringSpanOrdinalKey"/>. 176public static unsafe bool TryGetValue<TValue>(this Dictionary<StringSpanOrdinalKey, TValue> map, ReadOnlySpan<char> key, out TValue value) 184public static bool TryGetValue<TValue>(this Dictionary<StringSpanOrdinalKey, TValue> map, string key, out TValue value) =>