34 instantiations of StringSpanOrdinalKey
Microsoft.ML.Tokenizers (34)
Model\BertTokenizer.cs (7)
39SeparatorTokenId = vocab[new StringSpanOrdinalKey(options.SeparatorToken)]; 42PaddingTokenId = vocab[new StringSpanOrdinalKey(options.PaddingToken)]; 45ClassificationTokenId = vocab[new StringSpanOrdinalKey(options.ClassificationToken)]; 48MaskingTokenId = vocab[new StringSpanOrdinalKey(options.MaskingToken)]; 778if (!vocab.TryGetValue(new StringSpanOrdinalKey(kvp.Key), out int id) || id != kvp.Value) 816if (token is null || !vocab.TryGetValue(new StringSpanOrdinalKey(token), out int id)) 835vocab[new StringSpanOrdinalKey(normalizedToken)] = id;
Model\BPETokenizer.cs (1)
259_specialTokens = specialTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key));
Model\CodeGenTokenizer.cs (2)
134_specialTokens = specialTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key)); 1628return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)].Id, tokenValue, new Range(mapping[0], mapping[0] + 1)) };
Model\EnglishRobertaTokenizer.cs (2)
961return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)], tokenValue, new Range(indexMapping[0], indexMapping[0] + 1)) }; 1050tokens.Add(new EncodedToken(_vocab[new StringSpanOrdinalKey(w)], w, new Range(indexMapping[index], indexMapping[index] + w.Length)));
Model\SentencePieceTokenizer.cs (2)
51_vocab.Add(new StringSpanOrdinalKey(piece.Piece), (i, piece.Score, (byte)piece.Type)); 85_specialTokens.Add(new StringSpanOrdinalKey(item.Key), item.Value);
Model\TiktokenTokenizer.cs (1)
247vocab[new StringSpanOrdinalKey(decodedToken)] = (rank, decodedToken);
Model\WordPieceTokenizer.cs (1)
110vocab.Add(new StringSpanOrdinalKey(line), lineNumber);
Utils\LruCache.cs (5)
51if (_cache.TryGetValue(new StringSpanOrdinalKey(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached)) 78if (_cache.TryGetValue(new StringSpanOrdinalKey(ptr, key.Length), out LinkedListNode<KeyValuePair<string, TValue>>? cached)) 101if (_cache.TryGetValue(new StringSpanOrdinalKey(key), out LinkedListNode<KeyValuePair<string, TValue>>? cached)) 113_cache.Remove(new StringSpanOrdinalKey(nodeToEvict!.Value.Key)); 117_cache[new StringSpanOrdinalKey(key)] = node;
Utils\StringSpanOrdinalKey.cs (13)
56_left = new StringSpanOrdinalKey(ptr1, length1); 57_right = new StringSpanOrdinalKey(ptr2, length2); 62_left = new StringSpanOrdinalKey(data1); 63_right = new StringSpanOrdinalKey(data2); 93return _map.TryGetValue(new StringSpanOrdinalKey(key), out value!); 103return _map.TryGetValue(new StringSpanOrdinalKey(ptr, key.Length), out value!); 112_map.Remove(new StringSpanOrdinalKey(key)); 122_map[new StringSpanOrdinalKey(k)] = v; 137new StringSpanOrdinalKey(reader.GetString()!); 142public override StringSpanOrdinalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new StringSpanOrdinalKey(reader.GetString()!); 163dictionary.Add(new StringSpanOrdinalKey(key!), (value, key!)); 181return map.TryGetValue(new StringSpanOrdinalKey(ptr, key.Length), out value!); 186map.TryGetValue(new StringSpanOrdinalKey(key), out value!);
60 references to StringSpanOrdinalKey
Microsoft.ML.Tokenizers (60)
Model\BertTokenizer.cs (5)
28Dictionary<StringSpanOrdinalKey, int> vocab, 702(Dictionary<StringSpanOrdinalKey, int> vocab, Dictionary<int, string> vocabReverse) = await LoadVocabAsync(vocabStream, useAsync: true, cancellationToken).ConfigureAwait(false); 743(Dictionary<StringSpanOrdinalKey, int> vocab, Dictionary<int, string> vocabReverse) = LoadVocabAsync(vocabStream, useAsync: false).GetAwaiter().GetResult(); 757Dictionary<StringSpanOrdinalKey, int> vocab, 814private static void AddSpecialToken(Dictionary<StringSpanOrdinalKey, int> vocab, Dictionary<string, int> specialTokens, string token, bool lowerCase, Dictionary<string, int>? notNormalizedSpecialTokens = null)
Model\BPETokenizer.cs (10)
32private readonly Dictionary<StringSpanOrdinalKey, (int, string)>? _specialTokens; 129(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult(); 176(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult(); 212(Dictionary<StringSpanOrdinalKey, int>? vocab, Vec<(string, string)> merges) result = await ReadModelDataAsync(vocabStream, mergesStream, useAsync: true).ConfigureAwait(false); 230Dictionary<StringSpanOrdinalKey, int>? vocab, 246_vocab = vocab ?? new Dictionary<StringSpanOrdinalKey, int>(); 251foreach (KeyValuePair<StringSpanOrdinalKey, int> kvp in _vocab) 773internal static async ValueTask<(Dictionary<StringSpanOrdinalKey, int>?, Vec<(string, string)>)> ReadModelDataAsync(Stream vocab, Stream? merges, bool useAsync, CancellationToken cancellationToken = default) 775Dictionary<StringSpanOrdinalKey, int>? dic = useAsync 787private readonly Dictionary<StringSpanOrdinalKey, int> _vocab;
Model\CodeGenTokenizer.cs (3)
25private readonly Dictionary<StringSpanOrdinalKey, (int Id, string Token)> _vocab; 28private readonly Dictionary<StringSpanOrdinalKey, (int, string)>? _specialTokens; 1765private static Dictionary<StringSpanOrdinalKey, (int, string)> GetVocabulary(Stream vocabularyStream)
Model\EnglishRobertaTokenizer.cs (5)
23private readonly Dictionary<StringSpanOrdinalKey, int> _vocab; 25private readonly SortedDictionary<int, StringSpanOrdinalKey> _vocabReverse; 179private static Dictionary<StringSpanOrdinalKey, int> GetVocabulary(Stream vocabularyStream) 181Dictionary<StringSpanOrdinalKey, int>? vocab; 278if (_vocabReverse.TryGetValue(id, out var value))
Model\ModelSourceGenerationContext.cs (1)
10[JsonSerializable(typeof(Dictionary<StringSpanOrdinalKey, int>))]
Model\SentencePieceTokenizer.cs (3)
28private readonly Dictionary<StringSpanOrdinalKey, (int Id, float Score, byte Type)> _vocab = new(); 36private readonly Dictionary<StringSpanOrdinalKey, int>? _specialTokens; 80_specialTokens = new Dictionary<StringSpanOrdinalKey, int>();
Model\TiktokenTokenizer.cs (8)
30private readonly Dictionary<StringSpanOrdinalKey, (int Id, string Token)> _vocab; 79Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab, 157internal static async ValueTask<(Dictionary<ReadOnlyMemory<byte>, int>, Dictionary<StringSpanOrdinalKey, (int Id, string Token)>, Dictionary<int, ReadOnlyMemory<byte>>)> LoadTiktokenBpeAsync( 161Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab; 183vocab = new Dictionary<StringSpanOrdinalKey, (int Id, string Token)>(suggestedCapacity); 1197private static readonly ConcurrentDictionary<string, (Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder)> _tiktokenCache = new(StringComparer.OrdinalIgnoreCase); 1221out (Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder) cache)) 1312(Dictionary<ReadOnlyMemory<byte>, int> encoder, Dictionary<StringSpanOrdinalKey, (int Id, string Token)> vocab, Dictionary<int, ReadOnlyMemory<byte>> decoder) =
Model\WordPieceTokenizer.cs (7)
29private readonly Dictionary<StringSpanOrdinalKey, int> _vocab; 33Dictionary<StringSpanOrdinalKey, int> vocab, 92internal static async ValueTask<(Dictionary<StringSpanOrdinalKey, int>, Dictionary<int, string>)> LoadVocabAsync(Stream vocabStream, bool useAsync, CancellationToken cancellationToken = default) 99Dictionary<StringSpanOrdinalKey, int> vocab = new Dictionary<StringSpanOrdinalKey, int>(); 163(Dictionary<StringSpanOrdinalKey, int> vocab, Dictionary<int, string> vocabReverse) = LoadVocabAsync(vocabStream, useAsync: false).GetAwaiter().GetResult(); 227(Dictionary<StringSpanOrdinalKey, int> vocab, Dictionary<int, string> vocabReverse) = await LoadVocabAsync(vocabStream, useAsync: true, cancellationToken);
Utils\LruCache.cs (1)
17private readonly Dictionary<StringSpanOrdinalKey, LinkedListNode<KeyValuePair<string, TValue>>> _cache = new();
Utils\StringSpanOrdinalKey.cs (17)
19internal readonly unsafe struct StringSpanOrdinalKey : IEquatable<StringSpanOrdinalKey> 41obj is StringSpanOrdinalKey wrapper && Equals(wrapper); 43public bool Equals(StringSpanOrdinalKey other) => 51private readonly StringSpanOrdinalKey _left; 52private readonly StringSpanOrdinalKey _right; 77private readonly Dictionary<StringSpanOrdinalKey, TValue> _map; 86_map = new Dictionary<StringSpanOrdinalKey, TValue>(capacity); 129internal sealed class Vocabulary : Dictionary<StringSpanOrdinalKey, (int, string)>; 132/// Custom JSON converter for <see cref="StringSpanOrdinalKey"/>. 134internal sealed class StringSpanOrdinalKeyConverter : JsonConverter<StringSpanOrdinalKey> 136public override StringSpanOrdinalKey ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => 139public override void WriteAsPropertyName(Utf8JsonWriter writer, StringSpanOrdinalKey value, JsonSerializerOptions options) => 142public override StringSpanOrdinalKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new StringSpanOrdinalKey(reader.GetString()!); 143public override void Write(Utf8JsonWriter writer, StringSpanOrdinalKey value, JsonSerializerOptions options) => writer.WriteStringValue(value.Data!); 173/// Extension methods for <see cref="StringSpanOrdinalKey"/>. 177public static unsafe bool TryGetValue<TValue>(this Dictionary<StringSpanOrdinalKey, TValue> map, ReadOnlySpan<char> key, out TValue value) 185public static bool TryGetValue<TValue>(this Dictionary<StringSpanOrdinalKey, TValue> map, string key, out TValue value) =>