1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
246
_vocab
= vocab ?? new Dictionary<StringSpanOrdinalKey, int>();
10 references to _vocab
Microsoft.ML.Tokenizers (10)
Model\BPETokenizer.cs (10)
59
if (!
_vocab
.TryGetValue(value, out int id))
251
foreach (KeyValuePair<StringSpanOrdinalKey, int> kvp in
_vocab
)
272
if (!
_vocab
.TryGetValue(mergeValues.a, out int aId))
277
if (!
_vocab
.TryGetValue(mergeValues.b, out int bId))
288
if (!
_vocab
.TryGetValue(newToken, out int newId))
589
private int? MapTokenToId(ReadOnlySpan<char> token) =>
_vocab
.TryGetValue(token, out int value) ? value : null;
609
public IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= new ReadOnlyDictionary<string, int>(
_vocab
.ToDictionary(kvp => kvp.Key.Data!, kvp => kvp.Value));
928
if (
_vocab
.TryGetValue(s, out int id))
950
if (!
_vocab
.TryGetValue(UnknownToken, out int value))
959
if (!
_vocab
.TryGetValue(UnknownToken, out int value))