5 references to Vocabulary
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
62
throw new InvalidOperationException($"Unknown Token '{value}' was not present in '{nameof(
Vocabulary
)}'.");
Microsoft.ML.Tokenizers.Tests (4)
BpeTests.cs (4)
316
var reverseVocabulary = bpe.
Vocabulary
.ToDictionary(kvp => kvp.Value, kvp => kvp.Key);
325
Assert.Equal(encodingIds[i], bpe.
Vocabulary
[encoding[i].Value]);
814
private static IReadOnlyDictionary<int, string> _vocabReverse = _deepSeekR1Tokenizer.
Vocabulary
.ToDictionary(kvp => kvp.Value, kvp => kvp.Key);
908
Assert.True(bpeTokenizer.
Vocabulary
.TryGetValue(options.UnknownToken, out int unkId));