4 references to Vocabulary
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
62
throw new InvalidOperationException($"Unknown Token '{value}' was not present in '{nameof(
Vocabulary
)}'.");
Microsoft.ML.Tokenizers.Tests (3)
BpeTests.cs (3)
303
var reverseVocabulary = bpe.
Vocabulary
.ToDictionary(kvp => kvp.Value, kvp => kvp.Key);
312
Assert.Equal(encodingIds[i], bpe.
Vocabulary
[encoding[i].Value]);
801
private static IReadOnlyDictionary<int, string> _vocabReverse = _deepSeekR1Tokenizer.
Vocabulary
.ToDictionary(kvp => kvp.Value, kvp => kvp.Key);