3 references to Vocabulary
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
54
throw new InvalidOperationException($"Unknown Token '{value}' was not present in '{nameof(
Vocabulary
)}'.");
Microsoft.ML.Tokenizers.Tests (2)
BpeTests.cs (2)
272
var reverseVocabulary = bpe.
Vocabulary
.ToDictionary(kvp => kvp.Value, kvp => kvp.Key);
281
Assert.Equal(encodingIds[i], bpe.
Vocabulary
[encoding[i].Value]);