3 references to Vocabulary
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
61throw new InvalidOperationException($"Unknown Token '{value}' was not present in '{nameof(Vocabulary)}'.");
Microsoft.ML.Tokenizers.Tests (2)
BpeTests.cs (2)
272var reverseVocabulary = bpe.Vocabulary.ToDictionary(kvp => kvp.Value, kvp => kvp.Key); 281Assert.Equal(encodingIds[i], bpe.Vocabulary[encoding[i].Value]);