9 references to Vocabulary
Microsoft.ML.Tokenizers.Tests (9)
LlamaTests.cs (4)
253
var reverseVocabulary = bpe.
Vocabulary
.ToDictionary(kvp => kvp.Value, kvp => kvp.Key);
258
Assert.Equal(ids[i], bpe.
Vocabulary
[tokens[i]]);
360
Assert.True(bpe.
Vocabulary
.Count > 0);
361
Assert.True(bpe.
Vocabulary
.TryGetValue("▁", out _));
UnigramTests.cs (5)
831
Assert.Equal(0, _unigramTokenizerFromJson.
Vocabulary
["<s>"]);
832
Assert.Equal(1, _unigramTokenizerFromJson.
Vocabulary
["<pad>"]);
833
Assert.Equal(2, _unigramTokenizerFromJson.
Vocabulary
["</s>"]);
834
Assert.Equal(3, _unigramTokenizerFromJson.
Vocabulary
["<unk>"]);
835
Assert.Equal(250001, _unigramTokenizerFromJson.
Vocabulary
["<mask>"]);