3 writes to _encoder
Microsoft.ML.Tokenizers (3)
Model\TiktokenTokenizer.cs (3)
85_encoder = encoder ?? throw new ArgumentNullException(nameof(encoder)); 89_encoder = encoder!; 107(_encoder, _vocab, _decoder) = LoadTiktokenBpeAsync(vocabStream, useAsync: false).GetAwaiter().GetResult();
5 references to _encoder
Microsoft.ML.Tokenizers (5)
Model\TiktokenTokenizer.cs (5)
331(int Id, int TokenIndex, int TokenLength)[] encodedTokens = BytePairEncoder.BytePairEncode(arrayPoolArray.AsMemory(0, encodedLength), _encoder, indexMappingSpan.Slice(0, encodedLength + 1)); 448(int Id, int TokenIndex, int TokenLength)[] encodedTokens = BytePairEncoder.BytePairEncode(arrayPoolArray.AsMemory(0, encodedLength), _encoder, indexMappingSpan.Slice(0, encodedLength + 1)); 613(int Id, int TokenIndex, int TokenLength)[] encodedTokens = BytePairEncoder.BytePairEncode(arrayPoolArray.AsMemory(0, encodedLength), _encoder, indexMappingSpan.Slice(0, encodedLength + 1)); 739(int Id, int TokenIndex, int TokenLength)[] encodedTokens = BytePairEncoder.BytePairEncode(arrayPoolArray.AsMemory(0, encodedLength), _encoder, indexMappingSpan.Slice(0, encodedLength + 1)); 1001internal IReadOnlyDictionary<ReadOnlyMemory<byte>, int> Encoder => _encoder;