3 writes to _decoder
Microsoft.ML.Tokenizers (3)
Model\TiktokenTokenizer.cs (3)
86_decoder = decoder ?? throw new ArgumentNullException(nameof(decoder)); 90_decoder = decoder!; 107(_encoder, _vocab, _decoder) = LoadTiktokenBpeAsync(vocabStream, useAsync: false).GetAwaiter().GetResult();
5 references to _decoder
Microsoft.ML.Tokenizers (5)
Model\TiktokenTokenizer.cs (5)
137Debug.Assert(_decoder is not null); 143_decoder![specialToken.Value] = Encoding.UTF8.GetBytes(specialToken.Key); 817if (_decoder.TryGetValue(id, out ReadOnlyMemory<byte> tokenBytes)) 891if (_decoder.TryGetValue(id, out ReadOnlyMemory<byte> tokenBytes)) 1006internal IReadOnlyDictionary<int, ReadOnlyMemory<byte>> Decoder => _decoder;