1 write to Data
Microsoft.ML.Tokenizers (1)
Utils\StringSpanOrdinalKey.cs (1)
32
Data
= data;
8 references to Data
Microsoft.ML.Tokenizers (8)
Model\BPETokenizer.cs (2)
253
VocabReverse.Add(kvp.Value, kvp.Key.
Data
!);
609
public IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= new ReadOnlyDictionary<string, int>(_vocab.ToDictionary(kvp => kvp.Key.
Data
!, kvp => kvp.Value));
Model\EnglishRobertaTokenizer.cs (1)
280
string v = value.
Data
!;
Model\TiktokenTokenizer.cs (1)
991
internal IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= _vocab.ToDictionary(kvp => kvp.Key.
Data
!, kvp => kvp.Value.Id);
Utils\StringSpanOrdinalKey.cs (4)
36
Data
.AsSpan();
38
public override string ToString() =>
Data
?? Span.ToString();
140
writer.WriteStringValue(value.
Data
!);
143
public override void Write(Utf8JsonWriter writer, StringSpanOrdinalKey value, JsonSerializerOptions options) => writer.WriteStringValue(value.
Data
!);