1 write to Data
Microsoft.ML.Tokenizers (1)
Utils\StringSpanOrdinalKey.cs (1)
31
Data
= data;
8 references to Data
Microsoft.ML.Tokenizers (8)
Model\BPETokenizer.cs (2)
215
VocabReverse.Add(kvp.Value, kvp.Key.
Data
!);
564
public IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= new ReadOnlyDictionary<string, int>(_vocab.ToDictionary(kvp => kvp.Key.
Data
!, kvp => kvp.Value));
Model\EnglishRobertaTokenizer.cs (1)
269
string v = value.
Data
!;
Model\TiktokenTokenizer.cs (1)
994
internal IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= _vocab.ToDictionary(kvp => kvp.Key.
Data
!, kvp => kvp.Value.Id);
Utils\StringSpanOrdinalKey.cs (4)
35
Data
.AsSpan();
37
public override string ToString() =>
Data
?? Span.ToString();
137
writer.WriteStringValue(value.
Data
!);
140
public override void Write(Utf8JsonWriter writer, StringSpanOrdinalKey value, JsonSerializerOptions options) => writer.WriteStringValue(value.
Data
!);