4 references to ModelSourceGenerationContext
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (2)
1098
? await JsonSerializer.DeserializeAsync(vocab,
ModelSourceGenerationContext
.Default.DictionaryStringSpanOrdinalKeyInt32, cancellationToken).ConfigureAwait(false)
1099
: JsonSerializer.Deserialize(vocab,
ModelSourceGenerationContext
.Default.DictionaryStringSpanOrdinalKeyInt32);
Model\CodeGenTokenizer.cs (1)
1752
vocab = JsonSerializer.Deserialize(vocabularyStream,
ModelSourceGenerationContext
.Default.Vocabulary);
Model\EnglishRobertaTokenizer.cs (1)
184
vocab = JsonSerializer.Deserialize(vocabularyStream,
ModelSourceGenerationContext
.Default.DictionaryStringSpanOrdinalKeyInt32);