3 references to LoadTiktokenBpeAsync
Microsoft.ML.Tokenizers (3)
Model\TiktokenTokenizer.cs (3)
107
(_encoder, _vocab, _decoder) =
LoadTiktokenBpeAsync
(vocabStream, useAsync: false).GetAwaiter().GetResult();
1231
cache =
LoadTiktokenBpeAsync
(deflateStream, useAsync: false).GetAwaiter().GetResult();
1304
await
LoadTiktokenBpeAsync
(vocabStream, useAsync: true, cancellationToken).ConfigureAwait(false);