8 instantiations of Vec
Microsoft.ML.Tokenizers (8)
Model\BPETokenizer.cs (3)
167
merges = new
Vec
<(string, string)>(1000);
1138
return new
Vec
<(string, string)>();
1144
Vec<(string, string)> merges =
new
(1000);
Model\Word.cs (5)
17
public Word() => _symbols = new
Vec
<Symbol>();
25
_symbols = new
Vec
<Symbol>(capacity);
51
Vec<(Pair<int>, int)> changes =
new
();
105
Vec<Merge> skip = new
Vec
<Merge>(priorityQueue.Count);
265
Vec<int> chars = new
Vec
<int>();
15 references to Vec
Microsoft.ML.Tokenizers (15)
Model\BPETokenizer.cs (9)
130
(Dictionary<StringSpanOrdinalKey, int>? vocab,
Vec
<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult();
164
Vec
<(string, string)> merges = default;
242
(Dictionary<StringSpanOrdinalKey, int>? vocab,
Vec
<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult();
278
(Dictionary<StringSpanOrdinalKey, int>? vocab,
Vec
<(string, string)> merges) result = await ReadModelDataAsync(vocabStream, mergesStream, useAsync: true).ConfigureAwait(false);
300
Vec
<(string, string)> merges,
1099
internal static async ValueTask<(Dictionary<StringSpanOrdinalKey, int>?,
Vec
<(string, string)>)> ReadModelDataAsync(Stream vocab, Stream? merges, bool useAsync, CancellationToken cancellationToken = default)
1105
var
m = useAsync ?
1134
internal static async ValueTask<
Vec
<(string, string)>> ConvertMergesToHashmapAsync(Stream? mergesStream, bool useAsync = false, CancellationToken cancellationToken = default)
1144
Vec
<(string, string)> merges = new(1000);
Model\Word.cs (6)
15
private
Vec
<Symbol> _symbols;
49
public
Vec
<(Pair<int>, int)> Merge(int c1, int c2, int replacement)
51
Vec
<(Pair<int>, int)> changes = new();
105
Vec
<Merge> skip = new Vec<Merge>(priorityQueue.Count);
263
public
Vec
<int> GetChars()
265
Vec
<int> chars = new Vec<int>();