7 instantiations of Vec
Microsoft.ML.Tokenizers (7)
Model\BPETokenizer.cs (2)
812
return new
Vec
<(string, string)>();
818
Vec<(string, string)> merges =
new
(1000);
Model\Word.cs (5)
17
public Word() => _symbols = new
Vec
<Symbol>();
25
_symbols = new
Vec
<Symbol>(capacity);
51
Vec<(Pair<int>, int)> changes =
new
();
105
Vec<Merge> skip = new
Vec
<Merge>(priorityQueue.Count);
265
Vec<int> chars = new
Vec
<int>();
14 references to Vec
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (8)
129
(Dictionary<StringSpanOrdinalKey, int>? vocab,
Vec
<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult();
176
(Dictionary<StringSpanOrdinalKey, int>? vocab,
Vec
<(string, string)> merges) result = ReadModelDataAsync(vocabStream, mergesStream, useAsync: false).GetAwaiter().GetResult();
212
(Dictionary<StringSpanOrdinalKey, int>? vocab,
Vec
<(string, string)> merges) result = await ReadModelDataAsync(vocabStream, mergesStream, useAsync: true).ConfigureAwait(false);
231
Vec
<(string, string)> merges,
773
internal static async ValueTask<(Dictionary<StringSpanOrdinalKey, int>?,
Vec
<(string, string)>)> ReadModelDataAsync(Stream vocab, Stream? merges, bool useAsync, CancellationToken cancellationToken = default)
779
var
m = useAsync ?
808
internal static async ValueTask<
Vec
<(string, string)>> ConvertMergesToHashmapAsync(Stream? mergesStream, bool useAsync = false, CancellationToken cancellationToken = default)
818
Vec
<(string, string)> merges = new(1000);
Model\Word.cs (6)
15
private
Vec
<Symbol> _symbols;
49
public
Vec
<(Pair<int>, int)> Merge(int c1, int c2, int replacement)
51
Vec
<(Pair<int>, int)> changes = new();
105
Vec
<Merge> skip = new Vec<Merge>(priorityQueue.Count);
263
public
Vec
<int> GetChars()
265
Vec
<int> chars = new Vec<int>();