2 instantiations of State
Microsoft.ML.Tokenizers (2)
Utils\BytePairEncoder.cs (2)
162
state[0] = new
State
182
state[i + 1] = new
State
5 references to State
Microsoft.ML.Tokenizers (5)
Utils\BytePairEncoder.cs (5)
159
State
[] statePoolArray = ArrayPool<
State
>.Shared.Rent(stateLength);
160
Span<
State
> state = statePoolArray.AsSpan(0, stateLength);
193
void PotentialMerge(Span<
State
> stateSpan, PriorityQueue<MergeEntry> heapQueue, int start, int nextEndItem)
283
ArrayPool<
State
>.Shared.Return(statePoolArray);