1 instantiation of BpeSymbol
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
1619
symbols[i] = new
BpeSymbol
(
3 references to BpeSymbol
Microsoft.ML.Tokenizers (3)
Model\CodeGenTokenizer.cs (3)
1613
BpeSymbol
[] symbols = ArrayPool<
BpeSymbol
>.Shared.Rent(text.Length);
1677
ArrayPool<
BpeSymbol
>.Shared.Return(symbols);