1 instantiation of BpeSymbol
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBpeModel.cs (1)
1141
BpeSymbol s =
new
(
16 references to BpeSymbol
Microsoft.ML.Tokenizers (16)
Model\SentencePieceBpeModel.cs (16)
152
BpeSymbol
[] symbols = ArrayPool<
BpeSymbol
>.Shared.Rent(text.Length);
199
ArrayPool<
BpeSymbol
>.Shared.Return(symbols);
435
BpeSymbol
[] symbols = ArrayPool<
BpeSymbol
>.Shared.Rent(text.Length);
464
ArrayPool<
BpeSymbol
>.Shared.Return(symbols);
478
ArrayPool<
BpeSymbol
>.Shared.Return(symbols);
491
ArrayPool<
BpeSymbol
>.Shared.Return(symbols);
703
BpeSymbol
[] symbols = ArrayPool<
BpeSymbol
>.Shared.Rent(text.Length);
756
ArrayPool<
BpeSymbol
>.Shared.Return(symbols);
972
BpeSymbol
[] symbols = ArrayPool<
BpeSymbol
>.Shared.Rent(text.Length);
1032
ArrayPool<
BpeSymbol
>.Shared.Return(symbols);
1129
private Dictionary<(int Index, int Len), (int LeftIndex, int LeftLen, int RightIndex, int RightLen)>? Encode(ReadOnlySpan<char> text,
BpeSymbol
[] symbols)
1141
BpeSymbol
s = new(