1 instantiation of BpeSymbol
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBpeModel.cs (1)
1141BpeSymbol s = new(
16 references to BpeSymbol
Microsoft.ML.Tokenizers (16)
Model\SentencePieceBpeModel.cs (16)
152BpeSymbol[] symbols = ArrayPool<BpeSymbol>.Shared.Rent(text.Length); 199ArrayPool<BpeSymbol>.Shared.Return(symbols); 435BpeSymbol[] symbols = ArrayPool<BpeSymbol>.Shared.Rent(text.Length); 464ArrayPool<BpeSymbol>.Shared.Return(symbols); 478ArrayPool<BpeSymbol>.Shared.Return(symbols); 491ArrayPool<BpeSymbol>.Shared.Return(symbols); 703BpeSymbol[] symbols = ArrayPool<BpeSymbol>.Shared.Rent(text.Length); 756ArrayPool<BpeSymbol>.Shared.Return(symbols); 972BpeSymbol[] symbols = ArrayPool<BpeSymbol>.Shared.Rent(text.Length); 1032ArrayPool<BpeSymbol>.Shared.Return(symbols); 1129private Dictionary<(int Index, int Len), (int LeftIndex, int LeftLen, int RightIndex, int RightLen)>? Encode(ReadOnlySpan<char> text, BpeSymbol[] symbols) 1141BpeSymbol s = new(