9 references to _spaceSymbol
Microsoft.ML.Tokenizers (9)
Normalizer\SentencePieceNormalizer.cs (9)
484if (normalized.Length <= normalizedIndex + _spaceSymbol.Length) 486Helpers.ArrayPoolGrow(ref normalized, ref poolArray, (normalizedIndex + _spaceSymbol.Length) << 1); 490_spaceSymbol.AsSpan().CopyTo(normalized.Slice(normalizedIndex)); 491normalizedIndex += _spaceSymbol.Length; 521Span<byte> space = EscapeWhiteSpaces ? _spaceSymbol : _space; 548if (normalized.Length <= normalizedIndex + _spaceSymbol.Length) 550Helpers.ArrayPoolGrow(ref normalized, ref poolArray, (normalizedIndex + _spaceSymbol.Length) << 1); 552_spaceSymbol.AsSpan().CopyTo(normalized.Slice(normalizedIndex)); 553normalizedIndex += _spaceSymbol.Length;