9 references to _spaceSymbol
Microsoft.ML.Tokenizers (9)
Normalizer\SentencePieceNormalizer.cs (9)
403if (normalized.Length <= normalizedIndex + _spaceSymbol.Length) 405Helpers.ArrayPoolGrow(ref normalized, ref poolArray, (normalizedIndex + _spaceSymbol.Length) << 1); 409_spaceSymbol.AsSpan().CopyTo(normalized.Slice(normalizedIndex)); 410normalizedIndex += _spaceSymbol.Length; 440Span<byte> space = EscapeWhiteSpaces ? _spaceSymbol : _space; 467if (normalized.Length <= normalizedIndex + _spaceSymbol.Length) 469Helpers.ArrayPoolGrow(ref normalized, ref poolArray, (normalizedIndex + _spaceSymbol.Length) << 1); 471_spaceSymbol.AsSpan().CopyTo(normalized.Slice(normalizedIndex)); 472normalizedIndex += _spaceSymbol.Length;