| File: Utils\ValueStringBuilder.cs | Web Access |
| Project: src\src\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj (Microsoft.ML.Tokenizers) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Buffers; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Microsoft.ML.Tokenizers; #nullable enable namespace System.Text { internal ref partial struct ValueStringBuilder { private char[]? _arrayToReturnToPool; private Span<char> _chars; private int _pos; public ValueStringBuilder(Span<char> initialBuffer) { _arrayToReturnToPool = null; _chars = initialBuffer; _pos = 0; } public ValueStringBuilder(int initialCapacity) { _arrayToReturnToPool = ArrayPool<char>.Shared.Rent(initialCapacity); _chars = _arrayToReturnToPool; _pos = 0; } public int Length { get => _pos; set { Debug.Assert(value >= 0); Debug.Assert(value <= _chars.Length); _pos = value; } } public int Capacity => _chars.Length; public void EnsureCapacity(int capacity) { // This is not expected to be called this with negative capacity Debug.Assert(capacity >= 0); // If the caller has a bug and calls this with negative capacity, make sure to call Grow to throw an exception. if ((uint)capacity > (uint)_chars.Length) Grow(capacity - _pos); } /// <summary> /// Get a pinnable reference to the builder. /// Does not ensure there is a null char after <see cref="Length"/> /// This overload is pattern matched in the C# 7.3+ compiler so you can omit /// the explicit method call, and write eg "fixed (char* c = builder)" /// </summary> public ref char GetPinnableReference() { return ref MemoryMarshal.GetReference(_chars); } /// <summary> /// Get a pinnable reference to the builder. /// </summary> /// <param name="terminate">Ensures that the builder has a null char after <see cref="Length"/></param> public ref char GetPinnableReference(bool terminate) { if (terminate) { EnsureCapacity(Length + 1); _chars[Length] = '\0'; } return ref MemoryMarshal.GetReference(_chars); } public ref char this[int index] { get { Debug.Assert(index < _pos); return ref _chars[index]; } } // Replace a char before returning the string public string ToString(char oldValue, char newValue) { Span<char> span = _chars.Slice(0, _pos); Helpers.Replace(span, oldValue, newValue); string s = span.ToString(); Dispose(); return s; } public override string ToString() { string s = _chars.Slice(0, _pos).ToString(); Dispose(); return s; } /// <summary>Returns the underlying storage of the builder.</summary> public Span<char> RawChars => _chars; /// <summary> /// Returns a span around the contents of the builder. /// </summary> /// <param name="terminate">Ensures that the builder has a null char after <see cref="Length"/></param> public ReadOnlySpan<char> AsSpan(bool terminate) { if (terminate) { EnsureCapacity(Length + 1); _chars[Length] = '\0'; } return _chars.Slice(0, _pos); } /// <summary> /// Remove last character in the builder. /// </summary> public void RemoveLastChar() { if (_pos > 0) { _pos--; } } public ReadOnlySpan<char> AsSpan() => _chars.Slice(0, _pos); public ReadOnlySpan<char> AsSpan(int start) => _chars.Slice(start, _pos - start); public ReadOnlySpan<char> AsSpan(int start, int length) => _chars.Slice(start, length); public bool TryCopyTo(Span<char> destination, out int charsWritten) { if (_chars.Slice(0, _pos).TryCopyTo(destination)) { charsWritten = _pos; Dispose(); return true; } else { charsWritten = 0; Dispose(); return false; } } public void Replace(string oldValue, string newValue) { int oldLength = oldValue.Length; int newLength = newValue.Length; int index = 0; while (index <= _pos - oldLength) { ReadOnlySpan<char> buffer = _chars.Slice(index, _pos - index); int subIndex = buffer.IndexOf(oldValue.AsSpan(), StringComparison.Ordinal); if (subIndex < 0) { break; } index += subIndex; if (oldLength >= newLength) { newValue.AsSpan().CopyTo(_chars.Slice(index)); if (oldLength > newLength) { int newIndex = index + oldLength; _chars.Slice(newIndex, _pos - newIndex).CopyTo(_chars.Slice(index + newLength)); _pos -= oldLength - newLength; } } else { Insert(index, newValue); int newIndex = index + newLength + oldLength; _chars.Slice(newIndex, _pos - newIndex).CopyTo(_chars.Slice(index + newLength)); _pos -= oldLength; } index += newLength; } } public bool RemoveSuffix(string value) { if (EndsWith(value)) { _pos -= value.Length; return true; } return false; } public void Remove(int start, int length) { if (length > 0 && start + length <= _pos) { int remaining = _pos - start - length; _chars.Slice(start + length, remaining).CopyTo(_chars.Slice(start)); _pos -= length; } } public bool EndsWith(string value) { int valueLength = value.Length; if (valueLength > _pos) { return false; } return _chars.Slice(_pos - valueLength, valueLength).SequenceEqual(value.AsSpan()); } public void Insert(int index, char value, int count) { if (_pos > _chars.Length - count) { Grow(count); } int remaining = _pos - index; _chars.Slice(index, remaining).CopyTo(_chars.Slice(index + count)); _chars.Slice(index, count).Fill(value); _pos += count; } public void Insert(int index, string? s) { if (s == null) { return; } int count = s.Length; if (_pos > (_chars.Length - count)) { Grow(count); } int remaining = _pos - index; _chars.Slice(index, remaining).CopyTo(_chars.Slice(index + count)); s #if !NETCOREAPP .AsSpan() #endif .CopyTo(_chars.Slice(index)); _pos += count; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Append(char c) { int pos = _pos; Span<char> chars = _chars; if ((uint)pos < (uint)chars.Length) { chars[pos] = c; _pos = pos + 1; } else { GrowAndAppend(c); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Append(string? s) { if (s == null) { return; } int pos = _pos; if (s.Length == 1 && (uint)pos < (uint)_chars.Length) // very common case, e.g. appending strings from NumberFormatInfo like separators, percent symbols, etc. { _chars[pos] = s[0]; _pos = pos + 1; } else { AppendSlow(s); } } private void AppendSlow(string s) { int pos = _pos; if (pos > _chars.Length - s.Length) { Grow(s.Length); } s #if !NETCOREAPP .AsSpan() #endif .CopyTo(_chars.Slice(pos)); _pos += s.Length; } public void Append(char c, int count) { if (_pos > _chars.Length - count) { Grow(count); } Span<char> dst = _chars.Slice(_pos, count); for (int i = 0; i < dst.Length; i++) { dst[i] = c; } _pos += count; } public unsafe void Append(char* value, int length) { int pos = _pos; if (pos > _chars.Length - length) { Grow(length); } Span<char> dst = _chars.Slice(_pos, length); for (int i = 0; i < dst.Length; i++) { dst[i] = *value++; } _pos += length; } public void Append(scoped ReadOnlySpan<char> value) { int pos = _pos; if (pos > _chars.Length - value.Length) { Grow(value.Length); } value.CopyTo(_chars.Slice(_pos)); _pos += value.Length; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public Span<char> AppendSpan(int length) { int origPos = _pos; if (origPos > _chars.Length - length) { Grow(length); } _pos = origPos + length; return _chars.Slice(origPos, length); } [MethodImpl(MethodImplOptions.NoInlining)] private void GrowAndAppend(char c) { Grow(1); Append(c); } private const uint ArrayMaxLength = 0x7FFFFFC7; // same as Array.MaxLength /// <summary> /// Resize the internal buffer either by doubling current buffer size or /// by adding <paramref name="additionalCapacityBeyondPos"/> to /// <see cref="_pos"/> whichever is greater. /// </summary> /// <param name="additionalCapacityBeyondPos"> /// Number of chars requested beyond current position. /// </param> [MethodImpl(MethodImplOptions.NoInlining)] private void Grow(int additionalCapacityBeyondPos) { Debug.Assert(additionalCapacityBeyondPos > 0); Debug.Assert(_pos > _chars.Length - additionalCapacityBeyondPos, "Grow called incorrectly, no resize is needed."); // Increase to at least the required size (_pos + additionalCapacityBeyondPos), but try // to double the size if possible, bounding the doubling to not go beyond the max array length. int newCapacity = (int)Math.Max( (uint)(_pos + additionalCapacityBeyondPos), Math.Min((uint)_chars.Length * 2, ArrayMaxLength)); // Make sure to let Rent throw an exception if the caller has a bug and the desired capacity is negative. // This could also go negative if the actual required length wraps around. char[] poolArray = ArrayPool<char>.Shared.Rent(newCapacity); _chars.Slice(0, _pos).CopyTo(poolArray); char[]? toReturn = _arrayToReturnToPool; _chars = _arrayToReturnToPool = poolArray; if (toReturn != null) { ArrayPool<char>.Shared.Return(toReturn); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Dispose() { char[]? toReturn = _arrayToReturnToPool; this = default; // for safety, to avoid using pooled array if this instance is erroneously appended to again if (toReturn != null) { ArrayPool<char>.Shared.Return(toReturn); } } } }