Helpers

Model\BPETokenizer.cs (1)

824await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) :

Model\CodeGenTokenizer.cs (8)

459int encodedLength = Helpers.EncodeToUtf8AndTransform(textSpan, token, mapping); 1142int encodedLength = Helpers.EncodeToUtf8AndTransform(textSpan, token, mapping); 1206int encodedLength = Helpers.EncodeToUtf8AndTransform(textSpan, token, mapping); 1291Helpers.ArrayPoolGrow(ref bytes, (bytes.Length + bytesCountToEncode) * 2); 1295bytesIndex += Helpers.GetUtf8Bytes(removePrefixSpace ? specialToken.AsSpan().Slice(1) : specialToken.AsSpan(), bytes.AsSpan().Slice(bytesIndex)); 1480if (!Helpers.ConvertUtf8ToUtf16(current, vocabBuffer.Slice(utf16CharsInBuffer), out int utf8BytesConsumed, out int utf16CharsWritten)) 1573Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2); 1581i += Helpers.EncodeCodePointToUtf8(text, i, ref bytes, ref bytesIndex) - 1;

Model\SentencePieceTokenizer.cs (10)

401int bytesWritten = Helpers.GetUtf8Bytes(text.Slice(i), utf8Bytes); 772int bytesWritten = Helpers.GetUtf8Bytes(text.Slice(i), utf8Bytes); 1091int encodedCount = Helpers.GetUtf8Bytes(text.Slice(i), utf8Bytes); 1471int encodedCount = Helpers.GetUtf8Bytes(text.Slice(0, i + 1), utf8Bytes); 1623Helpers.ArrayPoolGrow(ref bytesPoolArray, bytesCount * 2); 1687Helpers.ArrayPoolGrow(ref charPoolArray, len); 1690int charCount = Helpers.GetChars(bytesPoolArray.AsSpan(0, bytesCount), charPoolArray); 1906Helpers.ArrayPoolGrow(ref bytesPoolArray, bytesCount * 2); 2029Helpers.ArrayPoolGrow(ref charPoolArray, len); 2032int charCount = Helpers.GetChars(bytesPoolArray.AsSpan(0, bytesCount), charPoolArray);

Model\TiktokenTokenizer.cs (15)

168string? line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine(); 174if (!Helpers.TryParseInt32(line, capacity.Length, out suggestedCapacity)) 179line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine(); 189line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine(); 203} while ((line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine()) is not null); 216if (Helpers.TryParseInt32(line, spaceIndex + 1, out int rank)) 218AddData(Helpers.FromBase64String(line, 0, spaceIndex), rank); 226await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : 327int encodedLength = Helpers.EncodeToUtf8(text, arrayPoolArray, indexMappingSpan); 444int encodedLength = Helpers.EncodeToUtf8(text, arrayPoolArray, indexMappingSpan); 609int encodedLength = Helpers.EncodeToUtf8(text, arrayPoolArray, indexMappingSpan); 735int encodedLength = Helpers.EncodeToUtf8(text, arrayPoolArray, indexMappingSpan); 829return Helpers.GetString(utf8Bytes.Slice(0, utf8ByteCount)); 912if (!Helpers.ConvertUtf8ToUtf16(tokenBytes.Span, tempBuffer, out int bytesConsumed, out incompleteCharsWritten)) 945if (!Helpers.ConvertUtf8ToUtf16(utf8Bytes.Slice(utf8BytesIncompleteIndex, utf8BytesIncompleteCount + tokenBytes.Length), tempBuffer, out int bytesConsumed, out int charsConsumed))

Model\WordPieceTokenizer.cs (2)

103string? line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine(); 115line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine();

Normalizer\BertNormalizer.cs (2)

154Helpers.ArrayPoolGrow(ref buffer, index + 40); 165Helpers.ArrayPoolGrow(ref buffer, index + buffer.Length + 10);

Utils\Helpers.netcoreapp.cs (1)

83Helpers.ArrayPoolGrow(ref destination, destination.Length * 2);

Utils\StringSpanOrdinalKey.cs (1)

46public override int GetHashCode() => Helpers.GetHashCode(Span);

Utils\ValueStringBuilder.cs (1)

96Helpers.Replace(span, oldValue, newValue);