Helpers

Model\BPETokenizer.cs (8)

851Helpers.AppendToBytesArray(token.AsSpan(), ref bytes, ref bytesIndex); 868Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex); 872string decodedString = Helpers.GetString(bytes.AsSpan(0, bytesIndex)); 1057if (!Helpers.ConvertUtf8ToUtf16(current, vocabBuffer.Slice(utf16CharsInBuffer), out int utf8BytesConsumed, out int utf16CharsWritten)) 1150await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : 1338int encodedLength = Helpers.EncodeToUtf8AndTransform(text, token, mapping); 1451int encodedLength = Helpers.EncodeToUtf8AndTransform(text, token, mapping); 1531int encodedLength = Helpers.EncodeToUtf8AndTransform(text, token, mapping);

Model\CodeGenTokenizer.cs (10)

462int encodedLength = Helpers.EncodeToUtf8AndTransform(textSpan, token, mapping); 1145int encodedLength = Helpers.EncodeToUtf8AndTransform(textSpan, token, mapping); 1209int encodedLength = Helpers.EncodeToUtf8AndTransform(textSpan, token, mapping); 1266Helpers.AppendToBytesArray(BeginningOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex); 1275Helpers.AppendToBytesArray(EndOfSentenceToken!.AsSpan(), ref bytes, ref bytesIndex); 1284Helpers.AppendToBytesArray(UnknownToken!.AsSpan(), ref bytes, ref bytesIndex); 1294Helpers.ArrayPoolGrow(ref bytes, (bytes.Length + bytesCountToEncode) * 2); 1298bytesIndex += Helpers.GetUtf8Bytes(removePrefixSpace ? specialToken.AsSpan().Slice(1) : specialToken.AsSpan(), bytes.AsSpan().Slice(bytesIndex)); 1309Helpers.AppendToBytesArray(span, ref bytes, ref bytesIndex); 1483if (!Helpers.ConvertUtf8ToUtf16(current, vocabBuffer.Slice(utf16CharsInBuffer), out int utf8BytesConsumed, out int utf16CharsWritten))

Model\SentencePieceBaseModel.cs (9)

270Helpers.ArrayPoolGrow(ref bytesPoolArray, bytesCount * 2); 330Helpers.ArrayPoolGrow(ref charPoolArray, len); 333int charCount = Helpers.GetChars(bytesPoolArray.AsSpan(0, bytesCount), charPoolArray); 543Helpers.ArrayPoolGrow(ref bytesPoolArray, bytesCount * 2); 668Helpers.ArrayPoolGrow(ref charPoolArray, len); 671int charCount = Helpers.GetChars(bytesPoolArray.AsSpan(0, bytesCount), charPoolArray); 729Helpers.Replace(tokenSpan, buffer, prefixSuffixChar, ' '); 756Helpers.Replace(tokenSpan, buffer, prefixSuffixChar, ' '); 778Helpers.Replace(tokenSpan, buffer, prefixSuffixChar, ' ');

Model\SentencePieceBpeModel.cs (4)

236int bytesWritten = Helpers.GetUtf8Bytes(text.Slice(i), utf8Bytes); 536int bytesWritten = Helpers.GetUtf8Bytes(text.Slice(i), utf8Bytes); 799int encodedCount = Helpers.GetUtf8Bytes(text.Slice(i), utf8Bytes); 1075int encodedCount = Helpers.GetUtf8Bytes(text.Slice(0, i + 1), utf8Bytes);

Model\SentencePieceUnigramModel.cs (14)

437int expectedCount = Helpers.GetUtf16LengthFromUtf8Bytes(utf8Bytes); 447bool res = Helpers.ConvertUtf8ToUtf16(utf8Bytes, normalizedString.AsSpan(normalizedString.Length - normalizedStringCountFromEnd - expectedCount), out int bytesConsumed, out int charsWritten); 460Helpers.ArrayPoolGrow(ref normalizedString, normalizedString.Length << 1); 473Helpers.ArrayPoolGrow(ref normalizedString, normalizedString.Length << 1); 477bool res = Helpers.ConvertUtf8ToUtf16(normalizationSpan, normalizedUtf16Span, out int bytesConsumed, out int charsWritten); 574int byteCount = Helpers.GetUtf8Bytes(text, byteSpan); 637string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 703int codepointLength = Helpers.EncodeNextUtf8(normalizationSpan.Slice(offsetStart), destination); 770int mbLen = Helpers.OneCharLen(normalized[startsAt]); 1098Helpers.ArrayPoolGrow(ref unknownTokensTracking, unknownTokensCount << 1); 1185charsConsumed += Helpers.GetUtf16LengthFromUtf8Bytes(utf8UnknownBytes); 1443charsConsumed += Helpers.GetUtf16LengthFromUtf8Bytes(normalizationSpan); 1480charsConsumed += Helpers.GetUtf16LengthFromUtf8Bytes(normalizationSpan.Slice(ids[i].UtfStartOffset, ids[i].Utf8Length)); 1701consumedCharacters += Helpers.GetUtf16LengthFromUtf8Bytes(normalizationSpan.Slice(node.StartsAt, length));

Model\TiktokenTokenizer.cs (15)

168string? line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine(); 174if (!Helpers.TryParseInt32(line, capacity.Length, out suggestedCapacity)) 179line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine(); 189line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine(); 203} while ((line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine()) is not null); 216if (Helpers.TryParseInt32(line, spaceIndex + 1, out int rank)) 218AddData(Helpers.FromBase64String(line, 0, spaceIndex), rank); 226await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : 327int encodedLength = Helpers.EncodeToUtf8(text, arrayPoolArray, indexMappingSpan); 444int encodedLength = Helpers.EncodeToUtf8(text, arrayPoolArray, indexMappingSpan); 609int encodedLength = Helpers.EncodeToUtf8(text, arrayPoolArray, indexMappingSpan); 735int encodedLength = Helpers.EncodeToUtf8(text, arrayPoolArray, indexMappingSpan); 829return Helpers.GetString(utf8Bytes.Slice(0, utf8ByteCount)); 912if (!Helpers.ConvertUtf8ToUtf16(tokenBytes.Span, tempBuffer, out int bytesConsumed, out incompleteCharsWritten)) 945if (!Helpers.ConvertUtf8ToUtf16(utf8Bytes.Slice(utf8BytesIncompleteIndex, utf8BytesIncompleteCount + tokenBytes.Length), tempBuffer, out int bytesConsumed, out int charsConsumed))

Model\WordPieceTokenizer.cs (2)

103string? line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine(); 115line = useAsync ? await Helpers.ReadLineAsync(reader, cancellationToken).ConfigureAwait(false) : reader.ReadLine();

Normalizer\BertNormalizer.cs (2)

154Helpers.ArrayPoolGrow(ref buffer, index + 40); 165Helpers.ArrayPoolGrow(ref buffer, index + buffer.Length + 10);

Normalizer\SentencePieceNormalizer.cs (10)

121int inputLength = Helpers.GetUtf8Bytes(text.AsSpan(), inputArray); 352if (!Helpers.IsValidDecodeUtf8(input, out int length)) 395string chained = NormalizationChain.Normalize(Helpers.GetString(input)); 404int chainedLength = Helpers.GetUtf8Bytes(chained.AsSpan(), chainedBytes); 486Helpers.ArrayPoolGrow(ref normalized, ref poolArray, (normalizedIndex + _spaceSymbol.Length) << 1); 498Helpers.ArrayPoolGrow(ref normalized, ref poolArray, (normalizedIndex + 1) << 1); 550Helpers.ArrayPoolGrow(ref normalized, ref poolArray, (normalizedIndex + _spaceSymbol.Length) << 1); 559Helpers.ArrayPoolGrow(ref normalized, ref poolArray, (normalizedIndex + 1) << 1); 583trieBlobSize = Helpers.Swap32(trieBlobSize); 602data[i] = Helpers.Swap32(data[i]);

Utils\DoubleArrayTrie.cs (1)

758encodingLength = Helpers.GetUtf8Bytes(pair.Key.AsSpan(), bytes);

Utils\Helpers.cs (1)

223Helpers.ArrayPoolGrow<byte>(ref bytes, bytes.Length * 2);

Utils\Helpers.netcoreapp.cs (1)

89Helpers.ArrayPoolGrow(ref destination, destination.Length * 2);

Utils\OrdinalUtf8StringComparer.cs (2)

73xLen = Helpers.GetUtf8Bytes(x.AsSpan(i), buffer1); 74yLen = Helpers.GetUtf8Bytes(y.AsSpan(i), buffer2);

Utils\StringSpanOrdinalKey.cs (1)

46public override int GetHashCode() => Helpers.GetHashCode(Span);

Utils\ValueStringBuilder.cs (1)

96Helpers.Replace(span, oldValue, newValue);