Length

Model\BPETokenizer.cs (8)

466EncodeWithCache(textSpanToEncode.Slice(split.Offset, split.Length), tokens, split.Offset, ref priorityQueue); 527EncodeToIdsWithCache(textSpanToEncode.Slice(split.Offset, split.Length), ids, maxTokenCount - ids.Count, out int length, ref priorityQueue); 530if (length < split.Length || ids.Count >= maxTokenCount) 588count += EncodeToIdsWithCache(textSpanToEncode.Slice(split.Offset, split.Length), null, maxTokenCount - count, out int length, ref priorityQueue); 591if (length < split.Length || count >= maxTokenCount) 668count += EncodeToIdsWithCache(textSpanToEncode.Slice(split.Offset, split.Length), null, maxTokenCount - count, out int length, ref priorityQueue); 671if (length < split.Length || count >= maxTokenCount) 722tokenCount += EncodeToIdsFromEndWithCache(textSpanToEncode.Slice(split.Offset, split.Length), null, maxTokenCount - tokenCount, out int textIndex, ref priorityQueue);

Model\CodeGenTokenizer.cs (19)

391EncodeInternal(null, textSpanToEncode.Slice(split.Offset, split.Length), tokens, addPrefixSpace, split.Offset, agenda); 632EncodeToIdsInternal(null, textSpanToEncode.Slice(split.Offset, split.Length), ids, agenda, out int length, maxTokenCount - ids.Count); 635if (length < split.Length || ids.Count >= maxTokenCount) 844count += EncodeToIdsInternal(null, textSpanToEncode.Slice(split.Offset, split.Length), null, agenda, out int length, maxTokenCount - count); 847if (length < split.Length || count >= maxTokenCount) 983tokenCount += EncodeToIdsFromEndInternal(null, textSpanToEncode.Slice(split.Offset, split.Length), null, agenda, out int textIndex, maxTokenCount - tokenCount); 1636if (symbols[top.Left].pieceSpan.Length == 0 || symbols[top.Right].pieceSpan.Length == 0 || 1637symbols[top.Left].pieceSpan.Length + symbols[top.Right].pieceSpan.Length != top.Length) 1643symbols[top.Left].pieceSpan = (symbols[top.Left].pieceSpan.Index, symbols[top.Left].pieceSpan.Length + symbols[top.Right].pieceSpan.Length); 1663if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, string Token) value)) 1665result.Add(GetToken(value.Id, value.Token, symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length, originalText, mapping)); 1669result.Add(GetToken(UnknownTokenId.Value, UnknownToken!, symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length, originalText, mapping)); 1693if (!_mergeRanks.TryGetValue(textSpan.Slice(symbols[left].pieceSpan.Index, symbols[left].pieceSpan.Length), textSpan.Slice(symbols[right].pieceSpan.Index, symbols[right].pieceSpan.Length), out int rank)) 1698SymbolPair pair = new(left, right, rank, symbols[left].pieceSpan.Length + symbols[right].pieceSpan.Length);

Model\EnglishRobertaTokenizer.cs (6)

337foreach (EncodedToken t in EncodeInternal(textSpanToEncode.Slice(split.Offset, split.Length))) 441EncodeToIdsInternal(textSpanToEncode.Slice(split.Offset, split.Length), ids, out int length, maxTokenCount - ids.Count); 444if (length < split.Length || ids.Count >= maxTokenCount) 525count += EncodeToIdsInternal(textSpanToEncode.Slice(split.Offset, split.Length), null, out int length, maxTokenCount - count); 528if (length < split.Length || count >= maxTokenCount) 572tokenCount += EncodeToIdsFromEndInternal(textSpanToEncode.Slice(split.Offset, split.Length), null, out int textIndex, maxTokenCount - tokenCount);

Model\SentencePieceBpeModel.cs (45)

Model\SentencePieceUnigramModel.cs (6)

1295if (current.Offset + current.Length < text.Length) 1297GetIndexByTokenCountFromEndInternal(text.Slice(current.Offset + current.Length), considerNormalization, ref tokenCount, buffer, ref normalizedString, ref normalizedStringCountFromEnd, ref charConsumedFromEnd, maxTokenCount); 1306if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id)) 1311charConsumedFromEnd += current.Length; 1316StoreNormalizedTextFromEnd(text.Slice(current.Offset, current.Length), ref normalizedString, ref normalizedStringCountFromEnd); 1321int start = i > 0 ? splits[i - 1].Offset + splits[i - 1].Length : 0;

Model\TiktokenTokenizer.cs (6)

282EncodeToTokens(textSpanToEncode.Slice(split.Offset, split.Length), tokens, split.Offset); 393EncodeToIds(textSpanToEncode.Slice(split.Offset, split.Length), ids, out int length, maxTokenCount - ids.Count); 396if (length < split.Length || ids.Count >= maxTokenCount) 560count += CountTokens(textSpanToEncode.Slice(split.Offset, split.Length), out int length, maxTokenCount - count); 563if (length < split.Length || count >= maxTokenCount) 687tokenCount += CountTokensFromEnd(textSpanToEncode.Slice(split.Offset, split.Length), out int textIndex, maxTokenCount - tokenCount);

Model\WordPieceTokenizer.cs (9)

296EncodeToTokens(textSpanToEncode.Slice(split.Offset, split.Length), tokens, split.Offset); 420EncodeToIds(textSpanToEncode.Slice(split.Offset, split.Length), ids, out int length, maxTokenCount - ids.Count); 422if (length < split.Length || ids.Count >= maxTokenCount) 570count += EncodeToIds(textSpanToEncode.Slice(split.Offset, split.Length), accumulatedIds: null, out int length, maxTokenCount - count); 572if (length < split.Length || count >= maxTokenCount) 649int count = EncodeToIds(textSpanToEncode.Slice(split.Offset, split.Length), accumulatedIds: null, out charsConsumed, settings.MaxTokenCount - tokenCount); 650if (charsConsumed != split.Length) 652return fromEnd ? split.Offset + split.Length : split.Offset; 659return fromEnd ? split.Offset : split.Offset + split.Length;

PreTokenizer\CompositePreTokenizer.cs (8)

88yield return (range.Offset, range.Length); 89beginning += range.Length; 116beginning = range.Offset + range.Length; 118yield return (offset + range.Offset, range.Length); 166yield return (range.Offset, range.Length); 167beginning += range.Length; 194beginning = range.Offset + range.Length; 196yield return (offset + range.Offset, range.Length);

PreTokenizer\PreTokenizer.cs (4)

38yield return (match.Offset, match.Length); 39beginning = match.Offset + match.Length; 146yield return (match.Offset, match.Length); 147beginning = match.Offset + match.Length;

PreTokenizer\RegexPreTokenizer.cs (12)

74yield return (match.Offset, match.Length); 75beginning = match.Offset + match.Length; 78yield return (specialMatch.Offset, specialMatch.Length); 79beginning = specialMatch.Offset + specialMatch.Length; 85yield return (match.Offset, match.Length); 86beginning = match.Length + match.Offset; 125yield return (match.Offset, match.Length); 126beginning = match.Offset + match.Length; 129yield return (specialMatch.Offset, specialMatch.Length); 130beginning = specialMatch.Offset + specialMatch.Length; 136yield return (match.Offset, match.Length); 137beginning = match.Length + match.Offset;