| File: Documentation\LexicalScoring.cs | Web Access |
| Project: src\src\Aspire.Cli\Aspire.Cli.csproj (aspire) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System.Text.RegularExpressions; namespace Aspire.Cli.Documentation; /// <summary> /// Controls which characters are treated as part of a lexical word when scoring boundaries. /// </summary> internal enum LexicalWordCharacterMode { /// <summary> /// Treats only letters and digits as word characters. /// </summary> Alphanumeric, /// <summary> /// Treats letters, digits, underscores, and hyphens as word characters. /// </summary> IdentifierWithHyphen } /// <summary> /// Provides shared helpers for lexical tokenization and field scoring. /// </summary> internal static class LexicalScoring { /// <summary> /// Tokenizes text with the specified split expression. /// </summary> /// <param name="text">The input text.</param> /// <param name="tokenSplitRegex">The regular expression used to split the text into tokens.</param> /// <param name="minTokenLength">The minimum token length to keep.</param> /// <returns>The normalized unique tokens.</returns> public static string[] Tokenize(string text, Regex tokenSplitRegex, int minTokenLength) { if (string.IsNullOrWhiteSpace(text)) { return []; } return [ .. tokenSplitRegex.Split(text) .Where(token => token.Length >= minTokenLength) .Select(static token => token.ToLowerInvariant()) .Distinct() ]; } /// <summary> /// Scores how well a normalized field matches the specified query tokens. /// </summary> /// <param name="lowerText">The normalized field text.</param> /// <param name="queryTokens">The normalized query tokens.</param> /// <param name="maxOccurrenceBonus">The maximum number of repeated-match bonuses to apply.</param> /// <param name="baseMatchScore">The base score for a token match.</param> /// <param name="wordBoundaryBonus">The additional score for a word-boundary match.</param> /// <param name="multipleOccurrenceBonus">The score applied for each additional token occurrence.</param> /// <param name="wordCharacterMode">The word-boundary behavior to use.</param> /// <returns>The lexical relevance score for the field.</returns> public static float ScoreField( string lowerText, string[] queryTokens, int maxOccurrenceBonus = 3, float baseMatchScore = 1.0f, float wordBoundaryBonus = 0.5f, float multipleOccurrenceBonus = 0.25f, LexicalWordCharacterMode wordCharacterMode = LexicalWordCharacterMode.Alphanumeric) { if (string.IsNullOrEmpty(lowerText)) { return 0; } var score = 0.0f; var textSpan = lowerText.AsSpan(); // Cap on how many occurrences we ever count toward the multi-occurrence bonus. // The bonus saturates at maxOccurrenceBonus, so beyond that one extra occurrence // there is no scoring reason to keep walking. Content fields can run several KB // and ScoreField is invoked once per (doc × field × query token), so capping the // per-field scan compounds across the whole corpus. var occurrenceLimit = maxOccurrenceBonus + 1; foreach (var token in queryTokens) { // Single forward scan: walk match-by-match, remember the first match // for the word-boundary bonus, and stop as soon as we've counted enough // occurrences to saturate the multi-occurrence bonus. The previous version // did IndexOf to find first match and then a second full-scan count, // doubling the bytes scanned in fields that contained a match. var startIndex = 0; var firstMatchIndex = -1; var count = 0; while (startIndex < textSpan.Length && count < occurrenceLimit) { var nextIndex = textSpan[startIndex..].IndexOf(token, StringComparison.Ordinal); if (nextIndex < 0) { break; } var absoluteIndex = startIndex + nextIndex; if (firstMatchIndex < 0) { firstMatchIndex = absoluteIndex; } count++; startIndex = absoluteIndex + token.Length; } if (count == 0) { continue; } score += baseMatchScore; if (IsWordBoundaryMatch(textSpan, token, firstMatchIndex, wordCharacterMode)) { score += wordBoundaryBonus; } if (count > 1) { score += Math.Min(count - 1, maxOccurrenceBonus) * multipleOccurrenceBonus; } } return score; } /// <summary> /// Determines whether a match occurs on word boundaries. /// </summary> /// <param name="text">The containing text.</param> /// <param name="token">The matched token.</param> /// <param name="index">The match start index.</param> /// <param name="wordCharacterMode">The word-boundary behavior to use.</param> /// <returns><c>true</c> if the match occurs on word boundaries; otherwise, <c>false</c>.</returns> public static bool IsWordBoundaryMatch( ReadOnlySpan<char> text, string token, int index, LexicalWordCharacterMode wordCharacterMode = LexicalWordCharacterMode.Alphanumeric) { var startsAtBoundary = index == 0 || !IsWordCharacter(text[index - 1], wordCharacterMode); var endIndex = index + token.Length; var endsAtBoundary = endIndex >= text.Length || !IsWordCharacter(text[endIndex], wordCharacterMode); return startsAtBoundary && endsAtBoundary; } private static bool IsWordCharacter(char value, LexicalWordCharacterMode wordCharacterMode) => wordCharacterMode switch { LexicalWordCharacterMode.IdentifierWithHyphen => char.IsLetterOrDigit(value) || value is '_' or '-', _ => char.IsLetterOrDigit(value) }; }