File: GroupWorkItems.cs
Web Access
Project: src\runtime\src\tasks\HelixTestTasks\HelixTestTasks.csproj (HelixTestTasks)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.IO;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;

namespace Microsoft.DotNet.HelixTestTasks;

/// <summary>
/// Groups work items into balanced batches using greedy bin-packing by file size.
/// Items exceeding <see cref="LargeThreshold"/>, or whose file name (without
/// extension) is listed in <see cref="SoloItems"/>, are placed into solo batches
/// with negative batch IDs (-1, -2, …). Remaining items are distributed across
/// <see cref="BatchSize"/> batches, always assigning the next item to the
/// lightest batch.
/// </summary>
public class GroupWorkItems : Task
{
    [Required]
    public ITaskItem[] Items { get; set; } = Array.Empty<ITaskItem>();

    public int BatchSize { get; set; } = 10;

    public long LargeThreshold { get; set; } = 52428800L; // 50 MB

    /// <summary>
    /// File names (without extension) of work items that must each run in their own
    /// solo batch regardless of size. Used to isolate slow suites so a single slow
    /// suite does not inflate the runtime of a shared batch.
    /// </summary>
    public ITaskItem[] SoloItems { get; set; } = Array.Empty<ITaskItem>();

    [Output]
    public ITaskItem[] GroupedItems { get; set; } = Array.Empty<ITaskItem>();

    public override bool Execute()
    {
        if (BatchSize <= 0)
            BatchSize = 10;
        if (LargeThreshold <= 0)
            LargeThreshold = 52428800L;

        var itemsWithSize = new List<(ITaskItem item, long size)>();
        foreach (var item in Items)
        {
            long size = 0;
            if (File.Exists(item.ItemSpec))
                size = new FileInfo(item.ItemSpec).Length;

            itemsWithSize.Add((item, size));
        }

        // Sort largest first for greedy bin-packing
        itemsWithSize.Sort((a, b) => b.size.CompareTo(a.size));

        // Match on the work item name, tolerating either a bare suite name or an archive path on
        // both sides. Only the ".zip" work item extension is stripped: suite names themselves
        // contain dots (e.g. "System.Text.Json.Tests"), so Path.GetFileNameWithoutExtension must
        // not be used as it would drop the trailing ".Tests" segment from a bare name.
        var soloNames = new HashSet<string>(StringComparer.Ordinal);
        foreach (var solo in SoloItems)
            soloNames.Add(NormalizeWorkItemName(solo.ItemSpec));

        var result = new List<ITaskItem>();
        int negativeBatchId = -1;

        // Separate items that must run alone (too large, or explicitly listed as solo)
        var smallItems = new List<(ITaskItem item, long size)>();
        foreach (var entry in itemsWithSize)
        {
            if (entry.size > LargeThreshold || soloNames.Contains(NormalizeWorkItemName(entry.item.ItemSpec)))
            {
                var newItem = new TaskItem(entry.item);
                newItem.SetMetadata("BatchId", negativeBatchId.ToString());
                negativeBatchId--;
                result.Add(newItem);
            }
            else
            {
                smallItems.Add(entry);
            }
        }

        // Greedy bin-packing for small items
        if (smallItems.Count > 0)
        {
            int numBatches = Math.Min(BatchSize, smallItems.Count);
            var batchSizes = new long[numBatches];
            var batchAssignments = new List<ITaskItem>[numBatches];
            for (int i = 0; i < numBatches; i++)
                batchAssignments[i] = new List<ITaskItem>();

            foreach (var entry in smallItems)
            {
                // Find batch with smallest total size
                int minIdx = 0;
                for (int i = 1; i < numBatches; i++)
                {
                    if (batchSizes[i] < batchSizes[minIdx])
                        minIdx = i;
                }
                batchSizes[minIdx] += entry.size;
                var newItem = new TaskItem(entry.item);
                newItem.SetMetadata("BatchId", minIdx.ToString());
                batchAssignments[minIdx].Add(newItem);
            }

            for (int i = 0; i < numBatches; i++)
                result.AddRange(batchAssignments[i]);
        }

        GroupedItems = result.ToArray();
        return !Log.HasLoggedErrors;

        static string NormalizeWorkItemName(string itemSpec)
        {
            string name = Path.GetFileName(itemSpec);
            return name.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)
                ? name.Substring(0, name.Length - 4)
                : name;
        }
    }
}