| File: GroupWorkItems.cs | Web Access |
| Project: src\runtime\src\tasks\HelixTestTasks\HelixTestTasks.csproj (HelixTestTasks) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System; using System.Collections.Generic; using System.IO; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; namespace Microsoft.DotNet.HelixTestTasks; /// <summary> /// Groups work items into balanced batches using greedy bin-packing by file size. /// Items exceeding <see cref="LargeThreshold"/>, or whose file name (without /// extension) is listed in <see cref="SoloItems"/>, are placed into solo batches /// with negative batch IDs (-1, -2, …). Remaining items are distributed across /// <see cref="BatchSize"/> batches, always assigning the next item to the /// lightest batch. /// </summary> public class GroupWorkItems : Task { [Required] public ITaskItem[] Items { get; set; } = Array.Empty<ITaskItem>(); public int BatchSize { get; set; } = 10; public long LargeThreshold { get; set; } = 52428800L; // 50 MB /// <summary> /// File names (without extension) of work items that must each run in their own /// solo batch regardless of size. Used to isolate slow suites so a single slow /// suite does not inflate the runtime of a shared batch. /// </summary> public ITaskItem[] SoloItems { get; set; } = Array.Empty<ITaskItem>(); [Output] public ITaskItem[] GroupedItems { get; set; } = Array.Empty<ITaskItem>(); public override bool Execute() { if (BatchSize <= 0) BatchSize = 10; if (LargeThreshold <= 0) LargeThreshold = 52428800L; var itemsWithSize = new List<(ITaskItem item, long size)>(); foreach (var item in Items) { long size = 0; if (File.Exists(item.ItemSpec)) size = new FileInfo(item.ItemSpec).Length; itemsWithSize.Add((item, size)); } // Sort largest first for greedy bin-packing itemsWithSize.Sort((a, b) => b.size.CompareTo(a.size)); // Match on the work item name, tolerating either a bare suite name or an archive path on // both sides. Only the ".zip" work item extension is stripped: suite names themselves // contain dots (e.g. "System.Text.Json.Tests"), so Path.GetFileNameWithoutExtension must // not be used as it would drop the trailing ".Tests" segment from a bare name. var soloNames = new HashSet<string>(StringComparer.Ordinal); foreach (var solo in SoloItems) soloNames.Add(NormalizeWorkItemName(solo.ItemSpec)); var result = new List<ITaskItem>(); int negativeBatchId = -1; // Separate items that must run alone (too large, or explicitly listed as solo) var smallItems = new List<(ITaskItem item, long size)>(); foreach (var entry in itemsWithSize) { if (entry.size > LargeThreshold || soloNames.Contains(NormalizeWorkItemName(entry.item.ItemSpec))) { var newItem = new TaskItem(entry.item); newItem.SetMetadata("BatchId", negativeBatchId.ToString()); negativeBatchId--; result.Add(newItem); } else { smallItems.Add(entry); } } // Greedy bin-packing for small items if (smallItems.Count > 0) { int numBatches = Math.Min(BatchSize, smallItems.Count); var batchSizes = new long[numBatches]; var batchAssignments = new List<ITaskItem>[numBatches]; for (int i = 0; i < numBatches; i++) batchAssignments[i] = new List<ITaskItem>(); foreach (var entry in smallItems) { // Find batch with smallest total size int minIdx = 0; for (int i = 1; i < numBatches; i++) { if (batchSizes[i] < batchSizes[minIdx]) minIdx = i; } batchSizes[minIdx] += entry.size; var newItem = new TaskItem(entry.item); newItem.SetMetadata("BatchId", minIdx.ToString()); batchAssignments[minIdx].Add(newItem); } for (int i = 0; i < numBatches; i++) result.AddRange(batchAssignments[i]); } GroupedItems = result.ToArray(); return !Log.HasLoggedErrors; static string NormalizeWorkItemName(string itemSpec) { string name = Path.GetFileName(itemSpec); return name.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) ? name.Substring(0, name.Length - 4) : name; } } }