File: AutoMLExperimentTests.cs
Web Access
Project: src\test\Microsoft.ML.AutoML.Tests\Microsoft.ML.AutoML.Tests.csproj (Microsoft.ML.AutoML.Tests)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
 
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Data.Analysis;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.ML.AutoML.CodeGen;
using Microsoft.ML.Fairlearn.AutoML;
using Microsoft.ML.Runtime;
using Microsoft.ML.TestFramework;
using Microsoft.ML.TestFramework.Attributes;
using Xunit;
using Xunit.Abstractions;
 
namespace Microsoft.ML.AutoML.Test
{
    public class AutoMLExperimentTests : BaseTestClass
    {
        public AutoMLExperimentTests(ITestOutputHelper output) : base(output)
        {
        }
 
        [Fact]
        public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_and_no_trial_completed_Async()
        {
            var context = new MLContext(1);
            var experiment = context.Auto().CreateExperiment();
 
            experiment.SetTrainingTimeInSeconds(1)
                      .SetTrialRunner((serviceProvider) =>
                      {
                          var channel = serviceProvider.GetService<IChannel>();
                          var settings = serviceProvider.GetService<AutoMLExperiment.AutoMLExperimentSettings>();
                          return new DummyTrialRunner(settings, 5, channel);
                      })
                      .SetTuner<RandomSearchTuner>();
 
            var cts = new CancellationTokenSource();
 
            context.Log += (o, e) =>
            {
                if (e.RawMessage.Contains("Update Running Trial"))
                {
                    cts.Cancel();
                }
            };
 
            var runExperimentAction = async () => await experiment.RunAsync(cts.Token);
 
            await runExperimentAction.Should().ThrowExactlyAsync<TimeoutException>();
        }
 
        [Fact]
        public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async()
        {
            var context = new MLContext(1);
            var experiment = context.Auto().CreateExperiment();
            context.Log += (o, e) =>
            {
                if (e.Source.StartsWith("AutoMLExperiment"))
                {
                    this.Output.WriteLine(e.RawMessage);
                }
            };
            // the following experiment set memory usage limit to 0.01mb
            // so all trials should be canceled and there should be no successful trials.
            // therefore when experiment finishes, it should throw timeout exception with no model trained message.
            experiment.SetMaxModelToExplore(10)
                      .SetTrialRunner((serviceProvider) =>
                      {
                          var channel = serviceProvider.GetService<IChannel>();
                          var settings = serviceProvider.GetService<AutoMLExperiment.AutoMLExperimentSettings>();
                          return new DummyTrialRunner(settings, 5, channel);
                      })
                      .SetTuner<RandomSearchTuner>()
                      .SetMaximumMemoryUsageInMegaByte(0.01);
 
            var runExperimentAction = async () => await experiment.RunAsync();
            await runExperimentAction.Should().ThrowExactlyAsync<TimeoutException>();
        }
 
        [LightGBMFact]
        public async Task AutoMLExperiment_lgbm_cancel_trial_when_exceeds_memory_limit_Async()
        {
            // this test is to verify that lightGbm can be cancelled during training booster.
            var context = new MLContext(1);
            context.Log += (o, e) =>
            {
                if (e.Message.Contains("LightGBM objective"))
                {
                    context.CancelExecution();
                }
            };
            var data = DatasetUtil.GetUciAdultDataView();
            var experiment = context.Auto().CreateExperiment();
            var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel })
                                .Append(context.BinaryClassification.Trainers.LightGbm(DatasetUtil.UciAdultLabel, "_Features_", numberOfIterations: 10000));
 
            experiment.SetDataset(context.Data.TrainTestSplit(data))
                    .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel)
                    .SetPipeline(pipeline)
                    .SetTrainingTimeInSeconds(10)
                    .SetMaximumMemoryUsageInMegaByte(10);
 
            var runExperimentAction = async () => await experiment.RunAsync();
            await runExperimentAction.Should().ThrowExactlyAsync<TimeoutException>();
        }
 
        [Fact]
        public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled_with_trial_completed_Async()
        {
            var context = new MLContext(1);
            var stopWatch = new Stopwatch();
            stopWatch.Start();
            var experiment = context.Auto().CreateExperiment();
            experiment.SetTrainingTimeInSeconds(10)
                      .SetTrialRunner((serviceProvider) =>
                      {
                          var channel = serviceProvider.GetService<IChannel>();
                          var settings = serviceProvider.GetService<AutoMLExperiment.AutoMLExperimentSettings>();
                          return new DummyTrialRunner(settings, 1, channel);
                      })
                      .SetTuner<RandomSearchTuner>();
 
            var cts = new CancellationTokenSource();
 
            context.Log += (o, e) =>
            {
                if (e.RawMessage.Contains("Update Completed Trial"))
                {
                    cts.CancelAfter(100);
                }
            };
            var res = await experiment.RunAsync(cts.Token);
 
            stopWatch.Stop();
            stopWatch.ElapsedMilliseconds.Should().BeLessOrEqualTo(2 * 1000 + 500);
            cts.IsCancellationRequested.Should().BeTrue();
            res.Metric.Should().BeGreaterThan(0);
        }
 
        [Fact]
        public async Task AutoMLExperiment_finish_training_when_time_is_up_Async()
        {
            var context = new MLContext(1);
            context.Log += (o, e) =>
            {
                if (e.Source.StartsWith("AutoMLExperiment"))
                {
                    this.Output.WriteLine(e.RawMessage);
                }
            };
 
            var experiment = context.Auto().CreateExperiment();
            experiment.SetTrainingTimeInSeconds(5)
                      .SetTrialRunner((serviceProvider) =>
                      {
                          var channel = serviceProvider.GetService<IChannel>();
                          var settings = serviceProvider.GetService<AutoMLExperiment.AutoMLExperimentSettings>();
                          return new DummyTrialRunner(settings, 1, channel);
                      })
                      .SetTuner<RandomSearchTuner>();
 
            var cts = new CancellationTokenSource();
            cts.CancelAfter(100 * 1000);
 
            var res = await experiment.RunAsync(cts.Token);
            res.Metric.Should().BeGreaterThan(0);
            cts.IsCancellationRequested.Should().BeFalse();
            cts.Dispose();
        }
 
        [Fact]
        public async Task AutoMLExperiment_finish_training_when_reach_to_max_model_async()
        {
            var context = new MLContext(1);
            var experiment = context.Auto().CreateExperiment();
            experiment.SetMaxModelToExplore(5)
                      .SetTrialRunner((serviceProvider) =>
                      {
                          var channel = serviceProvider.GetService<IChannel>();
                          var settings = serviceProvider.GetService<AutoMLExperiment.AutoMLExperimentSettings>();
                          return new DummyTrialRunner(settings, 1, channel);
                      })
                      .SetTuner<RandomSearchTuner>();
 
            var runModelCounts = 0;
            context.Log += (o, e) =>
            {
                if (e.RawMessage.Contains("Update Completed Trial"))
                {
                    runModelCounts++;
                }
            };
            await experiment.RunAsync();
            runModelCounts.Should().Be(5);
        }
 
 
        [Fact]
        public async Task AutoMLExperiment_UCI_Adult_Train_Test_Split_Test()
        {
            var context = new MLContext(1);
            context.Log += (o, e) =>
            {
                if (e.Source.StartsWith("AutoMLExperiment"))
                {
                    this.Output.WriteLine(e.RawMessage);
                }
            };
            var data = DatasetUtil.GetUciAdultDataView();
            var experiment = context.Auto().CreateExperiment();
            var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel })
                                .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdcaLogisticRegression: false, useLbfgsLogisticRegression: false));
 
            experiment.SetDataset(context.Data.TrainTestSplit(data))
                    .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel)
                    .SetPipeline(pipeline)
                    .SetMaxModelToExplore(1);
 
            var result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.8);
        }
 
        [Fact(Skip = "skip in CI build")]
        public async Task AutoMLExperiment_UCI_Adult_Fairlearn_Test()
        {
            var context = new MLContext(1);
            context.Log += (o, e) =>
            {
                if (e.Source.StartsWith("AutoMLExperiment"))
                {
                    this.Output.WriteLine(e.RawMessage);
                }
            };
            var data = DatasetUtil.GetUciAdultDataView();
            var experiment = context.Auto().CreateExperiment();
            var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel })
                                .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", exampleWeightColumnName: "signedWeight", useLgbm: false, useSdcaLogisticRegression: false, useLbfgsLogisticRegression: false));
 
            experiment.SetDataset(context.Data.TrainTestSplit(data))
                    .SetPipeline(pipeline)
                    .SetBinaryClassificationMetricWithFairLearn(DatasetUtil.UciAdultLabel, "PredictedLabel", "Workclass", "signedWeight")
                    .SetMaxModelToExplore(100);
 
            var result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.8);
        }
 
        [Fact]
        public async Task AutoMLExperiment_UCI_Adult_CV_5_Test()
        {
            var context = new MLContext(1);
            context.Log += (o, e) =>
            {
                if (e.Source.StartsWith("AutoMLExperiment"))
                {
                    this.Output.WriteLine(e.RawMessage);
                }
            };
            var data = DatasetUtil.GetUciAdultDataView();
            var experiment = context.Auto().CreateExperiment();
            var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel })
                                .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdcaLogisticRegression: false, useLbfgsLogisticRegression: false));
 
            experiment.SetDataset(data, 5)
                    .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel)
                    .SetPipeline(pipeline)
                    .SetMaxModelToExplore(1);
 
            var result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.8);
        }
 
        [Fact]
        public async Task AutoMLExperiment_Iris_CV_5_Test()
        {
            var context = new MLContext(1);
            context.Log += (o, e) =>
            {
                if (e.RawMessage.Contains("Trial"))
                {
                    this.Output.WriteLine(e.RawMessage);
                }
            };
            var data = DatasetUtil.GetIrisDataView();
            var experiment = context.Auto().CreateExperiment();
            var label = "Label";
            var pipeline = context.Auto().Featurizer(data, excludeColumns: new[] { label })
                                .Append(context.Transforms.Conversion.MapValueToKey(label, label))
                                .Append(context.Auto().MultiClassification(label, useLgbm: false, useSdcaMaximumEntrophy: false, useLbfgsMaximumEntrophy: false));
 
            experiment.SetDataset(data, 5)
                    .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MacroAccuracy, label)
                    .SetPipeline(pipeline)
                    .SetMaxModelToExplore(1);
 
            var result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.8);
        }
 
        [Fact]
        public async Task AutoMLExperiment_Iris_Train_Test_Split_Test()
        {
            var context = new MLContext(1);
            context.Log += (o, e) =>
            {
                if (e.Source.StartsWith("AutoMLExperiment"))
                {
                    this.Output.WriteLine(e.RawMessage);
                }
            };
            var data = DatasetUtil.GetIrisDataView();
            var experiment = context.Auto().CreateExperiment();
            var label = "Label";
            var pipeline = context.Auto().Featurizer(data, excludeColumns: new[] { label })
                                .Append(context.Transforms.Conversion.MapValueToKey(label, label))
                                .Append(context.Auto().MultiClassification(label, useLgbm: false, useSdcaMaximumEntrophy: false, useLbfgsMaximumEntrophy: false));
 
            experiment.SetDataset(context.Data.TrainTestSplit(data))
                    .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MacroAccuracy, label)
                    .SetPipeline(pipeline)
                    .SetMaxModelToExplore(1);
 
            var result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.8);
        }
 
        [Fact]
        public async Task AutoMLExperiment_Taxi_Fare_Train_Test_Split_Test()
        {
            var context = new MLContext(1);
            context.Log += (o, e) =>
            {
                if (e.Source.StartsWith("AutoMLExperiment"))
                {
                    this.Output.WriteLine(e.RawMessage);
                }
            };
            var train = DatasetUtil.GetTaxiFareTrainDataView();
            var test = DatasetUtil.GetTaxiFareTestDataView();
            var experiment = context.Auto().CreateExperiment();
            var label = DatasetUtil.TaxiFareLabel;
            var pipeline = context.Auto().Featurizer(train, excludeColumns: new[] { label })
                                .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgsPoissonRegression: false));
 
            experiment.SetDataset(train, test)
                    .SetRegressionMetric(RegressionMetric.RSquared, label)
                    .SetPipeline(pipeline)
                    .SetMaxModelToExplore(1);
 
            var result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.5);
 
            // test subsamping
            experiment = context.Auto().CreateExperiment();
            experiment.SetDataset(train, test, true)
                    .SetRegressionMetric(RegressionMetric.RSquared, label)
                    .SetPipeline(pipeline)
                    .SetMaxModelToExplore(1);
            result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.5);
            result.TrialSettings.Parameter[nameof(TrainValidateDatasetManager)]["TrainValidateDatasetSubsamplingKey"]
                .AsType<float>().Should().Be(0.1f);
        }
 
        [Fact]
        public async Task AutoMLExperiment_Taxi_Fare_CV_5_Test()
        {
            var context = new MLContext(1);
            var train = DatasetUtil.GetTaxiFareTrainDataView();
            var experiment = context.Auto().CreateExperiment();
            var label = DatasetUtil.TaxiFareLabel;
            var pipeline = context.Auto().Featurizer(train, excludeColumns: new[] { label })
                                .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgsPoissonRegression: false));
 
            experiment.SetDataset(train, 5)
                    .SetRegressionMetric(RegressionMetric.RSquared, label)
                    .SetPipeline(pipeline)
                    .SetMaxModelToExplore(1);
 
            var result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.5);
        }
 
        [Fact]
        public async Task AutoMLExperiment_Taxi_Fare_CV_5_SamplingKey_Test()
        {
            var context = new MLContext(1);
            var train = DatasetUtil.GetTaxiFareTrainDataView();
            var experiment = context.Auto().CreateExperiment();
            var label = DatasetUtil.TaxiFareLabel;
            var pipeline = context.Auto().Featurizer(train, excludeColumns: new[] { label })
                                .Append(context.Auto().Regression(label, useLgbm: false, useSdca: false, useLbfgsPoissonRegression: false));
 
            experiment.SetDataset(train, 5, "vendor_id")
                    .SetRegressionMetric(RegressionMetric.RSquared, label)
                    .SetPipeline(pipeline)
                    .SetMaxModelToExplore(1);
 
            var result = await experiment.RunAsync();
            result.Metric.Should().BeGreaterThan(0.2);
            result.Metric.Should().BeLessThan(0.5);
        }
 
        [Fact]
        public void AutoMLExperiment_should_use_seed_from_context_if_provided()
        {
            var context = new MLContext();
            var experiment = context.Auto().CreateExperiment();
            var settings = experiment.ServiceCollection.BuildServiceProvider().GetRequiredService<AutoMLExperiment.AutoMLExperimentSettings>();
            settings.Seed.Should().BeNull();
 
            context = new MLContext(1);
            experiment = context.Auto().CreateExperiment();
            settings = experiment.ServiceCollection.BuildServiceProvider().GetRequiredService<AutoMLExperiment.AutoMLExperimentSettings>();
            settings.Seed.Should().Be(1);
        }
    }
 
    class DummyTrialRunner : ITrialRunner
    {
        private readonly int _finishAfterNSeconds;
        private readonly IChannel _logger;
 
        public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, int finishAfterNSeconds, IChannel logger)
        {
            _finishAfterNSeconds = finishAfterNSeconds;
            _logger = logger;
        }
 
        public void Dispose()
        {
        }
 
        public async Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
        {
            _logger.Info("Update Running Trial");
            ct.ThrowIfCancellationRequested();
            await Task.Delay(_finishAfterNSeconds * 1000, ct);
            ct.ThrowIfCancellationRequested();
            _logger.Info("Update Completed Trial");
            var metric = 1.000 + 0.01 * settings.TrialId;
            return new TrialResult
            {
                TrialSettings = settings,
                DurationInMilliseconds = _finishAfterNSeconds * 1000,
                Metric = metric,
                Loss = - -metric,
            };
        }
    }
 
    class DummyPeformanceMonitor : IPerformanceMonitor
    {
        private readonly int _checkIntervalInMilliseconds;
        private System.Timers.Timer _timer;
 
        public DummyPeformanceMonitor()
        {
            _checkIntervalInMilliseconds = 1000;
        }
 
        public event EventHandler<TrialPerformanceMetrics> PerformanceMetricsUpdated;
 
        public void Dispose()
        {
        }
 
        public double? GetPeakCpuUsage()
        {
            return 100;
        }
 
        public double? GetPeakMemoryUsageInMegaByte()
        {
            return 1000;
        }
 
        public void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource)
        {
        }
 
        public void Start()
        {
            if (_timer == null)
            {
                _timer = new System.Timers.Timer(_checkIntervalInMilliseconds);
                _timer.Elapsed += (o, e) =>
                {
                    PerformanceMetricsUpdated?.Invoke(this, new TrialPerformanceMetrics() { PeakCpuUsage = 100, PeakMemoryUsage = 1000 });
                };
 
                _timer.AutoReset = true;
            }
            _timer.Enabled = true;
        }
 
        public void Pause()
        {
            _timer.Enabled = false;
        }
 
        public void Stop()
        {
            _timer?.Stop();
            _timer?.Dispose();
            _timer = null;
        }
    }
}