File: ConfigRunner.cs
Web Access
Project: src\src\Microsoft.ML.Sweeper\Microsoft.ML.Sweeper.csproj (Microsoft.ML.Sweeper)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
 
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.ML;
using Microsoft.ML.CommandLine;
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.Runtime;
using Microsoft.ML.Sweeper;
using ResultProcessorInternal = Microsoft.ML.ResultProcessor;
 
[assembly: LoadableClass(typeof(LocalExeConfigRunner), typeof(LocalExeConfigRunner.Options), typeof(SignatureConfigRunner),
    "Local Sweep Config Runner", "Local")]
 
namespace Microsoft.ML.Sweeper
{
    public delegate void SignatureConfigRunner();
 
    public interface IConfigRunner
    {
        IEnumerable<IRunResult> RunConfigs(ParameterSet[] sweeps, int min);
        void Finish();
        string GetOutputFolderPath(string folderName);
    }
 
    public abstract class ExeConfigRunnerBase : IConfigRunner
    {
        public abstract class OptionsBase
        {
            [Argument(ArgumentType.AtMostOnce, HelpText = "Command pattern for the sweeps", ShortName = "pattern")]
            public string OptionsPattern;
 
            [Argument(ArgumentType.AtMostOnce, HelpText = "output folder for the outputs of the sweeps", ShortName = "outfolder")]
            public string OutputFolderName;
 
            [Argument(ArgumentType.AtMostOnce, HelpText = "prefix to add to the output file names", ShortName = "pre")]
            public string Prefix;
 
            [Argument(ArgumentType.AtMostOnce, HelpText = "The executable name, including the path (the default is MAML.exe)")]
            public string Exe;
 
            [Argument(ArgumentType.Multiple, HelpText = "Specify how to extract the metrics from the result file.", ShortName = "ev", SignatureType = typeof(SignatureSweepResultEvaluator))]
            public IComponentFactory<ISweepResultEvaluator<string>> ResultProcessor = ComponentFactoryUtils.CreateFromFunction(
                env => new InternalSweepResultEvaluator(env, new InternalSweepResultEvaluator.Options()));
 
            [Argument(ArgumentType.AtMostOnce, Hide = true)]
            public bool CalledFromUnitTestSuite;
        }
 
        protected string Exe;
        protected readonly string OptionsPattern;
        protected readonly string OutputFolder;
        protected readonly string Prefix;
        protected readonly ISweepResultEvaluator<string> ResultProcessor;
        protected readonly List<int> RunNums;
 
        protected readonly IHost Host;
 
        private readonly bool _calledFromUnitTestSuite;
 
        protected ExeConfigRunnerBase(OptionsBase options, IHostEnvironment env, string registrationName)
        {
            Contracts.AssertValue(env);
            Host = env.Register(registrationName);
            Host.CheckUserArg(!string.IsNullOrEmpty(options.OptionsPattern), nameof(options.OptionsPattern), "The command pattern is missing");
            Host.CheckUserArg(!string.IsNullOrEmpty(options.OutputFolderName), nameof(options.OutputFolderName), "Please specify an output folder");
            OptionsPattern = options.OptionsPattern;
            OutputFolder = GetOutputFolderPath(options.OutputFolderName);
            Prefix = string.IsNullOrEmpty(options.Prefix) ? "" : options.Prefix;
            ResultProcessor = options.ResultProcessor.CreateComponent(Host);
            _calledFromUnitTestSuite = options.CalledFromUnitTestSuite;
            RunNums = new List<int>();
        }
 
        protected virtual void ProcessFullExePath(string exe)
        {
            Exe = GetFullExePath(exe);
 
            if (!File.Exists(Exe) && !File.Exists(Exe + ".exe"))
                throw Host.ExceptUserArg(nameof(OptionsBase.Exe), "Executable {0} not found", Exe);
        }
 
        protected virtual string GetFullExePath(string exe)
        {
            if (!string.IsNullOrWhiteSpace(exe))
                return exe;
#if CORECLR
            if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows))
                return Path.Combine(SweepCommand.LocalExePath, "../Win/maml.exe");
            //REVIEW: Need mac support
            return Path.Combine(SweepCommand.LocalExePath, "../Linux/maml");
#else
            return Path.Combine(SweepCommand.LocalExePath, "maml.exe");
#endif
        }
 
        public virtual void Finish()
        {
            if (Exe == null || Exe.EndsWith("maml", StringComparison.OrdinalIgnoreCase) ||
                Exe.EndsWith("maml.exe", StringComparison.OrdinalIgnoreCase))
            {
                string currentDirectory = Path.GetDirectoryName(typeof(ExeConfigRunnerBase).Module.FullyQualifiedName);
 
                using (var ch = Host.Start("Finish"))
#pragma warning disable CS0618 // As this deals with invoking command lines, this may be OK, though this code has some other problems.
                using (AssemblyLoadingUtils.CreateAssemblyRegistrar(Host, currentDirectory))
#pragma warning restore CS0618
                {
                    var runs = RunNums.ToArray();
                    var args = Utils.BuildArray(RunNums.Count + 2,
                        i =>
                        {
                            if (i == RunNums.Count)
                                return string.Format(@"o={{{0}\{1}.summary.txt}}", OutputFolder, Prefix);
                            if (i == RunNums.Count + 1)
                                return string.Format("calledFromUnitTestSuite{0}", _calledFromUnitTestSuite ? "+" : "-");
                            return string.Format("{{{0}}}", GetFilePath(runs[i], "out"));
                        });
 
                    ResultProcessorInternal.ResultProcessor.Main(args);
 
                    ch.Info(@"The summary of the run results has been saved to the file {0}\{1}.summary.txt", OutputFolder, Prefix);
                }
            }
        }
 
        public virtual string GetOutputFolderPath(string folderName)
        {
            var folderPath = Path.GetFullPath(folderName);
 
            try
            {
                if (!Directory.Exists(folderName))
                    Directory.CreateDirectory(folderName);
                return folderPath;
            }
            catch (Exception e)
            {
                throw Host.Except(e, e.Message);
            }
        }
 
        // REVIEW: in case we want to use sweep command on linux we need to reconsider our syntax.
        // $something get treated in bash as variable something and if you have command line which looks like:
        // lr=$LR$
        // you get lr=$ only as argument because $LR is variable and empty.
        protected string GetCommandLine(ParameterSet sweep)
        {
            var options = OptionsPattern;
            foreach (var parameterValue in sweep)
                options = options.Replace("$" + parameterValue.Name + "$", parameterValue.ValueText);
            return options;
        }
 
        public IEnumerable<IRunResult> RunConfigs(ParameterSet[] sweeps, int min)
        {
            RunNums.AddRange(Enumerable.Range(min, sweeps.Length));
 
            using (var ch = Host.Start("Evaluate"))
            {
                for (int i = 0; i < sweeps.Length; i++)
                    ch.Info("Parameter set: {0}", string.Join(", ", sweeps[i].Select(p => string.Format("{0}:{1}", p.Name, p.ValueText))));
 
                return RunConfigsCore(sweeps, ch, min);
            }
        }
 
        protected string GetFilePath(int i, string kind)
        {
            return string.Format(@"{0}\{1}{2}.{3}.txt", OutputFolder, Prefix, i, kind);
        }
 
        protected abstract IEnumerable<IRunResult> RunConfigsCore(ParameterSet[] sweeps, IChannel ch, int min);
    }
 
    public sealed class LocalExeConfigRunner : ExeConfigRunnerBase
    {
        public sealed class Options : OptionsBase
        {
            [Argument(ArgumentType.AtMostOnce, HelpText = "The number of threads to use for the sweep (default auto determined by the number of cores)", ShortName = "t")]
            public int? NumThreads;
        }
 
        private readonly ParallelOptions _parallelOptions;
 
        public LocalExeConfigRunner(IHostEnvironment env, Options options)
            : base(options, env, "LocalExeSweepEvaluator")
        {
            Contracts.CheckParam(options.NumThreads == null || options.NumThreads.Value > 0, nameof(options.NumThreads), "Cannot be 0 or negative");
            _parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = options.NumThreads ?? -1 };
            Contracts.AssertNonEmpty(options.OutputFolderName);
            ProcessFullExePath(options.Exe);
        }
 
        protected override IEnumerable<IRunResult> RunConfigsCore(ParameterSet[] sweeps, IChannel ch, int min)
        {
            Parallel.For(0, sweeps.Length, _parallelOptions, j =>
            {
                var outFile = GetFilePath(min + j, "out");
                var errorFile = GetFilePath(min + j, "err");
                var arguments = GetCommandLine(sweeps[j]);
                RunProcess(Exe, new string[] { arguments }, Environment.CurrentDirectory,
                    new StreamWriter(outFile),
                    new StreamWriter(errorFile));
 
                if (File.Exists(errorFile) && new FileInfo(errorFile).Length == 0)
                {
                    File.Delete(errorFile);
                }
            });
            return sweeps.Select((sweep, j) =>
                ResultProcessor.GetRunResult(sweep, string.Format(@"{0}\{1}.out.txt", OutputFolder, min + j)));
        }
 
        /// <summary>
        /// Run specified EXE with given arguments
        /// </summary>
        private void RunProcess(string exeFilename, string[] args, string workingDir,
            TextWriter standardOutputWriter = null, TextWriter standardErrorWriter = null)
        {
            var p = new System.Diagnostics.Process
            {
                StartInfo =
                {
                    UseShellExecute = false,
                    CreateNoWindow = true,
                    FileName = exeFilename,
                    Arguments = (args == null ? "" : string.Join(" ", args)),
                }
            };
 
            if (workingDir != null)
                p.StartInfo.WorkingDirectory = workingDir;
 
            if (standardOutputWriter != null)
            {
                p.StartInfo.RedirectStandardOutput = true;
                p.OutputDataReceived += (s, a) => { if (a.Data != null) standardOutputWriter.WriteLine(a.Data); };
            }
 
            if (standardErrorWriter != null)
            {
                p.StartInfo.RedirectStandardError = true;
                p.ErrorDataReceived += (s, a) => { if (a.Data != null) standardErrorWriter.WriteLine(a.Data); };
            }
 
            p.Start();
            //p.EnableRaisingEvents = true; // REVIEW: Why would you claim you wanted to
            // use the async exit handler, only to just use WaitForExit downstream?
            if (standardOutputWriter != null)
                p.BeginOutputReadLine();
            if (standardErrorWriter != null)
                p.BeginErrorReadLine();
            p.WaitForExit();
 
            if (standardOutputWriter != null)
            {
                standardOutputWriter.Flush();
                standardOutputWriter.Close();
            }
 
            if (standardErrorWriter != null)
            {
                standardErrorWriter.Flush();
                standardErrorWriter.Close();
            }
        }
    }
}