|
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#nullable enable
using System;
using System.Collections.Frozen;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Azure.Core;
using Microsoft.DiaSymReader.Tools;
using Microsoft.SymbolStore;
namespace Microsoft.DotNet.Internal.SymbolHelper;
/// <summary>
/// Helper class for uploading symbols to a symbol server. This file assumes the logger to be thread safe.
/// All state within this is immutable after construction, and the class is thread safe. Multiple uploads
/// can be done in parallel with the same instance.
/// The usual workflow is to create a request, add files and packages to it as needed. Finally, the request
/// can be finalized with some TTL if all uploads. Otherwise, if assets fail to upload, the request can be
/// deleted.
/// There's a few options for the helper that can be controlled by the <see cref="SymbolPublisherOptions"/> passed in,
/// notably the ability to convert portable PDBs to Windows PDBs and the ability to generate a special manifest
/// for the official runtime builds.
/// </summary>
public sealed class SymbolUploadHelper
{
public const string ConversionFolderName = "_convertedPdbs";
private const string AzureDevOpsResource = "499b84ac-1321-427f-aa17-267ca6975798";
private const string PathEnvVarName = "AzureDevOpsToken";
private static readonly FrozenSet<string> s_validExtensions = FrozenSet.ToFrozenSet(["", ".exe", ".dll", ".pdb", ".so", ".dbg", ".dylib", ".dwarf", ".r2rmap"]);
private readonly ScopedTracerFactory _tracerFactory;
private readonly ScopedTracer _globalTracer;
private readonly string _workingDir;
private readonly TokenCredential _credential;
private readonly string _commonArgs;
private readonly string _symbolToolPath;
private readonly PdbConverter? _pdbConverter;
private readonly uint _symbolToolTimeoutInMins;
private readonly bool _shouldGenerateManifest;
private readonly bool _shouldConvertPdbs;
private readonly bool _isDryRun;
private readonly FrozenSet<string> _packageFileExclusions;
private readonly bool _treatPdbConversionIssuesAsInfo;
/// <summary>
/// Initializes a new instance of the <see cref="SymbolUploadHelper"/> class.
/// </summary>
/// <param name="logger">The logger instance.</param>
/// <param name="symbolToolPath">The path to the symbol tool.</param>
/// <param name="options">The symbol publisher options.</param>
/// <param name="workingDir">The working directory.</param>
internal SymbolUploadHelper(ITracer logger, string symbolToolPath, SymbolPublisherOptions options, string? workingDir = null)
{
// These are all validated by the factory since this constructor is internal.
// If these invariants change, the factory should be updated.
Debug.Assert(logger is not null);
Debug.Assert(options is not null);
Debug.Assert(!string.IsNullOrEmpty(symbolToolPath) && (File.Exists(symbolToolPath) || options.IsDryRun));
_tracerFactory = new ScopedTracerFactory(logger!);
_globalTracer = _tracerFactory.CreateTracer(nameof(SymbolUploadHelper));
_symbolToolTimeoutInMins = options.OperationTimeoutInMins;
_commonArgs = $"-s https://artifacts.dev.azure.com/{options!.AzdoOrg} --patAuthEnvVar {PathEnvVarName} -t --timeout {_symbolToolTimeoutInMins}";
if (options.VerboseClient)
{
// the true verbosity level is "verbose" but the tool is very chatty at that level.
// "info" is a good balance for the errors that tend to come up in our layer.
_commonArgs += " --tracelevel info";
}
else
{
_commonArgs += " --tracelevel warn";
}
_workingDir = workingDir ?? Path.GetTempPath();
_credential = options.Credential;
_symbolToolPath = symbolToolPath;
// This is a special case for dotnet internal builds, particularly to control the special indexing of
// diagnostic artifacts coming from the runtime build. Any runtime pack or cross OS diagnostic symbol
// package needs this - and it will generate a special JSON manifest for the symbol client to consume.
// All other builds should not set this flag in the interest of perf.
_shouldGenerateManifest = options.DotnetInternalPublishSpecialClrFiles;
// This is an extremely slow operation and should be used sparingly. We usually only want to do this
// in the staging/release pipeline, not in the nightly build pipeline.
_shouldConvertPdbs = options.ConvertPortablePdbs;
_isDryRun = options.IsDryRun;
_packageFileExclusions = options.PackageFileExcludeList;
if (_shouldConvertPdbs)
{
_treatPdbConversionIssuesAsInfo = options.TreatPdbConversionIssuesAsInfo;
_pdbConverter = new PdbConverter(diagnostic =>
{
string message = diagnostic.ToString();
if (_treatPdbConversionIssuesAsInfo)
{
_globalTracer.Information(message);
}
else if (options.PdbConversionTreatAsWarning.Contains((int)diagnostic.Id))
{
_globalTracer.Warning(message);
}
else
{
_globalTracer.Error(message);
}
});
}
}
public async Task<int> GetClientDiagnosticInfo()
{
ScopedTracer logger = _tracerFactory.CreateTracer(nameof(GetClientDiagnosticInfo));
logger.Information("Client Path: {0}", _symbolToolPath);
return await RunSymbolCommand("help", ".", logger).ConfigureAwait(false);
}
/// <summary>
/// Creates a symbol request.
/// </summary>
/// <param name="name">The name of the symbol request.</param>
/// <returns>The result of the operation.</returns>
public async Task<int> CreateRequest(string? name)
{
ScopedTracer logger = _tracerFactory.CreateTracer(nameof(CreateRequest));
SymbolRequestHelpers.ValidateRequestName(name, logger);
logger.Information("Creating symbol request: {0}", name!);
string arguments = $"create {_commonArgs} --name {name}";
return await RunSymbolCommand(arguments, ".", logger).ConfigureAwait(false);
}
/// <summary>
/// Adds directory to a symbol request. This will convert portable PDBs as long as the PE is next to the
/// PDB and the options specified conversion.
/// </summary>
/// <param name="name">The name of the symbol request to append to. Must be non-finalized.</param>
/// <param name="files">The files to add.</param>
/// <returns>The result of the operation.</returns>
public async Task<int> AddDirectory(string? name, string pathToAdd)
{
ScopedTracer logger = _tracerFactory.CreateTracer(nameof(AddDirectory));
SymbolRequestHelpers.ValidateRequestName(name, logger);
try
{
if (_shouldConvertPdbs)
{
ConvertPortablePdbsInDirectory(logger, pathToAdd);
}
return await AddDirectoryCore(name!, pathToAdd, manifestPath: null, logger).ConfigureAwait(false);
}
finally
{
string convertedFolder = GetConvertedPdbFolder(pathToAdd);
if (_shouldConvertPdbs && !Directory.Exists(convertedFolder))
{
logger.Information("Cleaning up symbol conversion directory {0}", convertedFolder);
try { Directory.Delete(convertedFolder, recursive: true); } catch { }
}
}
}
/// <summary>
/// Adds a package to a symbol request. This respects conversion requests and manifest generation
/// if such options were specified at helper creation time.
/// </summary>
/// <param name="name">The name of the symbol request.</param>
/// <param name="packagePath">The path to the package.</param>
/// <returns>The result of the operation.</returns>
public async Task<int> AddPackageToRequest(string? name, string packagePath)
{
ScopedTracer logger = _tracerFactory.CreateTracer(nameof(AddPackagesToRequest));
SymbolRequestHelpers.ValidateRequestName(name, logger);
string packageName = Path.GetFileName(packagePath);
using IDisposable scopeToken = logger.AddSubScope(packageName);
return await AddPackageToRequestCore(name!, packagePath, logger).ConfigureAwait(false);
}
/// <summary>
/// Adds multiple packages to a symbol request. This respects conversion requests and manifest generation
/// if such options were specified at helper creation time.
/// </summary>
/// <param name="name">The name of the symbol request.</param>
/// <param name="packagePaths">The paths to the packages.</param>
/// <returns>The result of the operation.</returns>
public async Task<int> AddPackagesToRequest(string? name, IEnumerable<string> packagePaths)
{
ScopedTracer logger = _tracerFactory.CreateTracer(nameof(AddPackagesToRequest));
SymbolRequestHelpers.ValidateRequestName(name, logger);
int result = 0;
foreach (string package in packagePaths)
{
string packageName = Path.GetFileName(package);
using IDisposable scopeToken = logger.AddSubScope(packageName);
result = await AddPackageToRequestCore(name!, package, logger).ConfigureAwait(false);
if (result != 0)
{
break;
}
}
return result;
}
/// <summary>
/// Finalizes a symbol request.
/// </summary>
/// <param name="name">The name of the symbol request.</param>
/// <param name="daysToRetain">The number of days to retain the request.</param>
/// <returns>The result of the operation.</returns>
public async Task<int> FinalizeRequest(string? name, uint daysToRetain)
{
ScopedTracer logger = _tracerFactory.CreateTracer(nameof(FinalizeRequest));
SymbolRequestHelpers.ValidateRequestName(name, logger);
logger.WriteLine("Finalize symbol request: {0}", name!);
string arguments = $"finalize {_commonArgs} --name {name} --expirationInDays {daysToRetain}";
return await RunSymbolCommand(arguments, ".", logger).ConfigureAwait(false);
}
/// <summary>
/// Deletes a symbol request.
/// </summary>
/// <param name="name">The name of the symbol request.</param>
/// <returns>The result of the operation.</returns>
public async Task<int> DeleteRequest(string? name, bool synchronous = false)
{
ScopedTracer logger = _tracerFactory.CreateTracer(nameof(DeleteRequest));
SymbolRequestHelpers.ValidateRequestName(name, logger);
logger.WriteLine("Deleting symbol request: {0}", name!);
string arguments = $"delete {_commonArgs} --name {name} --quiet";
if (synchronous)
{
arguments += " --synchronous";
}
return await RunSymbolCommand(arguments, ".", logger).ConfigureAwait(false);
}
private async Task<int> AddDirectoryCore(string name, string pathToAdd, string? manifestPath, ScopedTracer logger)
{
logger.WriteLine("Adding directory {0} to request {1}", pathToAdd, name);
string arguments = $"adddirectory {_commonArgs} -n {name} --directory {pathToAdd} --recurse true";
if (manifestPath is not null)
{
arguments += " --manifest " + manifestPath;
}
return await RunSymbolCommand(arguments, pathToAdd, logger).ConfigureAwait(false);
}
private async Task<int> AddPackageToRequestCore(string name, string packagePath, ScopedTracer logger)
{
// Create a temporary directory to extract the package contents.
DirectoryInfo packageDirInfo = CreateTempDirectory();
string packageExtractDir = packageDirInfo.FullName;
try
{
logger.WriteLine("Processing package");
using ZipArchive archive = ZipFile.Open(packagePath, ZipArchiveMode.Read);
logger.Information("Extracting symbol package {0} to {1}", packagePath, packageExtractDir);
foreach (ZipArchiveEntry entry in archive.Entries)
{
if (entry.FullName.EndsWith('/'))
{
Debug.Assert(entry.Length == 0);
continue;
}
if (!ShouldIndexPackageFile(entry.FullName))
{
logger.Verbose("Skipping {0}", entry.FullName);
continue;
}
logger.Verbose("Extracting {0}", entry.FullName);
string entryPath = Path.Combine(packageExtractDir, entry.FullName);
_ = Directory.CreateDirectory(Path.GetDirectoryName(entryPath)!);
using Stream entryStream = entry.Open();
using FileStream entryFile = File.Create(entryPath);
await entryStream.CopyToAsync(entryFile).ConfigureAwait(false);
}
if (_shouldConvertPdbs)
{
ConvertPortablePdbsInDirectory(logger, packageExtractDir);
}
string? manifest = null;
if (_shouldGenerateManifest)
{
manifest = Path.Combine(packageExtractDir, "correlatedSymKeysManifest.json");
if (!SymbolManifestGenerator.SymbolManifestGenerator.GenerateManifest(logger, packageDirInfo, manifest, specialFilesRequireAdjacentRuntime: false))
{
logger.Error("Failed to generate symbol manifest");
return -1;
}
logger.Verbose("Generated manifest in {0}", manifest);
}
logger.WriteLine("Adding package {0} to request {1}", packagePath, name);
return await AddDirectoryCore(name, packageExtractDir, manifest, logger).ConfigureAwait(false);
}
catch (Exception ex)
{
logger.Error("Failed to process package {0}: {1}", packagePath, ex);
return -1;
}
finally
{
logger.Information("Cleaning up temporary directory {0}", packageDirInfo.FullName);
try { packageDirInfo.Delete(recursive: true); } catch {}
}
bool ShouldIndexPackageFile(string relativeFilePath)
{
if (relativeFilePath.StartsWith("ref/")
|| relativeFilePath.StartsWith("_rels/")
|| relativeFilePath.StartsWith("package/")
|| relativeFilePath.EndsWith("_.pdb"))
{
// Quick bail - special nupkg files and ref assemblies are not indexed.
return false;
}
relativeFilePath = relativeFilePath.Replace("//", "/");
if (_packageFileExclusions.Contains(relativeFilePath))
{
return false;
}
string extension = Path.GetExtension(relativeFilePath);
return s_validExtensions.Contains(extension);
}
}
private void ConvertPortablePdbsInDirectory(ScopedTracer logger, string filesDir)
{
Action<string> logWarning = _treatPdbConversionIssuesAsInfo ? logger.Information : logger.Error;
string convertedPdbFolder = GetConvertedPdbFolder(filesDir);
_ = Directory.CreateDirectory(convertedPdbFolder);
foreach (string file in Directory.EnumerateFiles(filesDir, "*.pdb", SearchOption.AllDirectories))
{
using Stream pdbStream = File.OpenRead(file);
if (!PdbConverter.IsPortable(pdbStream))
{
continue;
}
logger.Verbose("Converting {0} to classic PDB format", file);
string pePath = Path.ChangeExtension(file, ".dll");
// Try to fall back to the framework exe scenario.
if (!File.Exists(pePath))
{
pePath = Path.ChangeExtension(file, ".exe");
}
if (!File.Exists(pePath))
{
logWarning($"Conversion error: could not find matching PE file for {file}");
continue;
}
string convertedPdbPath = Path.Combine(convertedPdbFolder, Path.GetFileName(file));
try
{
using Stream peStream = File.OpenRead(pePath);
using Stream convertedPdbStream = File.Create(convertedPdbPath);
_pdbConverter!.ConvertPortableToWindows(peStream, pdbStream, convertedPdbStream);
}
catch (Exception ex)
{
logWarning($"Conversion error: {ex.Message}");
continue;
}
logger.Verbose("Converted successfully to {0}.", convertedPdbPath);
}
}
private static string GetConvertedPdbFolder(string filesDir) => Path.Combine(filesDir, ConversionFolderName);
private DirectoryInfo CreateTempDirectory()
{
string tempDir = Path.Combine(_workingDir, Path.GetRandomFileName());
while (Directory.Exists(tempDir) || File.Exists(tempDir))
{
tempDir = Path.Combine(_workingDir, Path.GetRandomFileName());
}
return Directory.CreateDirectory(tempDir);
}
private async Task<int> RunSymbolCommand(string arguments, string directory, ScopedTracer logger, CancellationToken ct = default)
{
// TODO: Add retry logic. Need to parse output stream for this.
logger.Verbose("Running command: {0} {1} from '{2}'", _symbolToolPath, arguments, directory);
using IDisposable scopedTrace = logger.AddSubScope("symbol.exe");
if (_isDryRun)
{
logger.Information("Would run command: {0} {1} from '{2}'", _symbolToolPath, arguments, directory);
return 0;
}
// This sentinel task is used to indicate that the output has been fully read. It's never completed.
TaskCompletionSource<string?> outputFinishedSentinel = new();
Stopwatch sw = Stopwatch.StartNew();
try
{
AccessToken token = await _credential.GetTokenAsync(new TokenRequestContext([AzureDevOpsResource]), ct).ConfigureAwait(false);
ProcessStartInfo info = new(_symbolToolPath, arguments)
{
UseShellExecute = false,
RedirectStandardError = true,
RedirectStandardOutput = true,
WorkingDirectory = directory,
Environment = { [PathEnvVarName] = token.Token }
};
using CancellationTokenSource lcts = CancellationTokenSource.CreateLinkedTokenSource(ct);
using Process process = new()
{
StartInfo = info
};
_ = process.Start();
lcts.CancelAfter(TimeSpan.FromMinutes(_symbolToolTimeoutInMins));
ct = lcts.Token;
Task processExit = process.WaitForExitAsync(ct);
StreamReader standardOutput = process.StandardOutput;
Task<string?> outputAvailable = standardOutput.ReadLineAsync(ct).AsTask();
StreamReader standardError = process.StandardError;
Task<string?> errorAvailable = standardError.ReadLineAsync(ct).AsTask();
while (!ct.IsCancellationRequested && (outputAvailable != outputFinishedSentinel.Task || errorAvailable != outputFinishedSentinel.Task))
{
if (processExit.IsCompleted)
{
// We already did the work. Might as well drain the IO.
lcts.Dispose();
logger.Verbose("uploader completion detected after {0}. Draining I/O streams.", sw.Elapsed);
}
Task alertedTask = await Task.WhenAny(outputAvailable, errorAvailable).ConfigureAwait(false);
if (alertedTask == outputAvailable)
{
outputAvailable = await LogFromStreamReader(outputAvailable, standardOutput.ReadLineAsync, logger.Verbose, ct);
}
else if (alertedTask == errorAvailable)
{
errorAvailable = await LogFromStreamReader(errorAvailable, standardError.ReadLineAsync, logger.Error, ct);
}
}
if (ct.IsCancellationRequested && !process.HasExited)
{
try { process.Kill(); } catch (InvalidOperationException) { }
return -1;
}
// This should be a no-op if the process has already exited. Since it's not the ct doing this or we'd have exited, and we drained both
// output streams, this is the expected scenario.
await processExit.ConfigureAwait(false);
logger.Information("completed after {0} with exit code {1}", sw.Elapsed, process.ExitCode);
return process.ExitCode;
}
catch (Exception ex)
{
logger.Error("Unable to finish invocation or drain its output after {0}: {1}", sw.Elapsed, ex);
return -1;
}
async Task<Task<string?>> LogFromStreamReader(Task<string?> outputTask, Func<CancellationToken, ValueTask<string?>> readLine, Action<string> logMethod, CancellationToken ct)
{
ValueTask<string?> vt = new(outputTask);
while (!ct.IsCancellationRequested && outputTask.IsCompleted)
{
string? line = await vt.ConfigureAwait(false);
if (line is not null)
{
logMethod(line);
vt = readLine(ct);
}
else
{
return outputFinishedSentinel.Task;
}
}
return vt.AsTask();
}
}
}
|