// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System;
using System.CodeDom;
using System.IO;
using System.Linq;
using System.Security;
using System.Xml.Linq;
using Microsoft.Arcade.Common;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using Microsoft.DotNet.Build.Manifest;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
namespace Microsoft.DotNet.Build.Tasks.Feed;
/// <summary>
/// This task is used to gather a set of artifacts produced by a build and move them
/// to a staging location, either locally or in Azure DevOps, as well as produce a manifest describing those
/// artifacts. This is used today in the following scenarios:
/// - VMR builds, for each repo that builds within a vertical. the artifacts are copied to a local directory,
/// and the manifest is produced. These artifacts are used for downstream repos in the vertical. In this case:
/// - PushToLocalStorage is set to true
/// - AssetsLocalStorageDir, ShippingPackagesLocalStorageDir, NonShippingPackagesLocalStorageDir, AssetManifestsLocalStorageDir,
/// and PdbArtifactsLocalStorageDir are set.
/// - FutureArtifactName and FutureArtifactPublishBasePath are not set.
/// - Publishing version should be v3 or v4.
/// - VMR builds, at the end of a vertical. Or non-VMR builds using v4 publishing. In these cases, the artifacts are copied to a staging directory,
/// ready for upload as pipeline artifacts. The manifest is produced for all artifacts that should exit the vertical,
/// and the assets future pipeline artifact location is recorded in the manifest based on
/// FutureArtifactName and FutureArtifactPublishBasePath. In this case,
/// - PushToLocalStorage is set to true
/// - AssetsLocalStorageDir, ShippingPackagesLocalStorageDir, NonShippingPackagesLocalStorageDir, AssetManifestsLocalStorageDir,
/// and PdbArtifactsLocalStorageDir are set.
/// - FutureArtifactName and FutureArtifactPublishBasePath are set.
/// - Publishing version should be v4.
/// - Non-VMR builds, with publishing V3. In this case, the artifacts are uploaded to Azure DevOps using logging commands to
/// a set of well known artifact locations. PackageArtifacts, PdbArtifacts, BlobArtifacts, and AssetManifests.
/// When this scenario is active,
/// - PushToLocalStorage is set to false
/// - AssetsLocalStorageDir, ShippingPackagesLocalStorageDir, NonShippingPackagesLocalStorageDir, AssetManifestsLocalStorageDir.
/// - PdbArtifactsLocalStorageDir must be set if PDBs are present.
/// - FutureArtifactName and FutureArtifactPublishBasePath are not set.
/// - Publishing version should be v3.
/// </summary>
public class PushToBuildStorage : MSBuildTaskBase
{
[Required]
public ITaskItem[] ItemsToPush { get; set; }
public string ManifestRepoName { get; set; }
public string ManifestRepoUri { get; set; }
public string ManifestBuildId { get; set; } = "no build id provided";
public string ManifestBranch { get; set; }
public string ManifestCommit { get; set; }
/// <summary>
/// Indicates the source of the artifacts. For a VMR build, the `ManifestRepoName` is dotnet/dotnet,
/// while the `ManifestRepoOrigin` corresponds to the actual product repository.
/// </summary>
public string ManifestRepoOrigin { get; set; }
public string[] ManifestBuildData { get; set; }
public string AzureDevOpsCollectionUri { get; set; }
public string AzureDevOpsProject { get; set; }
public int AzureDevOpsBuildId { get; set; }
// Sign* parameters are deprecated and no longer used.
public ITaskItem[] ItemsToSign { get; set; }
public ITaskItem[] StrongNameSignInfo { get; set; }
public ITaskItem[] FileSignInfo { get; set; }
public ITaskItem[] FileExtensionSignInfo { get; set; }
public ITaskItem[] CertificatesSignInfo { get; set; }
public string AssetManifestPath { get; set; }
public bool IsStableBuild { get; set; }
public bool IsReleaseOnlyPackageVersion { get; set; }
/// <summary>
/// Represents where assets should be copied locally, either for staging for upload
/// or for propagation to another phase of the VMR build.
/// </summary>
public string AssetsLocalStorageDir { get; set; }
/// <summary>
/// Represents where shipping packages should be copied locally, either for staging for upload
/// or for propagation to another phase of the VMR build.
/// </summary>
public string ShippingPackagesLocalStorageDir { get; set; }
/// <summary>
/// Represents where nonshipping packages should be copied locally, either for staging for upload
/// or for propagation to another phase of the VMR build.
/// </summary>
public string NonShippingPackagesLocalStorageDir { get; set; }
/// <summary>
/// Represents where asset manifests should be copied locally, either for staging for upload
/// or for propagation to another phase of the VMR build.
/// </summary>
public string AssetManifestsLocalStorageDir { get; set; }
/// <summary>
/// Represents where pdb artifacts should be copied locally, either for staging for upload
/// or for propagation to another phase of the VMR build.
///
/// NOTE: In non-VMR builds, this represents the location of the PDBs that are copied
/// to before uploading to the PDBArtifacts dir.
/// </summary>
public string PdbArtifactsLocalStorageDir { get; set; }
/// <summary>
/// If true, the artifacts will be copied to the local storage directories.
/// This is also true if the publishing version is V4.
/// </summary>
public bool PushToLocalStorage { get; set; }
/// <summary>
/// The final path for any packages published to <see cref="ShippingPackagesLocalStorageDir"/>
/// or <see cref="NonShippingPackagesLocalStorageDir"/> should have the artifact's RepoOrigin
/// appended as a subfolder to the published path.
/// </summary>
public bool PreserveRepoOrigin { get; set; }
/// <summary>
/// The visibility of the artifacts to put in the manifest.
/// </summary>
public ITaskItem[] ArtifactVisibilitiesToPublish { get; set; }
/// <summary>
/// Which version should the build manifest be tagged with.
/// By default he latest version is used.
/// </summary>
public int PublishingVersion { get; set; }
/// <summary>
/// Gets or sets a value that indicates whether to use hard links for the copied files
/// rather than copy the files, if it's possible to do so.
/// </summary>
public bool UseHardlinksIfPossible { get; set; } = true;
/// <summary>
/// Under v4 publishing, the artifact name is the name of the file that will be published to the blob feed.
/// </summary>
public string FutureArtifactName { get; set; }
/// <summary>
/// Under v4 publishing, the path that will be published to FutureArtifactName. If FutureArtifactName is passed,
/// then FutureArtifactPublishBasePath must also be passed.
/// </summary>
public string FutureArtifactPublishBasePath { get; set; }
private bool ShouldLocallyStageArtifacts { get => PushToLocalStorage || _publishingVersion >= PublishingInfraVersion.V4; }
private PublishingInfraVersion _publishingVersion = PublishingInfraVersion.Latest;
private IFileSystem _fileSystem;
public override void ConfigureServices(IServiceCollection collection)
{
collection.TryAddSingleton<IBlobArtifactModelFactory, BlobArtifactModelFactory>();
collection.TryAddSingleton<IPackageArtifactModelFactory, PackageArtifactModelFactory>();
collection.TryAddSingleton<IPdbArtifactModelFactory, PdbArtifactModelFactory>();
collection.TryAddSingleton<IBuildModelFactory, BuildModelFactory>();
collection.TryAddSingleton<IFileSystem>(provider => new PushToBuildStorageFileSystem(UseHardlinksIfPossible, Log));
collection.TryAddSingleton<IPackageArchiveReaderFactory, PackageArchiveReaderFactory>();
collection.TryAddSingleton<INupkgInfoFactory, NupkgInfoFactory>();
collection.TryAddSingleton(Log);
}
public bool PrepAndValidateTask()
{
if (PublishingVersion != default)
{
if (!Enum.IsDefined(typeof(PublishingInfraVersion), PublishingVersion))
{
Log.LogError($"Invalid publishing version '{PublishingVersion}'");
return false;
}
_publishingVersion = (PublishingInfraVersion)PublishingVersion;
}
// Must always have something to push, or to create a manifest with.
if (ItemsToPush == null || ItemsToPush.Length == 0)
{
Log.LogError($"ItemsToPush is not specified.");
return false;
}
if (ShouldLocallyStageArtifacts)
{
if (string.IsNullOrEmpty(AssetsLocalStorageDir) ||
string.IsNullOrEmpty(ShippingPackagesLocalStorageDir) ||
string.IsNullOrEmpty(NonShippingPackagesLocalStorageDir) ||
string.IsNullOrEmpty(PdbArtifactsLocalStorageDir) ||
string.IsNullOrEmpty(AssetManifestsLocalStorageDir))
{
Log.LogError("AssetsLocalStorageDir, ShippingPackagesLocalStorageDir, NonShippingPackagesLocalStorageDir, PdbArtifactsLocalStorageDir and AssetManifestsLocalStorageDir need " +
"to be specified if PublishToLocalStorage is set to true or V4 publishing is enabled");
return false;
}
}
// Validation of parameters specific to a publishing version
switch (_publishingVersion)
{
case PublishingInfraVersion.UnsupportedV1:
case PublishingInfraVersion.UnsupportedV2:
Log.LogError($"Publishing version '{_publishingVersion}' is not supported.");
return false;
case PublishingInfraVersion.V3:
if (!string.IsNullOrEmpty(FutureArtifactName) ||
!string.IsNullOrEmpty(FutureArtifactPublishBasePath))
{
Log.LogError($"FutureArtifactName and FutureArtifactPublishBasePath are not supported in publishing version '{_publishingVersion}'.");
return false;
}
break;
case PublishingInfraVersion.V4:
if (string.IsNullOrEmpty(FutureArtifactName) !=
string.IsNullOrEmpty(FutureArtifactPublishBasePath))
{
Log.LogError($"FutureArtifactName and FutureArtifactPublishBasePath must be both be specified if either is specified.");
return false;
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(_publishingVersion), _publishingVersion, null);
}
if (ShouldLocallyStageArtifacts)
{
Log.LogMessage(MessageImportance.High, "Performing push to local artifacts storage.");
}
else
{
Log.LogMessage(MessageImportance.High, "Performing push to Azure DevOps artifacts storage.");
}
return true;
}
public bool ExecuteTask(IFileSystem fileSystem,
IBlobArtifactModelFactory blobArtifactModelFactory,
IPackageArtifactModelFactory packageArtifactModelFactory,
IPdbArtifactModelFactory pdbArtifactModelFactory,
IBuildModelFactory buildModelFactory)
{
_fileSystem = fileSystem;
try
{
if (!PrepAndValidateTask())
{
return false;
}
var artifactVisibilities = GetVisibilitiesToPublish(ArtifactVisibilitiesToPublish);
var buildModel = buildModelFactory.CreateModel(
ItemsToPush,
artifactVisibilities,
ManifestBuildId,
ManifestBuildData,
!string.IsNullOrEmpty(ManifestRepoName) ? ManifestRepoName : ManifestRepoUri,
ManifestBranch,
ManifestCommit,
ManifestRepoOrigin,
_publishingVersion <= PublishingInfraVersion.V3 ? IsStableBuild : false,
_publishingVersion,
_publishingVersion <= PublishingInfraVersion.V3 ? IsReleaseOnlyPackageVersion : false);
if (buildModel == null)
{
Log.LogError($"Failed to construct build model from input artifacts.");
return false;
}
foreach (var package in buildModel.Artifacts.Packages)
{
if (!fileSystem.FileExists(package.OriginalFile))
{
Log.LogError($"Could not find file {package.OriginalFile}.");
continue;
}
LocallyStageArtifactsOrPushToAzDO(package);
}
foreach (var blobArtifact in buildModel.Artifacts.Blobs)
{
if (!fileSystem.FileExists(blobArtifact.OriginalFile))
{
Log.LogError($"Could not find file {blobArtifact.OriginalFile}.");
continue;
}
LocallyStageArtifactsOrPushToAzDO(blobArtifact);
}
// We allow users to not specify a PdbArtifactsLocalStorageDir to avoid breaking
// some repos. Check now that it exists.
// Note that if publishing version was v4, or if we are pushing to local storage,
// then this was already checked.
if (string.IsNullOrEmpty(PdbArtifactsLocalStorageDir) && buildModel.Artifacts.Pdbs.Count > 0)
{
Log.LogError($"PdbArtifactsLocalStorageDir must be specified if PDBs are present.");
return false;
}
foreach (var pdbArtifact in buildModel.Artifacts.Pdbs)
{
if (!fileSystem.FileExists(pdbArtifact.OriginalFile))
{
Log.LogError($"Could not find file {pdbArtifact.OriginalFile}.");
continue;
}
LocallyStageArtifactsOrPushToAzDO(pdbArtifact);
}
if (_publishingVersion <= PublishingInfraVersion.V3 && !ShouldLocallyStageArtifacts && buildModel.Artifacts.Pdbs.Any())
{
// Upload the full set of PDBs
Log.LogMessage(MessageImportance.High,
$"##vso[artifact.upload containerfolder=PdbArtifacts;artifactname=PdbArtifacts]{PdbArtifactsLocalStorageDir}");
}
// Write the manifest, then create an artifact for it.
Log.LogMessage(MessageImportance.High, $"Writing build manifest file '{AssetManifestPath}'...");
fileSystem.WriteToFile(AssetManifestPath, buildModel.ToXml().ToString(SaveOptions.DisableFormatting));
// Generate an artifact for the asset manifest and push it to storage.
AssetManifestModel assetManifestModel = new AssetManifestModel
{
OriginalFile = AssetManifestPath,
Id = _fileSystem.GetFileName(AssetManifestPath)
};
LocallyStageArtifactsOrPushToAzDO(assetManifestModel);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return !Log.HasLoggedErrors;
}
private void LocallyStageArtifactsOrPushToAzDO(ArtifactModel artifactModel)
{
string originalArtifactPath = artifactModel.OriginalFile;
string artifactDestinationPath = originalArtifactPath;
if (ShouldLocallyStageArtifacts)
{
artifactDestinationPath = PushArtifactToLocalStorage(artifactModel, originalArtifactPath);
}
else if (_publishingVersion == PublishingInfraVersion.V3)
{
PushArtifactToBuildStorage(artifactModel, originalArtifactPath);
}
// If using V4, then record the pipeline artifact location of the asset.
if (_publishingVersion == PublishingInfraVersion.V4 && !string.IsNullOrEmpty(FutureArtifactPublishBasePath))
{
try
{
string relativePath = _fileSystem.GetRelativePath(FutureArtifactPublishBasePath, artifactDestinationPath);
// Set the pipeline artifact path to the relative path.
// This path should be in unix-style path form.
artifactModel.PipelineArtifactName = FutureArtifactName;
artifactModel.PipelineArtifactPath = relativePath.Replace(@"\", "/");
}
catch (ArgumentException)
{
Log.LogError($"Could not determine relative path from '{FutureArtifactPublishBasePath}' to '{artifactDestinationPath}'.");
}
}
}
/// <summary>
/// Copies the artifact to the locally specified storage directory.
/// </summary>
/// <param name="artifactModel">Artifact model</param>
/// <param name="originalArtifactPath">Original path to the artifact</param>
/// <param name="filename"></param>
/// <returns>Target path that the artifact was copied to.</returns>
/// <exception cref="ArgumentOutOfRangeException">Invalid publishing version.</exception>
private string PushArtifactToLocalStorage(ArtifactModel artifactModel, string originalArtifactPath)
{
string filename = _fileSystem.GetFileName(originalArtifactPath);
string artifactDestinationPath;
switch (artifactModel)
{
case AssetManifestModel _:
artifactDestinationPath = _fileSystem.PathCombine(AssetManifestsLocalStorageDir, filename);
EnsureDirectoryAndCopyFile(originalArtifactPath, artifactDestinationPath);
break;
case PackageArtifactModel _:
string packageDestinationDirectory = artifactModel.NonShipping
? NonShippingPackagesLocalStorageDir
: ShippingPackagesLocalStorageDir;
if (PreserveRepoOrigin)
{
packageDestinationDirectory = _fileSystem.PathCombine(packageDestinationDirectory, artifactModel.RepoOrigin);
}
artifactDestinationPath = _fileSystem.PathCombine(packageDestinationDirectory, filename);
EnsureDirectoryAndCopyFile(originalArtifactPath, artifactDestinationPath);
break;
case BlobArtifactModel _:
string relativeBlobPath = artifactModel.Id;
artifactDestinationPath = _fileSystem.PathCombine(
AssetsLocalStorageDir,
string.IsNullOrEmpty(relativeBlobPath) ? filename : relativeBlobPath);
EnsureDirectoryAndCopyFile(originalArtifactPath, artifactDestinationPath);
break;
case PdbArtifactModel _:
string relativePdbPath = artifactModel.Id;
artifactDestinationPath = _fileSystem.PathCombine(
PdbArtifactsLocalStorageDir,
string.IsNullOrEmpty(relativePdbPath) ? filename : relativePdbPath);
EnsureDirectoryAndCopyFile(originalArtifactPath, artifactDestinationPath);
break;
default:
throw new ArgumentOutOfRangeException(nameof(artifactModel));
}
return artifactDestinationPath;
void EnsureDirectoryAndCopyFile(string artifactPath, string artifactDestinationPath)
{
_fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(artifactDestinationPath));
_fileSystem.CopyFile(artifactPath, artifactDestinationPath, true);
}
}
/// <summary>
/// Pushes artifacts to build storage using logging commands, with the exception of PDBs, which are staged for upload as a unit.
/// </summary>
/// <param name="artifactModel">Artifact</param>
/// <param name="originalArtifactPath">Original path to the artifact</param>
/// <exception cref="ArgumentOutOfRangeException">Publishing version was out of range</exception>
private void PushArtifactToBuildStorage(ArtifactModel artifactModel, string originalArtifactPath)
{
switch (artifactModel)
{
case AssetManifestModel _:
Log.LogMessage(MessageImportance.High,
$"##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]{originalArtifactPath}");
break;
case PackageArtifactModel _:
Log.LogMessage(MessageImportance.High,
$"##vso[artifact.upload containerfolder=PackageArtifacts;artifactname=PackageArtifacts]{originalArtifactPath}");
break;
case BlobArtifactModel _:
Log.LogMessage(MessageImportance.High,
$"##vso[artifact.upload containerfolder=BlobArtifacts;artifactname=BlobArtifacts]{originalArtifactPath}");
break;
case PdbArtifactModel _:
string pdbArtifactTarget = _fileSystem.PathCombine(PdbArtifactsLocalStorageDir, artifactModel.Id);
_fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(pdbArtifactTarget));
// Copy the PDB artifact to the temp local dir.
_fileSystem.CopyFile(originalArtifactPath, pdbArtifactTarget, true);
break;
default:
throw new ArgumentOutOfRangeException(nameof(artifactModel));
}
}
private static ArtifactVisibility GetVisibilitiesToPublish(ITaskItem[] allowedVisibilities)
{
if (allowedVisibilities is null || allowedVisibilities.Length == 0)
{
return ArtifactVisibility.External;
}
ArtifactVisibility visibility = 0;
foreach (var item in allowedVisibilities)
{
if (Enum.TryParse(item.ItemSpec, true, out ArtifactVisibility parsedVisibility))
{
visibility |= parsedVisibility;
}
else
{
throw new ArgumentException($"Invalid visibility: {item.ItemSpec}");
}
}
return visibility;
}
}
/// <summary>
/// File system with some extra hard linking functionality useful for the PushToBuildStorage task.
/// Specifically, we override CopyFile to enable hard linking
/// </summary>
class PushToBuildStorageFileSystem : FileSystem
{
bool _enableHardLinking = false;
private readonly TaskLoggingHelper _log;
public PushToBuildStorageFileSystem(
bool enableHardLinking,
TaskLoggingHelper log)
{
_enableHardLinking = enableHardLinking;
_log = log;
}
/// <summary>
///
/// </summary>
/// <param name="basePath">Base path</param>
/// <param name="targetPath">Target path</param>
/// <returns>Relative path from basePath to targetPath</returns>
/// <exception cref="ArgumentNullException"></exception>
public override string GetRelativePath(string basePath, string targetPath)
{
// Resolve the basePath and targetPath to absolute paths.
basePath = GetFullPath(basePath);
targetPath = GetFullPath(targetPath);
if (targetPath.IndexOf(basePath) != 0)
{
throw new ArgumentException("targetPath is not relative to basePath");
}
return targetPath.Substring(basePath.Length).TrimStart('/', '\\');
}
/// <summary>
/// The below method implementation is copied from msbuild's Copy task and adjusted.
/// </summary>
/// <param name="sourceFileName">Source file for the copy</param>
/// <param name="destFileName">Destination file for the copy</param>
/// <param name="overwrite">Overwrite?</param>
public override void CopyFile(string sourceFileName, string destFileName, bool overwrite)
{
FileInfo destFile = new(destFileName);
if (_enableHardLinking)
{
// NativeMethods.MakeHardLink cannot overwrite an existing file or link
// so we need to delete the existing entry before we create the hard link.
if (destFile.Exists && !destFile.IsReadOnly)
{
try
{
DeleteFile(destFile.FullName);
}
catch (Exception ex) when (IsIoRelatedException(ex))
{
}
}
_log.LogMessage(MessageImportance.Normal, $"Creating hard link to copy \"{sourceFileName}\" to \"{destFileName}\".");
string errorMessage = string.Empty;
if (!NativeMethods.MakeHardLink(destFileName, sourceFileName, ref errorMessage))
{
_log.LogMessage(MessageImportance.Normal, $"Could not use a link to copy \"{sourceFileName}\" to \"{destFileName}\". Copying the file instead. {errorMessage}");
File.Copy(sourceFileName, destFileName, overwrite);
}
}
else
{
File.Copy(sourceFileName, destFileName, overwrite);
}
// If the destinationFile file exists, then make sure it's read-write.
// The File.Copy command copies attributes, but our copy needs to
// leave the file writeable.
if (new FileInfo(sourceFileName).IsReadOnly)
{
// Ensure the read-only attribute on the specified file is off, so
// the file is writeable.
if (destFile.Exists)
{
if (destFile.IsReadOnly)
{
_log.LogMessage(MessageImportance.Low, $"Removing read-only attribute from \"{destFile.FullName}\".");
File.SetAttributes(destFile.FullName, FileAttributes.Normal);
}
}
}
// Determine whether the exception is file-IO related.
static bool IsIoRelatedException(Exception e)
{
// These all derive from IOException
// DirectoryNotFoundException
// DriveNotFoundException
// EndOfStreamException
// FileLoadException
// FileNotFoundException
// PathTooLongException
// PipeException
return e is UnauthorizedAccessException
|| e is NotSupportedException
|| (e is ArgumentException && !(e is ArgumentNullException))
|| e is SecurityException
|| e is IOException;
}
}
}