File: src\ZipData.cs
Web Access
Project: src\src\Microsoft.DotNet.SignTool\Microsoft.DotNet.SignTool.csproj (Microsoft.DotNet.SignTool)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Data;
using System.Diagnostics;
using System.Runtime.InteropServices;
using NuGet.Packaging;
using Microsoft.DotNet.Build.Tasks.Installers;
 
#if NET472
using System.IO.Packaging;
#else
using System.Formats.Tar;
#endif
 
namespace Microsoft.DotNet.SignTool
{
    /// <summary>
    /// Data for a zip container. Can refer to any zip format such as VSIX, NuPkg, or a raw zip archive.
    /// </summary>
    internal sealed class ZipData
    {
        /// <summary>
        /// Signing information.
        /// </summary>
        internal FileSignInfo FileSignInfo { get; }
 
        /// <summary>
        /// The parts inside this container which need to be signed.
        /// </summary>
        internal ImmutableDictionary<string, ZipPart> NestedParts { get; }
 
        internal ZipData(FileSignInfo fileSignInfo, ImmutableDictionary<string, ZipPart> nestedBinaryParts)
        {
            FileSignInfo = fileSignInfo;
            NestedParts = nestedBinaryParts;
        }
 
        internal ZipPart? FindNestedPart(string relativeName)
        {
            if (NestedParts.TryGetValue(relativeName, out ZipPart part))
            {
                return part;
            }
 
            return null;
        }
 
        public static IEnumerable<(string relativePath, Stream content, long contentSize)> ReadEntries(TaskLoggingHelper log, string archivePath, string tempDir, string tarToolPath, string pkgToolPath, bool ignoreContent = false)
        {
            if (FileSignInfo.IsTarGZip(archivePath))
            {
                // Tar APIs not available on .NET FX. We need sign tool to run on desktop msbuild because building VSIX packages requires desktop.
#if NET472
                return ReadTarGZipEntries(archivePath, tempDir, tarToolPath, ignoreContent);
#else
                return ReadTarGZipEntries(archivePath)
                    .Select(entry => (entry.Name, entry.DataStream, entry.Length));
#endif
            }
            else if (FileSignInfo.IsPkg(archivePath) || FileSignInfo.IsAppBundle(archivePath))
            {
                return ReadPkgOrAppBundleEntries(log, archivePath, tempDir, pkgToolPath, ignoreContent);
            }
            else if (FileSignInfo.IsDeb(archivePath))
            {
#if NET472
                throw new NotImplementedException("Debian signing is not supported on .NET Framework");
#else
                return ReadDebContainerEntries(archivePath, "data.tar");
#endif
            }
 
            return ReadZipEntries(archivePath);
        }
 
        /// <summary>
        /// Repack the zip container with the signed files.
        /// </summary>
        public void Repack(TaskLoggingHelper log, string tempDir, string wixToolsPath, string tarToolPath, string pkgToolPath)
        {
#if NET472
            if (FileSignInfo.IsVsix())
            {
                RepackPackage(log);
            }
            else
#endif
            if (FileSignInfo.IsTarGZip())
            {
                RepackTarGZip(log, tempDir, tarToolPath);
            }
            else if (FileSignInfo.IsWixContainer())
            {
                RepackWixPack(log, tempDir, wixToolsPath);
            }
            else if (FileSignInfo.IsPkg() || FileSignInfo.IsAppBundle())
            {
                RepackPkgOrAppBundles(log, tempDir, pkgToolPath);
            }
            else if (FileSignInfo.IsDeb())
            {
#if NET472
                throw new NotImplementedException("Debian signing is not supported on .NET Framework");
#else
                RepackDebContainer(log, tempDir);
#endif
            }
            else 
            {
                RepackRawZip(log);
            }
        }
 
#if NET472
        /// <summary>
        /// Repack a zip container with a package structure.
        /// </summary>
        private void RepackPackage(TaskLoggingHelper log)
        {
            string getPartRelativeFileName(PackagePart part)
            {
                var path = part.Uri.OriginalString;
                if (!string.IsNullOrEmpty(path) && path[0] == '/')
                {
                    path = path.Substring(1);
                }
 
                return path;
            }
            
            using (var package = Package.Open(FileSignInfo.FullPath, FileMode.Open, FileAccess.ReadWrite))
            {
                foreach (var part in package.GetParts())
                {
                    var relativeName = getPartRelativeFileName(part);
                    var signedPart = FindNestedPart(relativeName);
                    if (!signedPart.HasValue)
                    {
                        log.LogMessage(MessageImportance.Low, $"Didn't find signed part for nested file: {FileSignInfo.FullPath} -> {relativeName}");
                        continue;
                    }
 
                    using (var signedStream = File.OpenRead(signedPart.Value.FileSignInfo.FullPath))
                    using (var partStream = part.GetStream(FileMode.Open, FileAccess.ReadWrite))
                    {
                        log.LogMessage(MessageImportance.Low, $"Copying signed stream from {signedPart.Value.FileSignInfo.FullPath} to {FileSignInfo.FullPath} -> {relativeName}.");
 
                        signedStream.CopyTo(partStream);
                        partStream.SetLength(signedStream.Length);
                    }
                }
            }
        }
#endif
 
        private static IEnumerable<(string relativePath, Stream content, long contentSize)> ReadZipEntries(string archivePath)
        {
            using var archive = new ZipArchive(File.OpenRead(archivePath), ZipArchiveMode.Read, leaveOpen: false);
 
            foreach (var entry in archive.Entries)
            {
                string relativePath = entry.FullName; // lgtm [cs/zipslip] Archive from trusted source
 
                // `entry` might be just a pointer to a folder. We skip those.
                if (relativePath.EndsWith("/") && entry.Name == "")
                {
                    yield return (relativePath, null, 0);
                }
                else
                {
                    var contentStream = entry.Open();
                    yield return (relativePath, contentStream, entry.Length);
                    contentStream.Close();
                }
            }
        }
 
        /// <summary>
        /// Repack raw zip container.
        /// </summary>
        private void RepackRawZip(TaskLoggingHelper log)
        {
            using (var zipStream = File.Open(FileSignInfo.FullPath, FileMode.Open))
            using (var archive = new ZipArchive(zipStream, ZipArchiveMode.Update))
            {
                foreach (ZipArchiveEntry entry in archive.Entries)
                {
                    string relativeName = entry.FullName;
                    var signedPart = FindNestedPart(relativeName);
                    if (!signedPart.HasValue)
                    {
                        log.LogMessage(MessageImportance.Low, $"Didn't find signed part for nested file: {FileSignInfo.FullPath} -> {relativeName}");
                        continue;
                    }
 
                    using (var signedStream = File.OpenRead(signedPart.Value.FileSignInfo.FullPath))
                    using (var entryStream = entry.Open())
                    {
                        log.LogMessage(MessageImportance.Low, $"Copying signed stream from {signedPart.Value.FileSignInfo.FullPath} to {FileSignInfo.FullPath} -> {relativeName}.");
 
                        signedStream.CopyTo(entryStream);
                        entryStream.SetLength(signedStream.Length);
                    }
                }
            }
        }
 
        private void RepackWixPack(TaskLoggingHelper log, string tempDir, string wixToolsPath)
        {
            // The wixpacks can have rather long paths when fully extracted.
            // To avoid issues, use the first element of the GUID (up to first -).
            // This does leave the very remote possibility of the dir already existing. In this case, the
            // create.cmd file will always end up being extracted twice, and ExtractToDirectory
            // will fail. Because of the very very remote possibility of this happening, no
            // attempt to workaround this possibility is made.
            var workingDirGuidSegment = Guid.NewGuid().ToString().Split('-')[0];
            var outputDirGuidSegment = Guid.NewGuid().ToString().Split('-')[0];
 
            string workingDir = Path.Combine(tempDir, "extract", workingDirGuidSegment);
            string outputDir = Path.Combine(tempDir, "output", outputDirGuidSegment);
            string createFileName = Path.Combine(workingDir, "create.cmd");
            string outputFileName = Path.Combine(outputDir, FileSignInfo.FileName);
 
            try
            {
                Directory.CreateDirectory(outputDir);
                ZipFile.ExtractToDirectory(FileSignInfo.WixContentFilePath, workingDir);
 
                var fileList = Directory.GetFiles(workingDir, "*", SearchOption.AllDirectories);
                foreach (var file in fileList)
                {
                    var relativeName = file.Substring($"{workingDir}\\".Length).Replace('\\', '/');
                    var signedPart = FindNestedPart(relativeName);
                    if (!signedPart.HasValue)
                    {
                        log.LogMessage(MessageImportance.Low, $"Didn't find signed part for nested file: {FileSignInfo.FullPath} -> {relativeName}");
                        continue;
                    }
                    log.LogMessage(MessageImportance.Low, $"Copying signed stream from {signedPart.Value.FileSignInfo.FullPath} to {file}.");
                    File.Copy(signedPart.Value.FileSignInfo.FullPath, file, true);
                }
 
                if (!BatchSignUtil.RunWixTool(createFileName, outputDir, workingDir, wixToolsPath, log))
                {
                    log.LogError($"Packaging of wix file '{FileSignInfo.FullPath}' failed");
                    return;
                }
 
                if (!File.Exists(outputFileName))
                {
                    log.LogError($"Wix tool execution passed, but output file '{outputFileName}' was not found.");
                    return;
                }
 
                log.LogMessage($"Created wix file {outputFileName}, replacing '{FileSignInfo.FullPath}' with '{outputFileName}'");
                File.Copy(outputFileName, FileSignInfo.FullPath, true);
            }
            finally
            {
                // Delete the intermediates
                Directory.Delete(workingDir, true);
                Directory.Delete(outputDir, true);
            }
        }
 
        internal static bool RunPkgProcess(TaskLoggingHelper log, string srcPath, string dstPath, string action, string pkgToolPath)
        {
            if (!RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
            {
                throw new Exception($"Pkg tooling is only support on MacOS.");
            }
 
            string args = $@"{action} ""{srcPath}""";
            
            if (action != "verify")
            {
                args += $@" ""{dstPath}""";
            }
 
            var process = Process.Start(new ProcessStartInfo()
            {
                FileName = "dotnet",
                Arguments = $@"exec ""{pkgToolPath}"" {args}",
                UseShellExecute = false,
                RedirectStandardError = true,
            });
 
            process.ErrorDataReceived += (sender, e) => log.LogError(e.Data);
 
            process.WaitForExit();
            return process.ExitCode == 0;
        }
 
        private static IEnumerable<(string relativePath, Stream content, long contentSize)> ReadPkgOrAppBundleEntries(TaskLoggingHelper log, string archivePath, string tempDir, string pkgToolPath, bool ignoreContent)
        {
            string extractDir = Path.Combine(tempDir, Guid.NewGuid().ToString());
            try
            {
                if (!RunPkgProcess(log, archivePath, extractDir, "unpack", pkgToolPath))
                {
                    throw new Exception($"Failed to unpack pkg {archivePath}");
                }
 
                foreach (var path in Directory.EnumerateFiles(extractDir, "*.*", SearchOption.AllDirectories))
                {
                    var relativePath = path.Substring(extractDir.Length + 1).Replace(Path.DirectorySeparatorChar, '/');
                    using var stream = ignoreContent ? null : (Stream)File.Open(path, FileMode.Open);
                    yield return (relativePath, stream, stream?.Length ?? 0);
                }
            }
            finally
            {
                if (Directory.Exists(extractDir))
                {
                    Directory.Delete(extractDir, recursive: true);
                }
            }
        }
 
        private void RepackPkgOrAppBundles(TaskLoggingHelper log, string tempDir, string pkgToolPath)
        {
            string extractDir = Path.Combine(tempDir, Guid.NewGuid().ToString());
            try
            {
                if (!RunPkgProcess(log, srcPath: FileSignInfo.FullPath, dstPath: extractDir, "unpack", pkgToolPath))
                {
                    return;
                }
 
                foreach (var path in Directory.EnumerateFiles(extractDir, "*.*", SearchOption.AllDirectories))
                {
                    var relativePath = path.Substring(extractDir.Length + 1).Replace(Path.DirectorySeparatorChar, '/');
 
                    var signedPart = FindNestedPart(relativePath);
                    if (!signedPart.HasValue)
                    {
                        log.LogMessage(MessageImportance.Low, $"Didn't find signed part for nested file: {FileSignInfo.FullPath} -> {relativePath}");
                        continue;
                    }
 
                    log.LogMessage(MessageImportance.Low, $"Copying signed stream from {signedPart.Value.FileSignInfo.FullPath} to {FileSignInfo.FullPath} -> {relativePath}.");
                    File.Copy(signedPart.Value.FileSignInfo.FullPath, path, overwrite: true);
                }
 
                if (!RunPkgProcess(log, srcPath: extractDir, dstPath: FileSignInfo.FullPath, "pack", pkgToolPath))
                {
                    return;
                }
            }
            finally
            {
                if (Directory.Exists(extractDir))
                {
                    Directory.Delete(extractDir, recursive: true);
                }
            }
        }
 
#if NETFRAMEWORK
        private static bool RunTarProcess(string srcPath, string dstPath, string tarToolPath)
        {
            var process = Process.Start(new ProcessStartInfo()
            {
                FileName = "dotnet",
                Arguments = $@"exec ""{tarToolPath}"" ""{srcPath}"" ""{dstPath}""",
                UseShellExecute = false
            });
 
            process.WaitForExit();
            return process.ExitCode == 0;
        }
 
        private static IEnumerable<(string relativePath, Stream content, long contentSize)> ReadTarGZipEntries(string archivePath, string tempDir, string tarToolPath, bool ignoreContent)
        {
            var extractDir = Path.Combine(tempDir, Guid.NewGuid().ToString());
            try
            {
                Directory.CreateDirectory(extractDir);
 
                if (!RunTarProcess(archivePath, extractDir, tarToolPath))
                {
                    throw new Exception($"Failed to unpack tar archive: {archivePath}");
                }
 
                foreach (var path in Directory.EnumerateFiles(extractDir, "*.*", SearchOption.AllDirectories))
                {
                    var relativePath = path.Substring(extractDir.Length + 1).Replace(Path.DirectorySeparatorChar, '/');
                    using var stream = ignoreContent  ? null : (Stream)File.Open(path, FileMode.Open);
                    yield return (relativePath, stream, stream?.Length ?? 0);
                }
            }
            finally
            {
                Directory.Delete(extractDir, recursive: true);
            }
        }
 
        private void RepackTarGZip(TaskLoggingHelper log, string tempDir, string tarToolPath)
        {
            var extractDir = Path.Combine(tempDir, Guid.NewGuid().ToString());
            try
            {
                Directory.CreateDirectory(extractDir);
 
                if (!RunTarProcess(srcPath: FileSignInfo.FullPath, dstPath: extractDir, tarToolPath))
                {
                    log.LogMessage(MessageImportance.Low, $"Failed to unpack tar archive: dotnet {tarToolPath} {FileSignInfo.FullPath}");
                    return;
                }
 
                foreach (var path in Directory.EnumerateFiles(extractDir, "*.*", SearchOption.AllDirectories))
                {
                    var relativePath = path.Substring(extractDir.Length + 1).Replace(Path.DirectorySeparatorChar, '/');
 
                    var signedPart = FindNestedPart(relativePath);
                    if (!signedPart.HasValue)
                    {
                        log.LogMessage(MessageImportance.Low, $"Didn't find signed part for nested file: {FileSignInfo.FullPath} -> {relativePath}");
                        continue;
                    }
 
                    log.LogMessage(MessageImportance.Low, $"Copying signed stream from {signedPart.Value.FileSignInfo.FullPath} to {FileSignInfo.FullPath} -> {relativePath}.");
                    File.Copy(signedPart.Value.FileSignInfo.FullPath, path, overwrite: true);
                }
 
                if (!RunTarProcess(srcPath: extractDir, dstPath: FileSignInfo.FullPath, tarToolPath))
                {
                    log.LogMessage(MessageImportance.Low, $"Failed to pack tar archive: dotnet {tarToolPath} {FileSignInfo.FullPath}");
                    return;
                }
            }
            finally
            {
                Directory.Delete(extractDir, recursive: true);
            }
        }
#else
        private void RepackTarGZip(TaskLoggingHelper log, string tempDir, string tarToolPath)
        {
            using MemoryStream streamToCompress = new();
            using (TarWriter writer = new(streamToCompress, leaveOpen: true))
            {
                foreach (TarEntry entry in ReadTarGZipEntries(FileSignInfo.FullPath))
                {
                    if (entry.DataStream != null)
                    {
                        string relativeName = entry.Name;
                        ZipPart? signedPart = FindNestedPart(relativeName);
 
                        if (signedPart.HasValue)
                        {
                            using FileStream signedStream = File.OpenRead(signedPart.Value.FileSignInfo.FullPath);
                            entry.DataStream = signedStream;
                            entry.DataStream.Position = 0;
                            writer.WriteEntry(entry);
 
                            log.LogMessage(MessageImportance.Low, $"Copying signed stream from {signedPart.Value.FileSignInfo.FullPath} to {FileSignInfo.FullPath} -> {relativeName}.");
                            continue;
                        }
 
                        log.LogMessage(MessageImportance.Low, $"Didn't find signed part for nested file: {FileSignInfo.FullPath} -> {relativeName}");
                    }
 
                    writer.WriteEntry(entry);
                }
            }
 
            streamToCompress.Position = 0;
            using (FileStream outputStream = File.Open(FileSignInfo.FullPath, FileMode.Truncate, FileAccess.Write))
            {
                using GZipStream compressor = new(outputStream, CompressionMode.Compress);
                streamToCompress.CopyTo(compressor);
            }
        }
 
        private static IEnumerable<TarEntry> ReadTarGZipEntries(string path)
        {
            using FileStream streamToDecompress = File.OpenRead(path);
            using GZipStream decompressor = new(streamToDecompress, CompressionMode.Decompress);
            using TarReader tarReader = new(decompressor);
            while (tarReader.GetNextEntry() is TarEntry entry)
            {
                yield return entry;
            }
        }
 
        /// <summary>
        /// Repack Deb container.
        /// </summary>
        private void RepackDebContainer(TaskLoggingHelper log, string tempDir)
        {
            // Data archive is the only expected nested part
            string dataArchive = NestedParts.Values.Single().FileSignInfo.FullPath;
 
            string controlArchive;
            try
            {
                controlArchive = GetUpdatedControlArchive(FileSignInfo.FullPath, dataArchive, tempDir);
            }
            catch(Exception e)
            {
                log.LogError(e.Message);
                return;
            }
 
            CreateDebPackage createDebPackageTask = new()
            {
                OutputDebPackagePath = FileSignInfo.FullPath,
                ControlFile = new TaskItem(controlArchive),
                DataFile = new TaskItem(dataArchive)
            };
 
            if (!createDebPackageTask.Execute())
            {
                log.LogError($"Failed to create new DEB package: {FileSignInfo.FileName}");
            }
        }
 
        /// <summary>
        /// Updates checksums of updated data file contents and updates the
        /// control file with the new checksums and new install size.
        /// </summary>
        /// <param name="debianPackage"></param>
        /// <param name="dataArchive"></param>
        /// <param name="tempDir"></param>
        /// <returns></returns>
        /// <exception cref="Exception"></exception>
        private string GetUpdatedControlArchive(string debianPackage, string dataArchive, string tempDir)
        {
            var workingDirGuidSegment = Guid.NewGuid().ToString().Split('-')[0];
 
            string workingDir = Path.Combine(tempDir, "work", workingDirGuidSegment);
            string controlLayout = Path.Combine(workingDir, "control");
            string dataLayout = Path.Combine(workingDir, "data");
 
            Directory.CreateDirectory(controlLayout);
            Directory.CreateDirectory(dataLayout);
 
            // Get the original control archive - to reuse package metadata and scripts
            var (relativePath, content, contentSize) = ReadDebContainerEntries(debianPackage, "control.tar").Single();
            string controlArchive = Path.Combine(workingDir, relativePath);
            File.WriteAllBytes(controlArchive, ((MemoryStream)content).ToArray());
 
            ExtractTarballContents(dataArchive, dataLayout);
            ExtractTarballContents(controlArchive, controlLayout);
 
            string sumsFile = Path.Combine(workingDir, "md5sums");
            CreateMD5SumsFile createMD5SumsFileTask = new()
            {
                OutputFile = sumsFile,
                RootDirectory = dataLayout,
                Files = Directory.GetFiles(dataLayout, "*", SearchOption.AllDirectories)
                            .Select(f => new TaskItem(f)).ToArray()
            };
 
            if (!createMD5SumsFileTask.Execute())
            {
                throw new Exception($"Failed to create MD5 checksums file for: {FileSignInfo.FileName}");
            }
 
            File.Copy(sumsFile, Path.Combine(controlLayout, "md5sums"), overwrite: true);
 
            // Update the optional Installed-Size field in control file
            string controlFile = Path.Combine(controlLayout, "control");
            string fileContents = File.ReadAllText(controlFile);
            string stringToFind = "Installed-Size: ";
            int index = fileContents.IndexOf(stringToFind);
            if (index != -1)
            {
                int end = fileContents.IndexOf('\n', index);
                fileContents = fileContents.Replace(fileContents.Substring(index, end - index), stringToFind + createMD5SumsFileTask.InstalledSize);
                File.WriteAllText(controlFile, fileContents);
            }
 
            // Repack the control tarball
            using (var dstStream = File.Open(controlArchive, FileMode.Create))
            {
                using var gzip = new GZipStream(dstStream, CompressionMode.Compress);
                TarFile.CreateFromDirectory(controlLayout, gzip, includeBaseDirectory: false);
            }
 
            return controlArchive;
        }
 
        internal static void ExtractTarballContents(string file, string destination, bool skipSymlinks = true)
        {
            foreach (TarEntry tar in ReadTarGZipEntries(file))
            {
                if (tar.EntryType == TarEntryType.Directory ||
                    (skipSymlinks && tar.EntryType == TarEntryType.SymbolicLink))
                {
                    continue;
                }
 
                string outputPath = Path.Join(destination, tar.Name);
                Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!);
 
                using FileStream outputFileStream = File.Create(outputPath);
                tar.DataStream?.CopyTo(outputFileStream);
            }
        }
 
        internal static IEnumerable<(string relativePath, Stream content, long contentSize)> ReadDebContainerEntries(string archivePath, string match = null)
        {
            using var archive = new ArReader(File.OpenRead(archivePath), leaveOpen: false);
 
            while (archive.GetNextEntry() is ArEntry entry)
            {
                string relativePath = entry.Name; // lgtm [cs/zipslip] Archive from trusted source
 
                if (match == null || relativePath.StartsWith(match))
                {
                    yield return (relativePath, entry.DataStream, entry.DataStream.Length);
                }
            }
        }
#endif
    }
}