File: TrackedDependencies\FlatTrackingData.cs
Web Access
Project: ..\..\..\src\Utilities\Microsoft.Build.Utilities.csproj (Microsoft.Build.Utilities.Core)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
#if FEATURE_FILE_TRACKER
 
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Resources;
using System.Text;
 
using Microsoft.Build.Framework;
using Microsoft.Build.Shared;
using Microsoft.Build.Shared.FileSystem;
 
#nullable disable
 
namespace Microsoft.Build.Utilities
{
    /// <summary>
    /// Class used to store and interrogate inputs and outputs recorded by tracking operations.
    /// </summary>
    public class FlatTrackingData
    {
#pragma warning disable format // region formatting is different in net7.0 and net472, and cannot be fixed for both
        #region Constants
        // The maximum number of outputs that should be logged, if more than this, then no outputs are logged
        private const int MaxLogCount = 100;
        #endregion
 
        #region Member Data
        // The .write. trackg log files
 
        // The tlog marker is used if the tracking data is empty
        // even if the tracked execution was successful
        private string _tlogMarker = string.Empty;
 
        // The TaskLoggingHelper that we log progress to
        private TaskLoggingHelper _log;
 
        // The oldest file that we have seen
        private DateTime _oldestFileTimeUtc = DateTime.MaxValue;
 
        // The newest file what we have seen
        private DateTime _newestFileTimeUtc = DateTime.MinValue;
 
        // Should rooting markers be treated as tracking entries
        private bool _treatRootMarkersAsEntries;
 
        // If we are not skipping missing files, what DateTime should they be given?
        private DateTime _missingFileTimeUtc = DateTime.MinValue;
 
        // The newest Tlog that we have seen
        private DateTime _newestTLogTimeUtc = DateTime.MinValue;
 
        // Cache of last write times
        private readonly IDictionary<string, DateTime> _lastWriteTimeUtcCache = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
 
        // The set of paths that contain files that are to be ignored during up to date check - these directories or their subdirectories
        private readonly List<string> _excludedInputPaths = new List<string>();
        #endregion
 
        #region Properties
 
        // The output dependency table
        internal Dictionary<string, DateTime> DependencyTable { get; private set; } = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
 
        /// <summary>
        /// Missing files have been detected in the TLog
        /// </summary>
        [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly", Justification = "Has shipped as public API, so we can't easily change it now. ")]
        public List<string> MissingFiles { get; set; } = new List<string>();
 
        /// <summary>
        /// The path for the oldest file we have seen
        /// </summary>
        public string OldestFileName { get; set; } = string.Empty;
 
        /// <summary>
        /// The time for the oldest file we have seen
        /// </summary>
        public DateTime OldestFileTime
        {
            get => _oldestFileTimeUtc.ToLocalTime();
            set => _oldestFileTimeUtc = value.ToUniversalTime();
        }
 
        /// <summary>
        /// The time for the oldest file we have seen
        /// </summary>
        public DateTime OldestFileTimeUtc
        {
            get => _oldestFileTimeUtc;
            set => _oldestFileTimeUtc = value.ToUniversalTime();
        }
 
        /// <summary>
        /// The path for the newest file we have seen
        /// </summary>
        public string NewestFileName { get; set; } = string.Empty;
 
        /// <summary>
        /// The time for the newest file we have seen
        /// </summary>
        public DateTime NewestFileTime
        {
            get => _newestFileTimeUtc.ToLocalTime();
            set => _newestFileTimeUtc = value.ToUniversalTime();
        }
 
        /// <summary>
        /// The time for the newest file we have seen
        /// </summary>
        public DateTime NewestFileTimeUtc
        {
            get => _newestFileTimeUtc;
            set => _newestFileTimeUtc = value.ToUniversalTime();
        }
 
        /// <summary>
        /// Should root markers in the TLog be treated as file accesses, or only as markers?
        /// </summary>
        public bool TreatRootMarkersAsEntries
        {
            get => _treatRootMarkersAsEntries;
            set => _treatRootMarkersAsEntries = value;
        }
 
        /// <summary>
        /// Should files in the TLog but no longer exist be skipped or recorded?
        /// </summary>
        public bool SkipMissingFiles { get; set; }
 
        /// <summary>
        /// The TLog files that back this structure
        /// </summary>
        [SuppressMessage("Microsoft.Performance", "CA1819:PropertiesShouldNotReturnArrays", Justification = "Has shipped as public API, so we can't easily change it now. ")]
        public ITaskItem[] TlogFiles { get; set; }
 
        /// <summary>
        /// The time of the newest Tlog
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public DateTime NewestTLogTime
        {
            get => _newestTLogTimeUtc.ToLocalTime();
            set => _newestTLogTimeUtc = value.ToUniversalTime();
        }
 
        /// <summary>
        /// The time of the newest Tlog
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public DateTime NewestTLogTimeUtc
        {
            get => _newestTLogTimeUtc;
            set => _newestTLogTimeUtc = value.ToUniversalTime();
        }
 
        /// <summary>
        /// The path of the newest TLog file
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public string NewestTLogFileName { get; set; } = string.Empty;
 
        /// <summary>
        /// Are all the TLogs that were passed to us actually available on disk?
        /// </summary>
        public bool TlogsAvailable { get; set; }
 
        #endregion
 
        #region Constructors
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .write. tlog files to interpret</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, DateTime missingFileTimeUtc) => InternalConstruct(null, tlogFiles, null, false, missingFileTimeUtc, null);
 
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .write. tlog files to interpret</param>
        /// <param name="tlogFilesToIgnore">The .tlog files to ignore</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, ITaskItem[] tlogFilesToIgnore, DateTime missingFileTimeUtc) => InternalConstruct(null, tlogFiles, tlogFilesToIgnore, false, missingFileTimeUtc, null);
 
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .tlog files to interpret</param>
        /// <param name="tlogFilesToIgnore">The .tlog files to ignore</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        /// <param name="excludedInputPaths">The set of paths that contain files that are to be ignored during up to date check, including any subdirectories.</param>
        /// <param name="sharedLastWriteTimeUtcCache">Cache to be used for all timestamp/exists comparisons, which can be shared between multiple FlatTrackingData instances.</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, ITaskItem[] tlogFilesToIgnore, DateTime missingFileTimeUtc, string[] excludedInputPaths, IDictionary<string, DateTime> sharedLastWriteTimeUtcCache)
        {
            if (sharedLastWriteTimeUtcCache != null)
            {
                _lastWriteTimeUtcCache = sharedLastWriteTimeUtcCache;
            }
 
            InternalConstruct(null, tlogFiles, tlogFilesToIgnore, false, missingFileTimeUtc, excludedInputPaths);
        }
 
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .tlog files to interpret</param>
        /// <param name="tlogFilesToIgnore">The .tlog files to ignore</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        /// <param name="excludedInputPaths">The set of paths that contain files that are to be ignored during up to date check, including any subdirectories.</param>
        /// <param name="sharedLastWriteTimeUtcCache">Cache to be used for all timestamp/exists comparisons, which can be shared between multiple FlatTrackingData instances.</param>
        /// <param name="treatRootMarkersAsEntries">Add root markers as inputs.</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, ITaskItem[] tlogFilesToIgnore, DateTime missingFileTimeUtc, string[] excludedInputPaths, IDictionary<string, DateTime> sharedLastWriteTimeUtcCache, bool treatRootMarkersAsEntries)
        {
            _treatRootMarkersAsEntries = treatRootMarkersAsEntries;
 
            if (sharedLastWriteTimeUtcCache != null)
            {
                _lastWriteTimeUtcCache = sharedLastWriteTimeUtcCache;
            }
 
            InternalConstruct(null, tlogFiles, tlogFilesToIgnore, false, missingFileTimeUtc, excludedInputPaths);
        }
 
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="ownerTask">The task that is using file tracker</param>
        /// <param name="tlogFiles">The tlog files to interpret</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        public FlatTrackingData(ITask ownerTask, ITaskItem[] tlogFiles, DateTime missingFileTimeUtc) => InternalConstruct(ownerTask, tlogFiles, null, false, missingFileTimeUtc, null);
 
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="tlogFiles">The .write. tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        public FlatTrackingData(ITaskItem[] tlogFiles, bool skipMissingFiles) => InternalConstruct(null, tlogFiles, null, skipMissingFiles, DateTime.MinValue, null);
 
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="ownerTask">The task that is using file tracker</param>
        /// <param name="tlogFiles">The tlog files to interpret</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        public FlatTrackingData(ITask ownerTask, ITaskItem[] tlogFiles, bool skipMissingFiles) => InternalConstruct(ownerTask, tlogFiles, null, skipMissingFiles, DateTime.MinValue, null);
 
        /// <summary>
        /// Internal constructor
        /// </summary>
        /// <param name="ownerTask">The task that is using file tracker</param>
        /// <param name="tlogFilesLocal">The local .tlog files.</param>
        /// <param name="tlogFilesToIgnore">The .tlog files to ignore</param>
        /// <param name="skipMissingFiles">Ignore files that do not exist on disk</param>
        /// <param name="missingFileTimeUtc">The DateTime that should be recorded for missing file.</param>
        /// <param name="excludedInputPaths">The set of paths that contain files that are to be ignored during up to date check</param>
        private void InternalConstruct(ITask ownerTask, ITaskItem[] tlogFilesLocal, ITaskItem[] tlogFilesToIgnore, bool skipMissingFiles, DateTime missingFileTimeUtc, string[] excludedInputPaths)
        {
            if (ownerTask != null)
            {
                _log = new TaskLoggingHelper(ownerTask)
                {
                    TaskResources = AssemblyResources.PrimaryResources,
                    HelpKeywordPrefix = "MSBuild."
                };
            }
 
            ITaskItem[] expandedTlogFiles = TrackedDependencies.ExpandWildcards(tlogFilesLocal);
 
            if (tlogFilesToIgnore != null)
            {
                ITaskItem[] expandedTlogFilesToIgnore = TrackedDependencies.ExpandWildcards(tlogFilesToIgnore);
 
                if (expandedTlogFilesToIgnore.Length > 0)
                {
                    var ignore = new HashSet<string>();
                    var remainingTlogFiles = new List<ITaskItem>();
 
                    foreach (ITaskItem tlogFileToIgnore in expandedTlogFilesToIgnore)
                    {
                        ignore.Add(tlogFileToIgnore.ItemSpec);
                    }
 
                    foreach (ITaskItem tlogFile in expandedTlogFiles)
                    {
                        if (!ignore.Contains(tlogFile.ItemSpec))
                        {
                            remainingTlogFiles.Add(tlogFile);
                        }
                    }
 
                    TlogFiles = remainingTlogFiles.ToArray();
                }
                else
                {
                    TlogFiles = expandedTlogFiles;
                }
            }
            else
            {
                TlogFiles = expandedTlogFiles;
            }
 
            // We have no TLog files on disk, create a TLog marker from the
            // TLogFiles ItemSpec so we can fabricate one if we need to
            // This becomes our "first" tlog, since on the very first run, no tlogs
            // will exist, and if a compaction has been run (as part of the initial up-to-date check) then this
            // marker tlog will be created as empty.
            if (TlogFiles == null || TlogFiles.Length == 0)
            {
                _tlogMarker = tlogFilesLocal[0].ItemSpec
                    .Replace("*", "1")
                    .Replace("?", "2");
            }
 
            if (excludedInputPaths != null)
            {
                // Assign our exclude paths to our lookup - and make sure that all recorded paths end in a slash so that
                // our "starts with" comparison doesn't pick up incomplete matches, such as C:\Foo matching C:\FooFile.txt
                foreach (string excludePath in excludedInputPaths)
                {
                    string fullexcludePath = FileUtilities.EnsureTrailingSlash(FileUtilities.NormalizePath(excludePath)).ToUpperInvariant();
                    _excludedInputPaths.Add(fullexcludePath);
                }
            }
 
            TlogsAvailable = TrackedDependencies.ItemsExist(TlogFiles);
            SkipMissingFiles = skipMissingFiles;
            _missingFileTimeUtc = missingFileTimeUtc.ToUniversalTime();
            if (TlogFiles != null)
            {
                // Read the TLogs into our internal structures
                ConstructFileTable();
            }
        }
        #endregion
 
        #region Methods
        /// <summary>
        /// Construct our dependency table for our source files
        /// </summary>
        private void ConstructFileTable()
        {
            string tLogRootingMarker;
            try
            {
                // construct a rooting marker from the tlog files
                tLogRootingMarker = DependencyTableCache.FormatNormalizedTlogRootingMarker(TlogFiles);
            }
            catch (ArgumentException e)
            {
                FileTracker.LogWarningWithCodeFromResources(_log, "Tracking_RebuildingDueToInvalidTLog", e.Message);
                return;
            }
            if (!TlogsAvailable)
            {
                lock (DependencyTableCache.DependencyTable)
                {
                    // The tracking logs are not available, they may have been deleted at some point.
                    // Be safe and remove any references from the cache.
                    DependencyTableCache.DependencyTable.Remove(tLogRootingMarker);
                }
                return;
            }
 
            DependencyTableCacheEntry cachedEntry;
 
            lock (DependencyTableCache.DependencyTable)
            {
                // Look in the dependency table cache to see if its available and up to date
                cachedEntry = DependencyTableCache.GetCachedEntry(tLogRootingMarker);
            }
 
            // We have an up to date cached entry
            if (cachedEntry != null)
            {
                DependencyTable = (Dictionary<string, DateTime>)cachedEntry.DependencyTable;
 
                // We may have stored the dependency table in the cache, but all the other information
                // (newest file time, number of missing files, etc.) has been reset to default.  Refresh
                // the data.
                UpdateFileEntryDetails();
 
                // Log information about what we're using
                FileTracker.LogMessageFromResources(_log, MessageImportance.Low, "Tracking_TrackingCached");
                foreach (ITaskItem tlogItem in cachedEntry.TlogFiles)
                {
                    FileTracker.LogMessage(_log, MessageImportance.Low, "\t{0}", tlogItem.ItemSpec);
                }
                return;
            }
 
            FileTracker.LogMessageFromResources(_log, MessageImportance.Low, "Tracking_TrackingLogs");
            // Now we need to construct the rest of the table from the TLOG files
            // If there are any errors in the tlogs, we want to warn, stop parsing tlogs, and empty
            // out the dependency table, essentially forcing a rebuild.
            bool encounteredInvalidTLogContents = false;
            string invalidTLogName = null;
            foreach (ITaskItem tlogFileName in TlogFiles)
            {
                try
                {
                    FileTracker.LogMessage(_log, MessageImportance.Low, "\t{0}", tlogFileName.ItemSpec);
 
                    DateTime tlogLastWriteTimeUtc = NativeMethodsShared.GetLastWriteFileUtcTime(tlogFileName.ItemSpec);
                    if (tlogLastWriteTimeUtc > _newestTLogTimeUtc)
                    {
                        _newestTLogTimeUtc = tlogLastWriteTimeUtc;
                        NewestTLogFileName = tlogFileName.ItemSpec;
                    }
 
                    using (StreamReader tlog = File.OpenText(tlogFileName.ItemSpec))
                    {
                        string tlogEntry = tlog.ReadLine();
 
                        while (tlogEntry != null)
                        {
                            if (tlogEntry.Length == 0) // empty lines are a sign that something has gone wrong
                            {
                                encounteredInvalidTLogContents = true;
                                invalidTLogName = tlogFileName.ItemSpec;
                                break;
                            }
                            // Preprocessing for the line entry
                            else if (tlogEntry[0] == '#') // a comment marker should be skipped
                            {
                                tlogEntry = tlog.ReadLine();
                                continue;
                            }
                            else if (tlogEntry[0] == '^' && TreatRootMarkersAsEntries && tlogEntry.IndexOf('|') < 0) // This is a rooting non composite record, and we should keep it
                            {
                                tlogEntry = tlogEntry.Substring(1);
 
                                if (tlogEntry.Length == 0)
                                {
                                    encounteredInvalidTLogContents = true;
                                    invalidTLogName = tlogFileName.ItemSpec;
                                    break;
                                }
                            }
                            else if (tlogEntry[0] == '^') // root marker is not being treated as an entry, skip it
                            {
                                tlogEntry = tlog.ReadLine();
                                continue;
                            }
 
                            // If we haven't seen this file before, then record it
                            if (!DependencyTable.ContainsKey(tlogEntry))
                            {
                                // It may be that this is one of the locations that we should ignore
                                if (!FileTracker.FileIsExcludedFromDependencies(tlogEntry))
                                {
                                    RecordEntryDetails(tlogEntry, true);
                                }
                            }
                            tlogEntry = tlog.ReadLine();
                        }
                    }
                }
                catch (Exception e) when (ExceptionHandling.IsIoRelatedException(e))
                {
                    FileTracker.LogWarningWithCodeFromResources(_log, "Tracking_RebuildingDueToInvalidTLog", e.Message);
                    break;
                }
 
                if (encounteredInvalidTLogContents)
                {
                    FileTracker.LogWarningWithCodeFromResources(_log, "Tracking_RebuildingDueToInvalidTLogContents", invalidTLogName);
                    break;
                }
            }
 
            lock (DependencyTableCache.DependencyTable)
            {
                // There were problems with the tracking logs -- we've already warned or errored; now we want to make
                // sure that we essentially force a rebuild of this particular root.
                if (encounteredInvalidTLogContents)
                {
                    DependencyTableCache.DependencyTable.Remove(tLogRootingMarker);
                    DependencyTable = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
                }
                else
                {
                    // Record the newly built dependency table in the cache
                    DependencyTableCache.DependencyTable[tLogRootingMarker] = new DependencyTableCacheEntry(TlogFiles, DependencyTable);
                }
            }
        }
 
        /// <summary>
        /// Update the current state of entry details for the dependency table
        /// </summary>
        public void UpdateFileEntryDetails()
        {
            OldestFileName = string.Empty;
            _oldestFileTimeUtc = DateTime.MaxValue;
 
            NewestFileName = string.Empty;
            _newestFileTimeUtc = DateTime.MinValue;
 
            NewestTLogFileName = string.Empty;
            _newestTLogTimeUtc = DateTime.MinValue;
 
            MissingFiles.Clear();
 
            // First update the details of our Tlogs
            foreach (ITaskItem tlogFileName in TlogFiles)
            {
                DateTime tlogLastWriteTimeUtc = NativeMethodsShared.GetLastWriteFileUtcTime(tlogFileName.ItemSpec);
                if (tlogLastWriteTimeUtc > _newestTLogTimeUtc)
                {
                    _newestTLogTimeUtc = tlogLastWriteTimeUtc;
                    NewestTLogFileName = tlogFileName.ItemSpec;
                }
            }
 
            // Now for each entry in the table
            foreach (string entry in DependencyTable.Keys)
            {
                RecordEntryDetails(entry, false);
            }
        }
 
        /// <summary>
        /// Test to see if the specified file is excluded from tracked dependency checking
        /// </summary>
        /// <param name="fileName">
        /// Full path of the file to test
        /// </param>
        /// <remarks>
        /// The file is excluded if it is within any of the specified excluded input paths or any subdirectory of the paths.
        /// It also assumes the file name is already converted to Uppercase Invariant.
        /// </remarks>
        public bool FileIsExcludedFromDependencyCheck(string fileName)
        {
            foreach (string path in _excludedInputPaths)
            {
                if (fileName.StartsWith(path, StringComparison.OrdinalIgnoreCase))
                {
                    return true;
                }
            }
 
            return false;
        }
 
        /// <summary>
        /// Record the time and missing state of the entry in the tlog
        /// </summary>
        private void RecordEntryDetails(string tlogEntry, bool populateTable)
        {
            if (FileIsExcludedFromDependencyCheck(tlogEntry))
            {
                return;
            }
 
            DateTime fileModifiedTimeUtc = GetLastWriteTimeUtc(tlogEntry);
            if (SkipMissingFiles && fileModifiedTimeUtc == DateTime.MinValue) // the file is missing
            {
                return;
            }
            else if (fileModifiedTimeUtc == DateTime.MinValue)
            {
                // Record the file in our table even though it was missing
                // use the missingFileTimeUtc as indicated.
                if (populateTable)
                {
                    DependencyTable[tlogEntry] = _missingFileTimeUtc.ToUniversalTime();
                }
                MissingFiles.Add(tlogEntry);
            }
            else
            {
                if (populateTable)
                {
                    DependencyTable[tlogEntry] = fileModifiedTimeUtc;
                }
            }
 
            // Record this file if it is newer than our current newest
            if (fileModifiedTimeUtc > _newestFileTimeUtc)
            {
                _newestFileTimeUtc = fileModifiedTimeUtc;
                NewestFileName = tlogEntry;
            }
 
            // Record this file if it is older than our current oldest
            if (fileModifiedTimeUtc < _oldestFileTimeUtc)
            {
                _oldestFileTimeUtc = fileModifiedTimeUtc;
                OldestFileName = tlogEntry;
            }
        }
 
        /// <summary>
        /// This method will re-write the tlogs from the output table
        /// </summary>
        public void SaveTlog() => SaveTlog(null);
 
        /// <summary>
        /// This method will re-write the tlogs from the current table
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public void SaveTlog(DependencyFilter includeInTLog)
        {
            if (TlogFiles?.Length > 0)
            {
                string tLogRootingMarker = DependencyTableCache.FormatNormalizedTlogRootingMarker(TlogFiles);
 
                lock (DependencyTableCache.DependencyTable)
                {
                    // The tracking logs in the cache will be invalidated by this write
                    // remove the cached entries to be sure
                    DependencyTableCache.DependencyTable.Remove(tLogRootingMarker);
                }
 
                string firstTlog = TlogFiles[0].ItemSpec;
 
                // empty all tlogs
                foreach (ITaskItem tlogFile in TlogFiles)
                {
                    File.WriteAllText(tlogFile.ItemSpec, "", Encoding.Unicode);
                }
 
                // Write out the dependency information as a new tlog
                using (StreamWriter newTlog = FileUtilities.OpenWrite(firstTlog, false, Encoding.Unicode))
                {
                    foreach (string fileEntry in DependencyTable.Keys)
                    {
                        // Give the task a chance to filter dependencies out of the written TLog
                        if (includeInTLog == null || includeInTLog(fileEntry))
                        {
                            // Write out the entry
                            newTlog.WriteLine(fileEntry);
                        }
                    }
                }
            }
            else if (_tlogMarker != string.Empty)
            {
                string markerDirectory = Path.GetDirectoryName(_tlogMarker);
                if (!FileSystems.Default.DirectoryExists(markerDirectory))
                {
                    Directory.CreateDirectory(markerDirectory);
                }
 
                // There were no TLogs to save, so use the TLog marker
                // to create a marker file that can be used for up-to-date check.
                File.WriteAllText(_tlogMarker, "");
            }
        }
 
        /// <summary>
        /// Returns cached value for last write time of file. Update the cache if it is the first
        /// time someone asking for that file
        /// </summary>
        public DateTime GetLastWriteTimeUtc(string file)
        {
            if (!_lastWriteTimeUtcCache.TryGetValue(file, out DateTime fileModifiedTimeUtc))
            {
                fileModifiedTimeUtc = NativeMethodsShared.GetLastWriteFileUtcTime(file);
                _lastWriteTimeUtcCache[file] = fileModifiedTimeUtc;
            }
 
            return fileModifiedTimeUtc;
        }
 
        #endregion
 
        #region Static Methods
 
        /// <summary>
        /// Checks to see if the tracking data indicates that everything is up to date according to UpToDateCheckType.
        /// Note: If things are not up to date, then the TLogs are compacted to remove all entries in preparation to
        /// re-track execution of work.
        /// </summary>
        /// <param name="hostTask">The <see cref="Task"/> host</param>
        /// <param name="upToDateCheckType">UpToDateCheckType</param>
        /// <param name="readTLogNames">The array of read tlogs</param>
        /// <param name="writeTLogNames">The array of write tlogs</param>
        /// <returns></returns>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public static bool IsUpToDate(Task hostTask, UpToDateCheckType upToDateCheckType, ITaskItem[] readTLogNames, ITaskItem[] writeTLogNames)
        {
            // Read the input graph (missing inputs are infinitely new - i.e. outputs are out of date)
            FlatTrackingData inputs = new FlatTrackingData(hostTask, readTLogNames, DateTime.MaxValue);
 
            // Read the output graph (missing outputs are infinitely old - i.e. outputs are out of date)
            FlatTrackingData outputs = new FlatTrackingData(hostTask, writeTLogNames, DateTime.MinValue);
 
            // Find out if we are up to date
            bool isUpToDate = IsUpToDate(hostTask.Log, upToDateCheckType, inputs, outputs);
 
            // We're going to execute, so clear out the tlogs so
            // the new execution will correctly populate the tlogs a-new
            if (!isUpToDate)
            {
                // Remove all from inputs tlog
                inputs.DependencyTable.Clear();
                inputs.SaveTlog();
 
                // Remove all from outputs tlog
                outputs.DependencyTable.Clear();
                outputs.SaveTlog();
            }
            return isUpToDate;
        }
 
        /// <summary>
        /// Simple check of up to date state according to the tracking data and the UpToDateCheckType.
        /// Note: No tracking log compaction will take place when using this overload
        /// </summary>
        /// <param name="Log">TaskLoggingHelper from the host task</param>
        /// <param name="upToDateCheckType">UpToDateCheckType to use</param>
        /// <param name="inputs">FlatTrackingData structure containing the inputs</param>
        /// <param name="outputs">FlatTrackingData structure containing the outputs</param>
        /// <returns></returns>
        [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Log", Justification = "Has shipped as public API; plus it is a closer match to other locations in our codebase where 'Log' is a property and cased properly")]
        public static bool IsUpToDate(TaskLoggingHelper Log, UpToDateCheckType upToDateCheckType, FlatTrackingData inputs, FlatTrackingData outputs)
        {
            bool isUpToDate = false;
            // Keep a record of the task resources that was in use before
            ResourceManager taskResources = Log.TaskResources;
 
            Log.TaskResources = AssemblyResources.PrimaryResources;
 
            inputs.UpdateFileEntryDetails();
            outputs.UpdateFileEntryDetails();
 
            if (!inputs.TlogsAvailable || !outputs.TlogsAvailable || inputs.DependencyTable.Count == 0)
            {
                // 1) The TLogs are somehow missing, which means we need to build
                // 2) Because we are flat tracking, there are no roots which means that all the input file information
                //    comes from the input Tlogs, if they are empty then we must build.
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_LogFilesNotAvailable");
            }
            else if (inputs.MissingFiles.Count > 0 || outputs.MissingFiles.Count > 0)
            {
                // Files are missing from either inputs or outputs, that means we need to build
 
                // Files are missing from inputs, that means we need to build
                if (inputs.MissingFiles.Count > 0)
                {
                    Log.LogMessageFromResources(MessageImportance.Low, "Tracking_MissingInputs");
                }
                // Too much logging leads to poor performance
                if (inputs.MissingFiles.Count > MaxLogCount)
                {
                    FileTracker.LogMessageFromResources(Log, MessageImportance.Low, "Tracking_InputsNotShown", inputs.MissingFiles.Count);
                }
                else
                {
                    // We have our set of inputs, log the details
                    foreach (string input in inputs.MissingFiles)
                    {
                        FileTracker.LogMessage(Log, MessageImportance.Low, "\t" + input);
                    }
                }
 
                // Files are missing from outputs, that means we need to build
                if (outputs.MissingFiles.Count > 0)
                {
                    Log.LogMessageFromResources(MessageImportance.Low, "Tracking_MissingOutputs");
                }
                // Too much logging leads to poor performance
                if (outputs.MissingFiles.Count > MaxLogCount)
                {
                    FileTracker.LogMessageFromResources(Log, MessageImportance.Low, "Tracking_OutputsNotShown", outputs.MissingFiles.Count);
                }
                else
                {
                    // We have our set of inputs, log the details
                    foreach (string output in outputs.MissingFiles)
                    {
                        FileTracker.LogMessage(Log, MessageImportance.Low, "\t" + output);
                    }
                }
            }
            else if (upToDateCheckType == UpToDateCheckType.InputOrOutputNewerThanTracking &&
                    inputs.NewestFileTimeUtc > inputs.NewestTLogTimeUtc)
            {
                // One of the inputs is newer than the input tlog
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", inputs.NewestFileName, inputs.NewestFileTimeUtc, inputs.NewestTLogFileName, inputs.NewestTLogTimeUtc);
            }
            else if (upToDateCheckType == UpToDateCheckType.InputOrOutputNewerThanTracking &&
                    outputs.NewestFileTimeUtc > outputs.NewestTLogTimeUtc)
            {
                // one of the outputs is newer than the output tlog
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", outputs.NewestFileName, outputs.NewestFileTimeUtc, outputs.NewestTLogFileName, outputs.NewestTLogTimeUtc);
            }
            else if (upToDateCheckType == UpToDateCheckType.InputNewerThanOutput &&
                    inputs.NewestFileTimeUtc > outputs.NewestFileTimeUtc)
            {
                // One of the inputs is newer than the outputs
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", inputs.NewestFileName, inputs.NewestFileTimeUtc, outputs.NewestFileName, outputs.NewestFileTimeUtc);
            }
            else if (upToDateCheckType == UpToDateCheckType.InputNewerThanTracking &&
                    inputs.NewestFileTimeUtc > inputs.NewestTLogTimeUtc)
            {
                // One of the inputs is newer than the one of the TLogs
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", inputs.NewestFileName, inputs.NewestFileTimeUtc, inputs.NewestTLogFileName, inputs.NewestTLogTimeUtc);
            }
            else if (upToDateCheckType == UpToDateCheckType.InputNewerThanTracking &&
                    inputs.NewestFileTimeUtc > outputs.NewestTLogTimeUtc)
            {
                // One of the inputs is newer than the one of the TLogs
                Log.LogMessageFromResources(MessageImportance.Low, "Tracking_DependencyWasModifiedAt", inputs.NewestFileName, inputs.NewestFileTimeUtc, outputs.NewestTLogFileName, outputs.NewestTLogTimeUtc);
            }
            else
            {
                // Nothing appears to have changed..
                isUpToDate = true;
                Log.LogMessageFromResources(MessageImportance.Normal, "Tracking_UpToDate");
            }
 
            // Set the task resources back now that we're done with it
            Log.TaskResources = taskResources;
 
            return isUpToDate;
        }
 
        /// <summary>
        /// Once tracked operations have been completed then we need to compact / finalize the Tlogs based
        /// on the success of the tracked execution. If it fails, then we clean out the TLogs. If it succeeds
        /// then we clean temporary files from the TLogs and re-write them.
        /// </summary>
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLogs", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", MessageId = "TLog", Justification = "Has now shipped as public API; plus it's unclear whether 'Tlog' or 'TLog' is the preferred casing")]
        public static void FinalizeTLogs(bool trackedOperationsSucceeded, ITaskItem[] readTLogNames, ITaskItem[] writeTLogNames, ITaskItem[] trackedFilesToRemoveFromTLogs)
        {
            // Read the input table, skipping missing files
            FlatTrackingData inputs = new FlatTrackingData(readTLogNames, true);
 
            // Read the output table, skipping missing files
            FlatTrackingData outputs = new FlatTrackingData(writeTLogNames, true);
 
            // If we failed we need to clean the Tlogs
            if (!trackedOperationsSucceeded)
            {
                // If the tool errors in some way, we assume that any and all inputs and outputs it wrote during
                // execution are wrong. So we compact the read and write tlogs to remove the entries for the
                // set of sources being compiled - the next incremental build will find no entries
                // and correctly cause the sources to be compiled
                // Remove all from inputs tlog
                inputs.DependencyTable.Clear();
                inputs.SaveTlog();
 
                // Remove all from outputs tlog
                outputs.DependencyTable.Clear();
                outputs.SaveTlog();
            }
            else
            {
                // If all went well with the tool execution, then compact the tlogs
                // to remove any files that are no longer on disk.
                // This removes any temporary files from the dependency graph
 
                // In addition to temporary file removal, an optional set of files to remove may be been supplied
 
                if (trackedFilesToRemoveFromTLogs?.Length > 0)
                {
                    IDictionary<string, ITaskItem> trackedFilesToRemove = new Dictionary<string, ITaskItem>(StringComparer.OrdinalIgnoreCase);
 
                    foreach (ITaskItem removeFile in trackedFilesToRemoveFromTLogs)
                    {
                        trackedFilesToRemove.Add(FileUtilities.NormalizePath(removeFile.ItemSpec), removeFile);
                    }
 
                    // UNDONE: If necessary we could have two independent sets of "ignore" files, one for inputs and one for outputs
                    // Use an anonymous methods to encapsulate the contains check for the input and output tlogs
                    // We need to answer the question "should fullTrackedPath be included in the TLog?"
                    outputs.SaveTlog(fullTrackedPath => !trackedFilesToRemove.ContainsKey(fullTrackedPath));
                    inputs.SaveTlog(fullTrackedPath => !trackedFilesToRemove.ContainsKey(fullTrackedPath));
                }
                else
                {
                    // Compact the write tlog
                    outputs.SaveTlog();
 
                    // Compact the read tlog
                    inputs.SaveTlog();
                }
            }
        }
        #endregion
#pragma warning restore format
    }
 
    /// <summary>
    /// The possible types of up to date check that we can support
    /// </summary>
    public enum UpToDateCheckType
    {
        /// <summary>
        /// The input is newer than the output.
        /// </summary>
        InputNewerThanOutput,
        /// <summary>
        /// The input or output are newer than the tracking file.
        /// </summary>
        InputOrOutputNewerThanTracking,
        /// <summary>
        /// The input is newer than the tracking file.
        /// </summary>
        InputNewerThanTracking
    }
}
 
#endif