File: UnreliableL2Tests.cs
Web Access
Project: src\test\Libraries\Microsoft.Extensions.Caching.Hybrid.Tests\Microsoft.Extensions.Caching.Hybrid.Tests.csproj (Microsoft.Extensions.Caching.Hybrid.Tests)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
using System.Diagnostics.CodeAnalysis;
using Microsoft.Extensions.Caching.Distributed;
using Microsoft.Extensions.Caching.Hybrid.Internal;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Xunit.Abstractions;
 
namespace Microsoft.Extensions.Caching.Hybrid.Tests;
 
// validate HC stability when the L2 is unreliable
public class UnreliableL2Tests(ITestOutputHelper testLog)
{
    [Theory]
    [InlineData(BreakType.None)]
    [InlineData(BreakType.Synchronous, Log.IdCacheBackendWriteFailure)]
    [InlineData(BreakType.Asynchronous, Log.IdCacheBackendWriteFailure)]
    [InlineData(BreakType.AsynchronousYield, Log.IdCacheBackendWriteFailure)]
    [SuppressMessage("Usage", "VSTHRD003:Avoid awaiting foreign Tasks", Justification = "Intentional; tracking for out-of-band support only")]
    public async Task WriteFailureInvisible(BreakType writeBreak, params int[] errorIds)
    {
        using (GetServices(out var hc, out var l1, out var l2, out var log))
        using (log)
        {
            // normal behaviour when working fine
            var x = await hc.GetOrCreateAsync("x", NewGuid);
            Assert.Equal(x, await hc.GetOrCreateAsync("x", NewGuid));
            Assert.NotNull(l2.Tail.Get("x")); // exists
 
            l2.WriteBreak = writeBreak;
            var y = await hc.GetOrCreateAsync("y", NewGuid);
            Assert.Equal(y, await hc.GetOrCreateAsync("y", NewGuid));
            if (writeBreak == BreakType.None)
            {
                Assert.NotNull(l2.Tail.Get("y")); // exists
            }
            else
            {
                Assert.Null(l2.Tail.Get("y")); // does not exist
            }
 
            await l2.LastWrite; // allows out-of-band write to complete
            await Task.Delay(150); // even then: thread jitter can cause problems
 
            log.WriteTo(testLog);
            log.AssertErrors(errorIds);
        }
    }
 
    [Theory]
    [InlineData(BreakType.None)]
    [InlineData(BreakType.Synchronous, Log.IdCacheBackendReadFailure, Log.IdCacheBackendReadFailure)]
    [InlineData(BreakType.Asynchronous, Log.IdCacheBackendReadFailure, Log.IdCacheBackendReadFailure)]
    [InlineData(BreakType.AsynchronousYield, Log.IdCacheBackendReadFailure, Log.IdCacheBackendReadFailure)]
    public async Task ReadFailureInvisible(BreakType readBreak, params int[] errorIds)
    {
        using (GetServices(out var hc, out var l1, out var l2, out var log))
        using (log)
        {
            // create two new values via HC; this should go down to l2
            var x = await hc.GetOrCreateAsync("x", NewGuid);
            var y = await hc.GetOrCreateAsync("y", NewGuid);
 
            // this should be reliable and repeatable
            Assert.Equal(x, await hc.GetOrCreateAsync("x", NewGuid));
            Assert.Equal(y, await hc.GetOrCreateAsync("y", NewGuid));
 
            // even if we clean L1, causing new L2 fetches
            l1.Clear();
            Assert.Equal(x, await hc.GetOrCreateAsync("x", NewGuid));
            Assert.Equal(y, await hc.GetOrCreateAsync("y", NewGuid));
 
            // now we break L2 in some predictable way, *without* clearing L1 - the
            // values should still be available via L1
            l2.ReadBreak = readBreak;
            Assert.Equal(x, await hc.GetOrCreateAsync("x", NewGuid));
            Assert.Equal(y, await hc.GetOrCreateAsync("y", NewGuid));
 
            // but if we clear L1 to force L2 hits, we anticipate problems
            l1.Clear();
            if (readBreak == BreakType.None)
            {
                Assert.Equal(x, await hc.GetOrCreateAsync("x", NewGuid));
                Assert.Equal(y, await hc.GetOrCreateAsync("y", NewGuid));
            }
            else
            {
                // because L2 is unavailable and L1 is empty, we expect the callback
                // to be used again, generating new values
                var a = await hc.GetOrCreateAsync("x", NewGuid, NoL2Write);
                var b = await hc.GetOrCreateAsync("y", NewGuid, NoL2Write);
 
                Assert.NotEqual(x, a);
                Assert.NotEqual(y, b);
 
                // but those *new* values are at least reliable inside L1
                Assert.Equal(a, await hc.GetOrCreateAsync("x", NewGuid));
                Assert.Equal(b, await hc.GetOrCreateAsync("y", NewGuid));
            }
 
            log.WriteTo(testLog);
            log.AssertErrors(errorIds);
        }
    }
 
    private static HybridCacheEntryOptions NoL2Write { get; } = new HybridCacheEntryOptions { Flags = HybridCacheEntryFlags.DisableDistributedCacheWrite };
 
    public enum BreakType
    {
        None, // async API works correctly
        Synchronous, // async API faults directly rather than return a faulted task
        Asynchronous, // async API returns a completed asynchronous fault
        AsynchronousYield, // async API returns an incomplete asynchronous fault
    }
 
    private static ValueTask<Guid> NewGuid(CancellationToken cancellationToken) => new(Guid.NewGuid());
 
    private static IDisposable GetServices(out HybridCache hc, out MemoryCache l1,
        out UnreliableDistributedCache l2, out LogCollector log)
    {
        // we need an entirely separate MC for the dummy backend, not connected to our
        // "real" services
        var services = new ServiceCollection();
        services.AddDistributedMemoryCache();
        var backend = services.BuildServiceProvider().GetRequiredService<IDistributedCache>();
 
        // now create the "real" services
        l2 = new UnreliableDistributedCache(backend);
        var collector = new LogCollector();
        log = collector;
        services = new ServiceCollection();
        services.AddSingleton<IDistributedCache>(l2);
        services.AddHybridCache();
        services.AddLogging(options =>
        {
            options.ClearProviders();
            options.AddProvider(collector);
        });
        var lifetime = services.BuildServiceProvider();
        hc = lifetime.GetRequiredService<HybridCache>();
        l1 = Assert.IsType<MemoryCache>(lifetime.GetRequiredService<IMemoryCache>());
        return lifetime;
    }
 
    private sealed class UnreliableDistributedCache : IDistributedCache
    {
        public UnreliableDistributedCache(IDistributedCache tail)
        {
            Tail = tail;
        }
 
        public IDistributedCache Tail { get; }
        public BreakType ReadBreak { get; set; }
        public BreakType WriteBreak { get; set; }
 
        public Task LastWrite { get; private set; } = Task.CompletedTask;
 
        public byte[]? Get(string key) => throw new NotSupportedException(); // only async API in use
 
        public Task<byte[]?> GetAsync(string key, CancellationToken token = default)
            => TrackLast(ThrowIfBrokenAsync<byte[]?>(ReadBreak) ?? Tail.GetAsync(key, token));
 
        public void Refresh(string key) => throw new NotSupportedException(); // only async API in use
 
        public Task RefreshAsync(string key, CancellationToken token = default)
            => TrackLast(ThrowIfBrokenAsync(WriteBreak) ?? Tail.RefreshAsync(key, token));
 
        public void Remove(string key) => throw new NotSupportedException(); // only async API in use
 
        public Task RemoveAsync(string key, CancellationToken token = default)
            => TrackLast(ThrowIfBrokenAsync(WriteBreak) ?? Tail.RemoveAsync(key, token));
 
        public void Set(string key, byte[] value, DistributedCacheEntryOptions options) => throw new NotSupportedException(); // only async API in use
 
        public Task SetAsync(string key, byte[] value, DistributedCacheEntryOptions options, CancellationToken token = default)
            => TrackLast(ThrowIfBrokenAsync(WriteBreak) ?? Tail.SetAsync(key, value, options, token));
 
        [DoesNotReturn]
        private static void Throw() => throw new IOException("L2 offline");
 
        private static async Task<T> ThrowAsync<T>(bool yield)
        {
            if (yield)
            {
                await Task.Yield();
            }
 
            Throw();
            return default; // never reached
        }
 
        private static Task? ThrowIfBrokenAsync(BreakType breakType) => ThrowIfBrokenAsync<int>(breakType);
 
        [SuppressMessage("Critical Bug", "S4586:Non-async \"Task/Task<T>\" methods should not return null", Justification = "Intentional for propagation")]
        private static Task<T>? ThrowIfBrokenAsync<T>(BreakType breakType)
        {
            switch (breakType)
            {
                case BreakType.Asynchronous:
                    return ThrowAsync<T>(false);
                case BreakType.AsynchronousYield:
                    return ThrowAsync<T>(true);
                case BreakType.None:
                    return null;
                default:
                    // includes BreakType.Synchronous and anything unknown
                    Throw();
                    break;
            }
 
            return null;
        }
 
        [SuppressMessage("Usage", "VSTHRD003:Avoid awaiting foreign Tasks", Justification = "Intentional; tracking for out-of-band support only")]
        [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "We don't need the failure type - just the timing")]
        private static Task IgnoreFailure(Task task)
        {
            return task.Status == TaskStatus.RanToCompletion
                ? Task.CompletedTask : IgnoreAsync(task);
 
            static async Task IgnoreAsync(Task task)
            {
                try
                {
                    await task;
                }
                catch
                {
                    // we only care about the "when"; failure is fine
                }
            }
        }
 
        [SuppressMessage("Usage", "VSTHRD003:Avoid awaiting foreign Tasks", Justification = "Intentional; tracking for out-of-band support only")]
        private Task TrackLast(Task lastWrite)
        {
            LastWrite = IgnoreFailure(lastWrite);
            return lastWrite;
        }
 
        [SuppressMessage("Usage", "VSTHRD003:Avoid awaiting foreign Tasks", Justification = "Intentional; tracking for out-of-band support only")]
        private Task<T> TrackLast<T>(Task<T> lastWrite)
        {
            LastWrite = IgnoreFailure(lastWrite);
            return lastWrite;
        }
    }
}