Merge branch 'claudedo/87105f5ec4f44af4ae6089cd2e153e3c'

tests/ClaudeDo.Worker.Tests/Services/QueueServiceTests.cs

@@ -50,7 +50,8 @@ public sealed class QueueServiceTests : IDisposable
 
     private (QueueService service, FakeClaudeProcess fakeProcess) CreateService(
         Func<string, string, IReadOnlyList<string>, Func<string, Task>, CancellationToken, Task<RunResult>>? handler = null,
-        FakeUsageGate? usageGate = null)
+        FakeUsageGate? usageGate = null,
+        UsageState? usageState = null)
     {
         var fake = new FakeClaudeProcess(handler);
         _hubContext = new CapturingHubContext();
@@ -67,10 +68,24 @@ public sealed class QueueServiceTests : IDisposable
         var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels);
         _usageGate = usageGate ?? new FakeUsageGate();
         var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, _waker, picker, overrideSlot, state, built.RunCancels,
-            _usageGate, new UsageState(), broadcaster);
+            _usageGate, usageState ?? new UsageState(), broadcaster);
         return (service, fake);
     }
 
+    private async Task SetAppSettingsAsync(
+        int maxParallel, int softPct = 50, int hardPct = 65, int gateFive = 80, int gateSeven = 90)
+    {
+        using var ctx = _db.CreateContext();
+        var repo = new AppSettingsRepository(ctx);
+        var settings = await repo.GetAsync();
+        settings.MaxParallelExecutions = maxParallel;
+        settings.UsageThrottleSoftPct = softPct;
+        settings.UsageThrottleHardPct = hardPct;
+        settings.UsageGateFiveHourPct = gateFive;
+        settings.UsageGateSevenDayPct = gateSeven;
+        await repo.UpdateAsync(settings);
+    }
+
     private async Task<string> SeedListAsync()
     {
         var listId = Guid.NewGuid().ToString();
@@ -452,4 +467,154 @@ public sealed class QueueServiceTests : IDisposable
             .Count(c => c.Method == "WorkerLog" && (WorkerLogLevel)c.Args[1]! == WorkerLogLevel.Warn);
         Assert.Equal(1, warnCalls);
     }
+
+    // Polls until `read()` reaches `expected` (or times out), then waits a further grace period
+    // to make sure the count doesn't keep climbing past it — needed because slot fills happen
+    // concurrently and a fixed sleep is either flaky (too short) or slow (too long).
+    private static async Task AssertStableCountAsync(Func<int> read, int expected)
+    {
+        var deadline = DateTime.UtcNow.AddSeconds(5);
+        while (read() < expected && DateTime.UtcNow < deadline)
+            await Task.Delay(20);
+
+        await Task.Delay(250);
+        Assert.Equal(expected, read());
+    }
+
+    [Fact]
+    public async Task Throttle_StepsDownEffectiveSlots_BelowConfiguredMax()
+    {
+        var listId = await SeedListAsync();
+        await SeedQueuedTask(listId);
+        await SeedQueuedTask(listId);
+        await SeedQueuedTask(listId);
+
+        await SetAppSettingsAsync(maxParallel: 3);
+
+        var usageState = new UsageState();
+        usageState.ReportSuccess(new UsageSnapshot(
+            new UsageBucket(60, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
+
+        var startedCount = 0;
+        var block = new TaskCompletionSource();
+        var (service, _) = CreateService(async (_, _, _, _, _) =>
+        {
+            Interlocked.Increment(ref startedCount);
+            await block.Task;
+            return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
+        }, usageState: usageState);
+
+        using var cts = new CancellationTokenSource();
+        await service.StartAsync(cts.Token);
+        _waker.Wake();
+
+        // 60% is between the soft (50) and hard (65) thresholds — capped at 2 slots even
+        // though 3 are configured and 3 tasks are queued.
+        await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 2);
+
+        block.SetResult();
+        cts.Cancel();
+    }
+
+    [Fact]
+    public async Task Throttle_AtHardThreshold_CapsToOneSlot()
+    {
+        var listId = await SeedListAsync();
+        await SeedQueuedTask(listId);
+        await SeedQueuedTask(listId);
+
+        await SetAppSettingsAsync(maxParallel: 3);
+
+        var usageState = new UsageState();
+        usageState.ReportSuccess(new UsageSnapshot(
+            new UsageBucket(70, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
+
+        var startedCount = 0;
+        var block = new TaskCompletionSource();
+        var (service, _) = CreateService(async (_, _, _, _, _) =>
+        {
+            Interlocked.Increment(ref startedCount);
+            await block.Task;
+            return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
+        }, usageState: usageState);
+
+        using var cts = new CancellationTokenSource();
+        await service.StartAsync(cts.Token);
+        _waker.Wake();
+
+        await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 1);
+
+        block.SetResult();
+        cts.Cancel();
+    }
+
+    [Fact]
+    public async Task NoUsageSnapshot_FallsBackToFullConfiguredParallelism()
+    {
+        var listId = await SeedListAsync();
+        await SeedQueuedTask(listId);
+        await SeedQueuedTask(listId);
+        await SeedQueuedTask(listId);
+
+        await SetAppSettingsAsync(maxParallel: 3);
+
+        // No snapshot has landed yet (fresh UsageState) — throttle must fail open.
+        var startedCount = 0;
+        var block = new TaskCompletionSource();
+        var (service, _) = CreateService(async (_, _, _, _, _) =>
+        {
+            Interlocked.Increment(ref startedCount);
+            await block.Task;
+            return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
+        }, usageState: new UsageState());
+
+        using var cts = new CancellationTokenSource();
+        await service.StartAsync(cts.Token);
+        _waker.Wake();
+
+        await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 3);
+
+        block.SetResult();
+        cts.Cancel();
+    }
+
+    [Fact]
+    public async Task Throttle_Engaging_Does_Not_Cancel_AlreadyRunning_Slot()
+    {
+        var listId = await SeedListAsync();
+        await SeedQueuedTask(listId);
+
+        await SetAppSettingsAsync(maxParallel: 3);
+        var usageState = new UsageState();
+
+        var running = new TaskCompletionSource();
+        var cancelled = false;
+        var (service, _) = CreateService(async (_, _, _, _, ct) =>
+        {
+            running.SetResult();
+            try
+            {
+                await Task.Delay(Timeout.Infinite, ct);
+            }
+            catch (OperationCanceledException)
+            {
+                cancelled = true;
+                throw;
+            }
+            return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
+        }, usageState: usageState);
+
+        using var cts = new CancellationTokenSource();
+        await service.StartAsync(cts.Token);
+        _waker.Wake();
+        await running.Task.WaitAsync(TimeSpan.FromSeconds(5));
+
+        // Throttle engages hard after the slot is already running — several backstop ticks pass.
+        usageState.ReportSuccess(new UsageSnapshot(
+            new UsageBucket(70, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
+        await Task.Delay(200);
+
+        Assert.False(cancelled);
+        cts.Cancel();
+    }
 }

tests/ClaudeDo.Worker.Tests/Usage/UsageSnapshotBuilderTests.cs

@@ -142,4 +142,86 @@ public sealed class UsageSnapshotBuilderTests : IDisposable
         Assert.True(dto.IsStale);
         Assert.Equal("boom", dto.LastError);
     }
+
+    private async Task SetThrottleAsync(int maxParallel, int softPct, int hardPct)
+    {
+        using var ctx = _db.CreateContext();
+        var repo = new AppSettingsRepository(ctx);
+        var settings = await repo.GetAsync();
+        settings.MaxParallelExecutions = maxParallel;
+        settings.UsageThrottleSoftPct = softPct;
+        settings.UsageThrottleHardPct = hardPct;
+        await repo.UpdateAsync(settings);
+    }
+
+    [Fact]
+    public async Task Throttled_slots_and_decisive_bucket_reported()
+    {
+        await SetThresholdsAsync(80, 90);
+        await SetThrottleAsync(maxParallel: 3, softPct: 50, hardPct: 65);
+
+        var state = new UsageState();
+        state.ReportSuccess(new UsageSnapshot(
+            new UsageBucket(70, null), new UsageBucket(20, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
+
+        var builder = CreateBuilder(state, new UsageGateDecision(false, null));
+
+        var dto = await builder.BuildAsync();
+
+        Assert.Equal(3, dto.ConfiguredSlots);
+        Assert.Equal(1, dto.EffectiveSlots);
+        Assert.Equal("five_hour", dto.ThrottleBucket);
+    }
+
+    [Fact]
+    public async Task SevenDay_decisive_bucket_reported_when_higher()
+    {
+        await SetThresholdsAsync(80, 90);
+        await SetThrottleAsync(maxParallel: 3, softPct: 50, hardPct: 65);
+
+        var state = new UsageState();
+        state.ReportSuccess(new UsageSnapshot(
+            new UsageBucket(10, null), new UsageBucket(70, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
+
+        var builder = CreateBuilder(state, new UsageGateDecision(false, null));
+
+        var dto = await builder.BuildAsync();
+
+        Assert.Equal("seven_day", dto.ThrottleBucket);
+    }
+
+    [Fact]
+    public async Task Not_throttled_reports_null_bucket_and_equal_slots()
+    {
+        await SetThresholdsAsync(80, 90);
+        await SetThrottleAsync(maxParallel: 3, softPct: 50, hardPct: 65);
+
+        var state = new UsageState();
+        state.ReportSuccess(new UsageSnapshot(
+            new UsageBucket(10, null), new UsageBucket(20, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
+
+        var builder = CreateBuilder(state, new UsageGateDecision(false, null));
+
+        var dto = await builder.BuildAsync();
+
+        Assert.Equal(3, dto.ConfiguredSlots);
+        Assert.Equal(3, dto.EffectiveSlots);
+        Assert.Null(dto.ThrottleBucket);
+    }
+
+    [Fact]
+    public async Task No_snapshot_falls_back_to_configured_slots_without_throttling()
+    {
+        await SetThresholdsAsync(80, 90);
+        await SetThrottleAsync(maxParallel: 3, softPct: 50, hardPct: 65);
+
+        var state = new UsageState();
+        var builder = CreateBuilder(state, new UsageGateDecision(false, null));
+
+        var dto = await builder.BuildAsync();
+
+        Assert.Equal(3, dto.ConfiguredSlots);
+        Assert.Equal(3, dto.EffectiveSlots);
+        Assert.Null(dto.ThrottleBucket);
+    }
 }

tests/ClaudeDo.Worker.Tests/Usage/UsageThrottleTests.cs

@@ -0,0 +1,111 @@
+using ClaudeDo.Worker.Usage;
+
+namespace ClaudeDo.Worker.Tests.Usage;
+
+public sealed class UsageThrottleTests
+{
+    private const int Soft = 50;
+    private const int Hard = 65;
+    private const int GateFive = 80;
+    private const int GateSeven = 90;
+
+    private static int Effective(double? five, double? seven, int configured = 3) =>
+        UsageThrottle.EffectiveSlots(configured, five, seven, Soft, Hard, GateFive, GateSeven);
+
+    [Fact]
+    public void BelowSoftThreshold_ReturnsFullConfiguredSlots()
+    {
+        Assert.Equal(3, Effective(30, 40));
+    }
+
+    [Fact]
+    public void AtSoftThreshold_CapsAtTwo()
+    {
+        Assert.Equal(2, Effective(50, 10));
+    }
+
+    [Fact]
+    public void BetweenSoftAndHard_CapsAtTwo()
+    {
+        Assert.Equal(2, Effective(60, 0));
+    }
+
+    [Fact]
+    public void AtHardThreshold_CapsAtOne()
+    {
+        Assert.Equal(1, Effective(65, 0));
+    }
+
+    [Fact]
+    public void BetweenHardAndGate_CapsAtOne()
+    {
+        Assert.Equal(1, Effective(70, 0));
+    }
+
+    [Fact]
+    public void AtGateThreshold_ReturnsZero()
+    {
+        Assert.Equal(0, Effective(80, 0));
+    }
+
+    [Fact]
+    public void AboveGateThreshold_ReturnsZero()
+    {
+        Assert.Equal(0, Effective(95, 0));
+    }
+
+    [Fact]
+    public void FiveHourAloneDecisive_SevenDayLow()
+    {
+        Assert.Equal(1, Effective(70, 5));
+    }
+
+    [Fact]
+    public void SevenDayAloneDecisive_FiveHourLow()
+    {
+        Assert.Equal(1, Effective(5, 70));
+    }
+
+    [Fact]
+    public void SevenDayAloneAtGate_ReturnsZero()
+    {
+        Assert.Equal(0, Effective(10, 90));
+    }
+
+    [Fact]
+    public void FiveHourAloneAtGate_ReturnsZero()
+    {
+        Assert.Equal(0, Effective(80, 10));
+    }
+
+    [Fact]
+    public void ConfiguredSlotsSmallerThanCap_NeverIncreased()
+    {
+        // Soft tier caps at 2, but configured is only 1 — throttle never raises parallelism.
+        Assert.Equal(1, Effective(55, 0, configured: 1));
+    }
+
+    [Fact]
+    public void NoUtilizationAtAll_ReturnsFullConfiguredSlots()
+    {
+        Assert.Equal(3, Effective(null, null));
+    }
+
+    [Fact]
+    public void ZeroSoftAndHardThresholds_NeverThrottleBelowGate()
+    {
+        Assert.Equal(3, UsageThrottle.EffectiveSlots(3, 79, 89, softPct: 0, hardPct: 0, gateFiveHourPct: GateFive, gateSevenDayPct: GateSeven));
+    }
+
+    [Fact]
+    public void ZeroGateThresholds_NeverHardBlock()
+    {
+        Assert.Equal(1, UsageThrottle.EffectiveSlots(3, 99, 99, softPct: Soft, hardPct: Hard, gateFiveHourPct: 0, gateSevenDayPct: 0));
+    }
+
+    [Fact]
+    public void ConfiguredSlotsBelowOne_ClampedToOneBeforeThrottling()
+    {
+        Assert.Equal(1, Effective(10, 10, configured: 0));
+    }
+}