Merge branch 'claudedo/87105f5ec4f44af4ae6089cd2e153e3c'

This commit is contained in:
mika kuns
2026-08-05 16:10:37 +02:00
24 changed files with 1466 additions and 19 deletions
@@ -50,7 +50,8 @@ public sealed class QueueServiceTests : IDisposable
private (QueueService service, FakeClaudeProcess fakeProcess) CreateService(
Func<string, string, IReadOnlyList<string>, Func<string, Task>, CancellationToken, Task<RunResult>>? handler = null,
FakeUsageGate? usageGate = null)
FakeUsageGate? usageGate = null,
UsageState? usageState = null)
{
var fake = new FakeClaudeProcess(handler);
_hubContext = new CapturingHubContext();
@@ -67,10 +68,24 @@ public sealed class QueueServiceTests : IDisposable
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels);
_usageGate = usageGate ?? new FakeUsageGate();
var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, _waker, picker, overrideSlot, state, built.RunCancels,
_usageGate, new UsageState(), broadcaster);
_usageGate, usageState ?? new UsageState(), broadcaster);
return (service, fake);
}
private async Task SetAppSettingsAsync(
int maxParallel, int softPct = 50, int hardPct = 65, int gateFive = 80, int gateSeven = 90)
{
using var ctx = _db.CreateContext();
var repo = new AppSettingsRepository(ctx);
var settings = await repo.GetAsync();
settings.MaxParallelExecutions = maxParallel;
settings.UsageThrottleSoftPct = softPct;
settings.UsageThrottleHardPct = hardPct;
settings.UsageGateFiveHourPct = gateFive;
settings.UsageGateSevenDayPct = gateSeven;
await repo.UpdateAsync(settings);
}
private async Task<string> SeedListAsync()
{
var listId = Guid.NewGuid().ToString();
@@ -452,4 +467,154 @@ public sealed class QueueServiceTests : IDisposable
.Count(c => c.Method == "WorkerLog" && (WorkerLogLevel)c.Args[1]! == WorkerLogLevel.Warn);
Assert.Equal(1, warnCalls);
}
// Polls until `read()` reaches `expected` (or times out), then waits a further grace period
// to make sure the count doesn't keep climbing past it — needed because slot fills happen
// concurrently and a fixed sleep is either flaky (too short) or slow (too long).
private static async Task AssertStableCountAsync(Func<int> read, int expected)
{
var deadline = DateTime.UtcNow.AddSeconds(5);
while (read() < expected && DateTime.UtcNow < deadline)
await Task.Delay(20);
await Task.Delay(250);
Assert.Equal(expected, read());
}
[Fact]
public async Task Throttle_StepsDownEffectiveSlots_BelowConfiguredMax()
{
var listId = await SeedListAsync();
await SeedQueuedTask(listId);
await SeedQueuedTask(listId);
await SeedQueuedTask(listId);
await SetAppSettingsAsync(maxParallel: 3);
var usageState = new UsageState();
usageState.ReportSuccess(new UsageSnapshot(
new UsageBucket(60, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
var startedCount = 0;
var block = new TaskCompletionSource();
var (service, _) = CreateService(async (_, _, _, _, _) =>
{
Interlocked.Increment(ref startedCount);
await block.Task;
return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
}, usageState: usageState);
using var cts = new CancellationTokenSource();
await service.StartAsync(cts.Token);
_waker.Wake();
// 60% is between the soft (50) and hard (65) thresholds — capped at 2 slots even
// though 3 are configured and 3 tasks are queued.
await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 2);
block.SetResult();
cts.Cancel();
}
[Fact]
public async Task Throttle_AtHardThreshold_CapsToOneSlot()
{
var listId = await SeedListAsync();
await SeedQueuedTask(listId);
await SeedQueuedTask(listId);
await SetAppSettingsAsync(maxParallel: 3);
var usageState = new UsageState();
usageState.ReportSuccess(new UsageSnapshot(
new UsageBucket(70, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
var startedCount = 0;
var block = new TaskCompletionSource();
var (service, _) = CreateService(async (_, _, _, _, _) =>
{
Interlocked.Increment(ref startedCount);
await block.Task;
return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
}, usageState: usageState);
using var cts = new CancellationTokenSource();
await service.StartAsync(cts.Token);
_waker.Wake();
await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 1);
block.SetResult();
cts.Cancel();
}
[Fact]
public async Task NoUsageSnapshot_FallsBackToFullConfiguredParallelism()
{
var listId = await SeedListAsync();
await SeedQueuedTask(listId);
await SeedQueuedTask(listId);
await SeedQueuedTask(listId);
await SetAppSettingsAsync(maxParallel: 3);
// No snapshot has landed yet (fresh UsageState) — throttle must fail open.
var startedCount = 0;
var block = new TaskCompletionSource();
var (service, _) = CreateService(async (_, _, _, _, _) =>
{
Interlocked.Increment(ref startedCount);
await block.Task;
return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
}, usageState: new UsageState());
using var cts = new CancellationTokenSource();
await service.StartAsync(cts.Token);
_waker.Wake();
await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 3);
block.SetResult();
cts.Cancel();
}
[Fact]
public async Task Throttle_Engaging_Does_Not_Cancel_AlreadyRunning_Slot()
{
var listId = await SeedListAsync();
await SeedQueuedTask(listId);
await SetAppSettingsAsync(maxParallel: 3);
var usageState = new UsageState();
var running = new TaskCompletionSource();
var cancelled = false;
var (service, _) = CreateService(async (_, _, _, _, ct) =>
{
running.SetResult();
try
{
await Task.Delay(Timeout.Infinite, ct);
}
catch (OperationCanceledException)
{
cancelled = true;
throw;
}
return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
}, usageState: usageState);
using var cts = new CancellationTokenSource();
await service.StartAsync(cts.Token);
_waker.Wake();
await running.Task.WaitAsync(TimeSpan.FromSeconds(5));
// Throttle engages hard after the slot is already running — several backstop ticks pass.
usageState.ReportSuccess(new UsageSnapshot(
new UsageBucket(70, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
await Task.Delay(200);
Assert.False(cancelled);
cts.Cancel();
}
}