Merge branch 'claudedo/87105f5ec4f44af4ae6089cd2e153e3c'
This commit is contained in:
@@ -50,7 +50,8 @@ public sealed class QueueServiceTests : IDisposable
|
||||
|
||||
private (QueueService service, FakeClaudeProcess fakeProcess) CreateService(
|
||||
Func<string, string, IReadOnlyList<string>, Func<string, Task>, CancellationToken, Task<RunResult>>? handler = null,
|
||||
FakeUsageGate? usageGate = null)
|
||||
FakeUsageGate? usageGate = null,
|
||||
UsageState? usageState = null)
|
||||
{
|
||||
var fake = new FakeClaudeProcess(handler);
|
||||
_hubContext = new CapturingHubContext();
|
||||
@@ -67,10 +68,24 @@ public sealed class QueueServiceTests : IDisposable
|
||||
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels);
|
||||
_usageGate = usageGate ?? new FakeUsageGate();
|
||||
var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, _waker, picker, overrideSlot, state, built.RunCancels,
|
||||
_usageGate, new UsageState(), broadcaster);
|
||||
_usageGate, usageState ?? new UsageState(), broadcaster);
|
||||
return (service, fake);
|
||||
}
|
||||
|
||||
private async Task SetAppSettingsAsync(
|
||||
int maxParallel, int softPct = 50, int hardPct = 65, int gateFive = 80, int gateSeven = 90)
|
||||
{
|
||||
using var ctx = _db.CreateContext();
|
||||
var repo = new AppSettingsRepository(ctx);
|
||||
var settings = await repo.GetAsync();
|
||||
settings.MaxParallelExecutions = maxParallel;
|
||||
settings.UsageThrottleSoftPct = softPct;
|
||||
settings.UsageThrottleHardPct = hardPct;
|
||||
settings.UsageGateFiveHourPct = gateFive;
|
||||
settings.UsageGateSevenDayPct = gateSeven;
|
||||
await repo.UpdateAsync(settings);
|
||||
}
|
||||
|
||||
private async Task<string> SeedListAsync()
|
||||
{
|
||||
var listId = Guid.NewGuid().ToString();
|
||||
@@ -452,4 +467,154 @@ public sealed class QueueServiceTests : IDisposable
|
||||
.Count(c => c.Method == "WorkerLog" && (WorkerLogLevel)c.Args[1]! == WorkerLogLevel.Warn);
|
||||
Assert.Equal(1, warnCalls);
|
||||
}
|
||||
|
||||
// Polls until `read()` reaches `expected` (or times out), then waits a further grace period
|
||||
// to make sure the count doesn't keep climbing past it — needed because slot fills happen
|
||||
// concurrently and a fixed sleep is either flaky (too short) or slow (too long).
|
||||
private static async Task AssertStableCountAsync(Func<int> read, int expected)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(5);
|
||||
while (read() < expected && DateTime.UtcNow < deadline)
|
||||
await Task.Delay(20);
|
||||
|
||||
await Task.Delay(250);
|
||||
Assert.Equal(expected, read());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Throttle_StepsDownEffectiveSlots_BelowConfiguredMax()
|
||||
{
|
||||
var listId = await SeedListAsync();
|
||||
await SeedQueuedTask(listId);
|
||||
await SeedQueuedTask(listId);
|
||||
await SeedQueuedTask(listId);
|
||||
|
||||
await SetAppSettingsAsync(maxParallel: 3);
|
||||
|
||||
var usageState = new UsageState();
|
||||
usageState.ReportSuccess(new UsageSnapshot(
|
||||
new UsageBucket(60, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
|
||||
|
||||
var startedCount = 0;
|
||||
var block = new TaskCompletionSource();
|
||||
var (service, _) = CreateService(async (_, _, _, _, _) =>
|
||||
{
|
||||
Interlocked.Increment(ref startedCount);
|
||||
await block.Task;
|
||||
return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
|
||||
}, usageState: usageState);
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
await service.StartAsync(cts.Token);
|
||||
_waker.Wake();
|
||||
|
||||
// 60% is between the soft (50) and hard (65) thresholds — capped at 2 slots even
|
||||
// though 3 are configured and 3 tasks are queued.
|
||||
await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 2);
|
||||
|
||||
block.SetResult();
|
||||
cts.Cancel();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Throttle_AtHardThreshold_CapsToOneSlot()
|
||||
{
|
||||
var listId = await SeedListAsync();
|
||||
await SeedQueuedTask(listId);
|
||||
await SeedQueuedTask(listId);
|
||||
|
||||
await SetAppSettingsAsync(maxParallel: 3);
|
||||
|
||||
var usageState = new UsageState();
|
||||
usageState.ReportSuccess(new UsageSnapshot(
|
||||
new UsageBucket(70, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
|
||||
|
||||
var startedCount = 0;
|
||||
var block = new TaskCompletionSource();
|
||||
var (service, _) = CreateService(async (_, _, _, _, _) =>
|
||||
{
|
||||
Interlocked.Increment(ref startedCount);
|
||||
await block.Task;
|
||||
return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
|
||||
}, usageState: usageState);
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
await service.StartAsync(cts.Token);
|
||||
_waker.Wake();
|
||||
|
||||
await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 1);
|
||||
|
||||
block.SetResult();
|
||||
cts.Cancel();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NoUsageSnapshot_FallsBackToFullConfiguredParallelism()
|
||||
{
|
||||
var listId = await SeedListAsync();
|
||||
await SeedQueuedTask(listId);
|
||||
await SeedQueuedTask(listId);
|
||||
await SeedQueuedTask(listId);
|
||||
|
||||
await SetAppSettingsAsync(maxParallel: 3);
|
||||
|
||||
// No snapshot has landed yet (fresh UsageState) — throttle must fail open.
|
||||
var startedCount = 0;
|
||||
var block = new TaskCompletionSource();
|
||||
var (service, _) = CreateService(async (_, _, _, _, _) =>
|
||||
{
|
||||
Interlocked.Increment(ref startedCount);
|
||||
await block.Task;
|
||||
return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
|
||||
}, usageState: new UsageState());
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
await service.StartAsync(cts.Token);
|
||||
_waker.Wake();
|
||||
|
||||
await AssertStableCountAsync(() => Volatile.Read(ref startedCount), 3);
|
||||
|
||||
block.SetResult();
|
||||
cts.Cancel();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Throttle_Engaging_Does_Not_Cancel_AlreadyRunning_Slot()
|
||||
{
|
||||
var listId = await SeedListAsync();
|
||||
await SeedQueuedTask(listId);
|
||||
|
||||
await SetAppSettingsAsync(maxParallel: 3);
|
||||
var usageState = new UsageState();
|
||||
|
||||
var running = new TaskCompletionSource();
|
||||
var cancelled = false;
|
||||
var (service, _) = CreateService(async (_, _, _, _, ct) =>
|
||||
{
|
||||
running.SetResult();
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
cancelled = true;
|
||||
throw;
|
||||
}
|
||||
return new RunResult { ExitCode = 0, ResultMarkdown = "ok" };
|
||||
}, usageState: usageState);
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
await service.StartAsync(cts.Token);
|
||||
_waker.Wake();
|
||||
await running.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
// Throttle engages hard after the slot is already running — several backstop ticks pass.
|
||||
usageState.ReportSuccess(new UsageSnapshot(
|
||||
new UsageBucket(70, null), new UsageBucket(0, null), Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
|
||||
await Task.Delay(200);
|
||||
|
||||
Assert.False(cancelled);
|
||||
cts.Cancel();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user