releases/ClaudeDo
2
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases 24 Wiki Activity

fix(usage): stop 429s with an activity-dependent poll cadence + manual refresh

Browse Source 
The usage monitor polled the undocumented OAuth usage endpoint every 60s and
earned 429s. It now polls every 5 min while any task is Running and every
15 min while idle (usage_poll_interval_active_seconds / _idle_seconds, both
clamped to >= 60; the old single usage_poll_interval_seconds key is gone).

A 429 comes back as UsageRateLimitedException carrying Retry-After and adds
exponential backoff on top, capped at 30 min and never shorter than the normal
cadence; the strike count resets on the first success. The schedule arithmetic
is the pure static UsagePollSchedule.NextDelay.

Since the idle cadence is slow on purpose, WorkerHub.RefreshUsage drives
UsageMonitorService.RefreshNowAsync behind a Refresh now button in the Usage
Monitor modal: an out-of-band poll that pushes the loop's next-due time out so
no double poll follows, with a 10s cooldown so click-spam can't earn a 429.

Staleness now measures against the slower (idle) interval so an idle worker
isn't flagged stale just for not polling.
This commit is contained in:
mika kuns
2026-08-05 16:40:34 +02:00
parent f6cb8250bb
commit 2700c3d817
24 changed files with 615 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ClaudeDo.Worker.Tests/Usage/ClaudeOAuthUsageClientTests.cs
+41
View File
@@ -44,6 +44,7 @@ public sealed class ClaudeOAuthUsageClientTests : IDisposable
public List<HttpRequestMessage> Requests { get; } = new();
public HttpStatusCode ResponseStatus { get; set; } = HttpStatusCode.OK;
public string ResponseBody { get; set; } = "{}";
public RetryConditionHeaderValue? RetryAfter { get; set; }
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
{
@@ -52,6 +53,7 @@ public sealed class ClaudeOAuthUsageClientTests : IDisposable
{
Content = new StringContent(ResponseBody, Encoding.UTF8, "application/json"),
};
if (RetryAfter is not null) resp.Headers.RetryAfter = RetryAfter;
return Task.FromResult(resp);
}
}
@@ -179,4 +181,43 @@ public sealed class ClaudeOAuthUsageClientTests : IDisposable
await Assert.ThrowsAsync<InvalidOperationException>(() => client.GetUsageAsync());
}
[Fact]
public async Task GetUsageAsync_429_ThrowsRateLimitedWithRetryAfterDelta()
{
WriteCredentials();
var (client, handler) = Build();
handler.ResponseStatus = HttpStatusCode.TooManyRequests;
handler.RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(45));
var ex = await Assert.ThrowsAsync<UsageRateLimitedException>(() => client.GetUsageAsync());
Assert.Equal(TimeSpan.FromSeconds(45), ex.RetryAfter);
}
[Fact]
public async Task GetUsageAsync_429_WithoutRetryAfter_HasNullRetryAfter()
{
WriteCredentials();
var (client, handler) = Build();
handler.ResponseStatus = HttpStatusCode.TooManyRequests;
var ex = await Assert.ThrowsAsync<UsageRateLimitedException>(() => client.GetUsageAsync());
Assert.Null(ex.RetryAfter);
Assert.Contains("429", ex.Message);
}
[Fact]
public async Task GetUsageAsync_429_WithPastRetryAfterDate_IgnoresIt()
{
WriteCredentials();
var (client, handler) = Build();
handler.ResponseStatus = HttpStatusCode.TooManyRequests;
handler.RetryAfter = new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddMinutes(-5));
var ex = await Assert.ThrowsAsync<UsageRateLimitedException>(() => client.GetUsageAsync());
Assert.Null(ex.RetryAfter);
}
}
tests/ClaudeDo.Worker.Tests/Usage/RunningTaskProbeTests.cs
+56
View File
@@ -0,0 +1,56 @@
using ClaudeDo.Data.Models;
using ClaudeDo.Worker.Tests.Infrastructure;
using ClaudeDo.Worker.Usage;
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
namespace ClaudeDo.Worker.Tests.Usage;
public sealed class RunningTaskProbeTests : IDisposable
{
private readonly DbFixture _db = new();
public void Dispose() => _db.Dispose();
private async Task SeedTaskAsync(TaskStatus status)
{
await using var context = _db.CreateFactory().CreateDbContext();
var list = new ListEntity { Id = Guid.NewGuid().ToString(), Name = "L", CreatedAt = DateTime.UtcNow };
context.Lists.Add(list);
context.Tasks.Add(new TaskEntity
{
Id = Guid.NewGuid().ToString(),
ListId = list.Id,
Title = $"task-{status}",
Status = status,
CreatedAt = DateTime.UtcNow,
});
await context.SaveChangesAsync();
}
[Fact]
public async Task AnyRunningAsync_NoTasks_ReturnsFalse()
{
var probe = new RunningTaskProbe(_db.CreateFactory());
Assert.False(await probe.AnyRunningAsync());
}
[Fact]
public async Task AnyRunningAsync_QueuedTaskOnly_ReturnsFalse()
{
await SeedTaskAsync(TaskStatus.Queued);
var probe = new RunningTaskProbe(_db.CreateFactory());
Assert.False(await probe.AnyRunningAsync());
}
[Fact]
public async Task AnyRunningAsync_RunningTask_ReturnsTrue()
{
await SeedTaskAsync(TaskStatus.Idle);
await SeedTaskAsync(TaskStatus.Running);
var probe = new RunningTaskProbe(_db.CreateFactory());
Assert.True(await probe.AnyRunningAsync());
}
}
tests/ClaudeDo.Worker.Tests/Usage/UsageMonitorServiceTests.cs
+101 -2
View File
@@ -32,16 +32,25 @@ public sealed class UsageMonitorServiceTests : IDisposable
Task.FromResult(new UsageGateDecision(false, null));
}
private sealed class FakeRunningProbe : IRunningTaskProbe
{
public bool AnyRunning { get; set; }
public Task<bool> AnyRunningAsync(CancellationToken ct = default) => Task.FromResult(AnyRunning);
}
private static UsageSnapshot MakeSnapshot() => new(new UsageBucket(1, null), null, [], DateTime.UtcNow);
private (UsageMonitorService Service, UsageState State, CapturingHubContext Hub) CreateService(FakeClient client, WorkerConfig? cfg = null)
private (UsageMonitorService Service, UsageState State, CapturingHubContext Hub) CreateService(
FakeClient client, WorkerConfig? cfg = null, IRunningTaskProbe? probe = null)
{
var state = new UsageState();
var config = cfg ?? new WorkerConfig();
var builder = new UsageSnapshotBuilder(state, new FakeGate(), _db.CreateFactory(), config);
var hubContext = new CapturingHubContext();
var broadcaster = new HubBroadcaster(hubContext);
var service = new UsageMonitorService(client, state, config, NullLogger<UsageMonitorService>.Instance, builder, broadcaster);
var service = new UsageMonitorService(
client, state, config, NullLogger<UsageMonitorService>.Instance, builder, broadcaster,
probe ?? new FakeRunningProbe());
return (service, state, hubContext);
}
@@ -100,4 +109,94 @@ public sealed class UsageMonitorServiceTests : IDisposable
var calls = hubContext.Proxy.Calls.Where(c => c.Method == "UsageUpdated").ToList();
Assert.Equal(2, calls.Count);
}
[Fact]
public async Task RefreshNowAsync_Polls_And_Returns_Fresh_Snapshot()
{
var client = new FakeClient();
client.Results.Enqueue(MakeSnapshot);
var (service, _, _) = CreateService(client);
var dto = await service.RefreshNowAsync();
Assert.Equal(1, client.CallCount);
Assert.False(dto.IsStale);
Assert.NotNull(dto.FiveHourPercent);
}
[Fact]
public async Task RefreshNowAsync_WithinCooldown_ReusesLastPoll()
{
var client = new FakeClient();
client.Results.Enqueue(MakeSnapshot);
var (service, _, _) = CreateService(client);
await service.TickAsync(CancellationToken.None);
var dto = await service.RefreshNowAsync();
// Second call would have thrown "no result queued" had it hit the client.
Assert.Equal(1, client.CallCount);
Assert.False(dto.IsStale);
}
[Fact]
public async Task RateLimit_Failure_Is_Recorded_As_Error()
{
var client = new FakeClient();
client.Results.Enqueue(() => throw new UsageRateLimitedException(TimeSpan.FromSeconds(30)));
var (service, state, _) = CreateService(client);
await service.TickAsync(CancellationToken.None);
Assert.NotNull(state.LastError);
Assert.Contains("429", state.LastError);
}
}
public sealed class UsagePollScheduleTests
{
[Fact]
public void Uses_active_interval_while_a_task_runs()
{
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 0, null);
Assert.Equal(TimeSpan.FromSeconds(300), delay);
}
[Fact]
public void Uses_idle_interval_while_nothing_runs()
{
var delay = UsagePollSchedule.NextDelay(false, 300, 900, 0, null);
Assert.Equal(TimeSpan.FromSeconds(900), delay);
}
[Fact]
public void Backs_off_exponentially_on_repeated_rate_limits()
{
var first = UsagePollSchedule.NextDelay(true, 300, 900, 1, null);
var second = UsagePollSchedule.NextDelay(true, 300, 900, 2, null);
Assert.Equal(TimeSpan.FromSeconds(600), first);
Assert.True(second > first);
}
[Fact]
public void Honours_retry_after_when_longer_than_the_base_interval()
{
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 1, TimeSpan.FromSeconds(420));
Assert.Equal(TimeSpan.FromSeconds(420), delay);
}
[Fact]
public void Never_polls_sooner_than_the_base_interval_after_a_rate_limit()
{
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 1, TimeSpan.FromSeconds(5));
Assert.Equal(TimeSpan.FromSeconds(300), delay);
}
[Fact]
public void Caps_the_backoff()
{
var delay = UsagePollSchedule.NextDelay(false, 300, 900, 4, TimeSpan.FromHours(4));
Assert.Equal(UsagePollSchedule.MaxDelay, delay);
}
}
tests/ClaudeDo.Worker.Tests/Usage/UsageSnapshotBuilderTests.cs
+2 -2
View File
@@ -96,7 +96,7 @@ public sealed class UsageSnapshotBuilderTests : IDisposable
public async Task Snapshot_older_than_4x_poll_interval_is_stale()
{
await SetThresholdsAsync(80, 90);
var cfg = new WorkerConfig { UsagePollIntervalSeconds = 60 };
var cfg = new WorkerConfig { UsagePollIntervalActiveSeconds = 60, UsagePollIntervalIdleSeconds = 60 };
var state = new UsageState();
state.ReportSuccess(new UsageSnapshot(
new UsageBucket(10, null), null, Array.Empty<UsageLimitRow>(),
@@ -113,7 +113,7 @@ public sealed class UsageSnapshotBuilderTests : IDisposable
public async Task Fresh_snapshot_is_not_stale()
{
await SetThresholdsAsync(80, 90);
var cfg = new WorkerConfig { UsagePollIntervalSeconds = 60 };
var cfg = new WorkerConfig { UsagePollIntervalActiveSeconds = 60, UsagePollIntervalIdleSeconds = 60 };
var state = new UsageState();
state.ReportSuccess(new UsageSnapshot(
new UsageBucket(10, null), null, Array.Empty<UsageLimitRow>(),