fix(usage): stop 429s with an activity-dependent poll cadence + manual refresh

The usage monitor polled the undocumented OAuth usage endpoint every 60s and
earned 429s. It now polls every 5 min while any task is Running and every
15 min while idle (usage_poll_interval_active_seconds / _idle_seconds, both
clamped to >= 60; the old single usage_poll_interval_seconds key is gone).

A 429 comes back as UsageRateLimitedException carrying Retry-After and adds
exponential backoff on top, capped at 30 min and never shorter than the normal
cadence; the strike count resets on the first success. The schedule arithmetic
is the pure static UsagePollSchedule.NextDelay.

Since the idle cadence is slow on purpose, WorkerHub.RefreshUsage drives
UsageMonitorService.RefreshNowAsync behind a Refresh now button in the Usage
Monitor modal: an out-of-band poll that pushes the loop's next-due time out so
no double poll follows, with a 10s cooldown so click-spam can't earn a 429.

Staleness now measures against the slower (idle) interval so an idle worker
isn't flagged stale just for not polling.
This commit is contained in:
mika kuns
2026-08-05 16:40:34 +02:00
parent f6cb8250bb
commit 2700c3d817
24 changed files with 615 additions and 50 deletions
@@ -32,16 +32,25 @@ public sealed class UsageMonitorServiceTests : IDisposable
Task.FromResult(new UsageGateDecision(false, null));
}
private sealed class FakeRunningProbe : IRunningTaskProbe
{
public bool AnyRunning { get; set; }
public Task<bool> AnyRunningAsync(CancellationToken ct = default) => Task.FromResult(AnyRunning);
}
private static UsageSnapshot MakeSnapshot() => new(new UsageBucket(1, null), null, [], DateTime.UtcNow);
private (UsageMonitorService Service, UsageState State, CapturingHubContext Hub) CreateService(FakeClient client, WorkerConfig? cfg = null)
private (UsageMonitorService Service, UsageState State, CapturingHubContext Hub) CreateService(
FakeClient client, WorkerConfig? cfg = null, IRunningTaskProbe? probe = null)
{
var state = new UsageState();
var config = cfg ?? new WorkerConfig();
var builder = new UsageSnapshotBuilder(state, new FakeGate(), _db.CreateFactory(), config);
var hubContext = new CapturingHubContext();
var broadcaster = new HubBroadcaster(hubContext);
var service = new UsageMonitorService(client, state, config, NullLogger<UsageMonitorService>.Instance, builder, broadcaster);
var service = new UsageMonitorService(
client, state, config, NullLogger<UsageMonitorService>.Instance, builder, broadcaster,
probe ?? new FakeRunningProbe());
return (service, state, hubContext);
}
@@ -100,4 +109,94 @@ public sealed class UsageMonitorServiceTests : IDisposable
var calls = hubContext.Proxy.Calls.Where(c => c.Method == "UsageUpdated").ToList();
Assert.Equal(2, calls.Count);
}
[Fact]
public async Task RefreshNowAsync_Polls_And_Returns_Fresh_Snapshot()
{
var client = new FakeClient();
client.Results.Enqueue(MakeSnapshot);
var (service, _, _) = CreateService(client);
var dto = await service.RefreshNowAsync();
Assert.Equal(1, client.CallCount);
Assert.False(dto.IsStale);
Assert.NotNull(dto.FiveHourPercent);
}
[Fact]
public async Task RefreshNowAsync_WithinCooldown_ReusesLastPoll()
{
var client = new FakeClient();
client.Results.Enqueue(MakeSnapshot);
var (service, _, _) = CreateService(client);
await service.TickAsync(CancellationToken.None);
var dto = await service.RefreshNowAsync();
// Second call would have thrown "no result queued" had it hit the client.
Assert.Equal(1, client.CallCount);
Assert.False(dto.IsStale);
}
[Fact]
public async Task RateLimit_Failure_Is_Recorded_As_Error()
{
var client = new FakeClient();
client.Results.Enqueue(() => throw new UsageRateLimitedException(TimeSpan.FromSeconds(30)));
var (service, state, _) = CreateService(client);
await service.TickAsync(CancellationToken.None);
Assert.NotNull(state.LastError);
Assert.Contains("429", state.LastError);
}
}
public sealed class UsagePollScheduleTests
{
[Fact]
public void Uses_active_interval_while_a_task_runs()
{
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 0, null);
Assert.Equal(TimeSpan.FromSeconds(300), delay);
}
[Fact]
public void Uses_idle_interval_while_nothing_runs()
{
var delay = UsagePollSchedule.NextDelay(false, 300, 900, 0, null);
Assert.Equal(TimeSpan.FromSeconds(900), delay);
}
[Fact]
public void Backs_off_exponentially_on_repeated_rate_limits()
{
var first = UsagePollSchedule.NextDelay(true, 300, 900, 1, null);
var second = UsagePollSchedule.NextDelay(true, 300, 900, 2, null);
Assert.Equal(TimeSpan.FromSeconds(600), first);
Assert.True(second > first);
}
[Fact]
public void Honours_retry_after_when_longer_than_the_base_interval()
{
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 1, TimeSpan.FromSeconds(420));
Assert.Equal(TimeSpan.FromSeconds(420), delay);
}
[Fact]
public void Never_polls_sooner_than_the_base_interval_after_a_rate_limit()
{
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 1, TimeSpan.FromSeconds(5));
Assert.Equal(TimeSpan.FromSeconds(300), delay);
}
[Fact]
public void Caps_the_backoff()
{
var delay = UsagePollSchedule.NextDelay(false, 300, 900, 4, TimeSpan.FromHours(4));
Assert.Equal(UsagePollSchedule.MaxDelay, delay);
}
}