fix(usage): stop 429s with an activity-dependent poll cadence + manual refresh

The usage monitor polled the undocumented OAuth usage endpoint every 60s and
earned 429s. It now polls every 5 min while any task is Running and every
15 min while idle (usage_poll_interval_active_seconds / _idle_seconds, both
clamped to >= 60; the old single usage_poll_interval_seconds key is gone).

A 429 comes back as UsageRateLimitedException carrying Retry-After and adds
exponential backoff on top, capped at 30 min and never shorter than the normal
cadence; the strike count resets on the first success. The schedule arithmetic
is the pure static UsagePollSchedule.NextDelay.

Since the idle cadence is slow on purpose, WorkerHub.RefreshUsage drives
UsageMonitorService.RefreshNowAsync behind a Refresh now button in the Usage
Monitor modal: an out-of-band poll that pushes the loop's next-due time out so
no double poll follows, with a 10s cooldown so click-spam can't earn a 429.

Staleness now measures against the slower (idle) interval so an idle worker
isn't flagged stale just for not polling.
This commit is contained in:
mika kuns
2026-08-05 16:40:34 +02:00
parent f6cb8250bb
commit 2700c3d817
24 changed files with 615 additions and 50 deletions
@@ -29,8 +29,19 @@ public class UsageMonitorModalViewModelTests
public int ModelUsageCalls;
public int TaskUsageCalls;
public UsageSnapshotDto? RefreshedSnapshot;
public int RefreshCalls;
public Exception? RefreshThrows;
public override Task<UsageSnapshotDto?> GetUsageSnapshotAsync() => Task.FromResult(Snapshot);
public override Task<UsageSnapshotDto?> RefreshUsageAsync()
{
RefreshCalls++;
if (RefreshThrows is not null) throw RefreshThrows;
return Task.FromResult(RefreshedSnapshot);
}
public override Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to)
{
ModelUsageCalls++;
@@ -68,6 +79,60 @@ public class UsageMonitorModalViewModelTests
isGateBlocked, gateReason, fetchedAtUtc ?? DateTime.UtcNow, isStale, lastError,
configuredSlots, effectiveSlots, throttleBucket);
// ── Manual refresh ──────────────────────────────────────────────────────
[Fact]
public async Task Refresh_ReplacesSnapshotAndReloadsTables()
{
var worker = new FakeWorker
{
Snapshot = Snapshot(new[] { Limit("session") }),
RefreshedSnapshot = Snapshot(new[] { Limit("session"), Limit("weekly_all") }),
};
var vm = new UsageMonitorModalViewModel(worker);
await vm.LoadAsync();
var callsAfterLoad = worker.ModelUsageCalls;
await vm.RefreshCommand.ExecuteAsync(null);
Assert.Equal(1, worker.RefreshCalls);
Assert.Equal(2, vm.GaugeRows.Count);
Assert.Equal(callsAfterLoad + 1, worker.ModelUsageCalls);
Assert.False(vm.IsRefreshing);
}
[Fact]
public async Task Refresh_NullResult_KeepsPreviousSnapshot()
{
var worker = new FakeWorker { Snapshot = Snapshot(new[] { Limit("session") }) };
var vm = new UsageMonitorModalViewModel(worker);
await vm.LoadAsync();
await vm.RefreshCommand.ExecuteAsync(null);
Assert.Single(vm.GaugeRows);
}
[Fact]
public async Task Refresh_Failure_ReportsErrorAndClearsBusyFlag()
{
var worker = new FakeWorker
{
Snapshot = Snapshot(new[] { Limit("session") }),
RefreshThrows = new InvalidOperationException("worker offline"),
};
var vm = new UsageMonitorModalViewModel(worker);
await vm.LoadAsync();
string? reported = null;
vm.ErrorReported += m => reported = m;
await vm.RefreshCommand.ExecuteAsync(null);
Assert.NotNull(reported);
Assert.Contains("worker offline", reported);
Assert.False(vm.IsRefreshing);
}
// ── Gauge label derivation ──────────────────────────────────────────────
[Fact]