fix(usage): stop 429s with an activity-dependent poll cadence + manual refresh
The usage monitor polled the undocumented OAuth usage endpoint every 60s and earned 429s. It now polls every 5 min while any task is Running and every 15 min while idle (usage_poll_interval_active_seconds / _idle_seconds, both clamped to >= 60; the old single usage_poll_interval_seconds key is gone). A 429 comes back as UsageRateLimitedException carrying Retry-After and adds exponential backoff on top, capped at 30 min and never shorter than the normal cadence; the strike count resets on the first success. The schedule arithmetic is the pure static UsagePollSchedule.NextDelay. Since the idle cadence is slow on purpose, WorkerHub.RefreshUsage drives UsageMonitorService.RefreshNowAsync behind a Refresh now button in the Usage Monitor modal: an out-of-band poll that pushes the loop's next-due time out so no double poll follows, with a 10s cooldown so click-spam can't earn a 429. Staleness now measures against the slower (idle) interval so an idle worker isn't flagged stale just for not polling.
This commit is contained in:
@@ -153,6 +153,7 @@ public abstract class StubWorkerClient : IWorkerClient
|
||||
public virtual Task ClearOnlineInboxAuthAsync() => Task.CompletedTask;
|
||||
|
||||
public virtual Task<UsageSnapshotDto?> GetUsageSnapshotAsync() => Task.FromResult<UsageSnapshotDto?>(null);
|
||||
public virtual Task<UsageSnapshotDto?> RefreshUsageAsync() => Task.FromResult<UsageSnapshotDto?>(null);
|
||||
public virtual Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to)
|
||||
=> Task.FromResult<IReadOnlyList<ModelUsageRowDto>>(Array.Empty<ModelUsageRowDto>());
|
||||
public virtual Task<IReadOnlyList<TaskUsageRowDto>> GetTaskUsageAsync(DateOnly from, DateOnly to)
|
||||
|
||||
@@ -29,8 +29,19 @@ public class UsageMonitorModalViewModelTests
|
||||
public int ModelUsageCalls;
|
||||
public int TaskUsageCalls;
|
||||
|
||||
public UsageSnapshotDto? RefreshedSnapshot;
|
||||
public int RefreshCalls;
|
||||
public Exception? RefreshThrows;
|
||||
|
||||
public override Task<UsageSnapshotDto?> GetUsageSnapshotAsync() => Task.FromResult(Snapshot);
|
||||
|
||||
public override Task<UsageSnapshotDto?> RefreshUsageAsync()
|
||||
{
|
||||
RefreshCalls++;
|
||||
if (RefreshThrows is not null) throw RefreshThrows;
|
||||
return Task.FromResult(RefreshedSnapshot);
|
||||
}
|
||||
|
||||
public override Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to)
|
||||
{
|
||||
ModelUsageCalls++;
|
||||
@@ -68,6 +79,60 @@ public class UsageMonitorModalViewModelTests
|
||||
isGateBlocked, gateReason, fetchedAtUtc ?? DateTime.UtcNow, isStale, lastError,
|
||||
configuredSlots, effectiveSlots, throttleBucket);
|
||||
|
||||
// ── Manual refresh ──────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_ReplacesSnapshotAndReloadsTables()
|
||||
{
|
||||
var worker = new FakeWorker
|
||||
{
|
||||
Snapshot = Snapshot(new[] { Limit("session") }),
|
||||
RefreshedSnapshot = Snapshot(new[] { Limit("session"), Limit("weekly_all") }),
|
||||
};
|
||||
var vm = new UsageMonitorModalViewModel(worker);
|
||||
await vm.LoadAsync();
|
||||
var callsAfterLoad = worker.ModelUsageCalls;
|
||||
|
||||
await vm.RefreshCommand.ExecuteAsync(null);
|
||||
|
||||
Assert.Equal(1, worker.RefreshCalls);
|
||||
Assert.Equal(2, vm.GaugeRows.Count);
|
||||
Assert.Equal(callsAfterLoad + 1, worker.ModelUsageCalls);
|
||||
Assert.False(vm.IsRefreshing);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_NullResult_KeepsPreviousSnapshot()
|
||||
{
|
||||
var worker = new FakeWorker { Snapshot = Snapshot(new[] { Limit("session") }) };
|
||||
var vm = new UsageMonitorModalViewModel(worker);
|
||||
await vm.LoadAsync();
|
||||
|
||||
await vm.RefreshCommand.ExecuteAsync(null);
|
||||
|
||||
Assert.Single(vm.GaugeRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_Failure_ReportsErrorAndClearsBusyFlag()
|
||||
{
|
||||
var worker = new FakeWorker
|
||||
{
|
||||
Snapshot = Snapshot(new[] { Limit("session") }),
|
||||
RefreshThrows = new InvalidOperationException("worker offline"),
|
||||
};
|
||||
var vm = new UsageMonitorModalViewModel(worker);
|
||||
await vm.LoadAsync();
|
||||
string? reported = null;
|
||||
vm.ErrorReported += m => reported = m;
|
||||
|
||||
await vm.RefreshCommand.ExecuteAsync(null);
|
||||
|
||||
Assert.NotNull(reported);
|
||||
Assert.Contains("worker offline", reported);
|
||||
Assert.False(vm.IsRefreshing);
|
||||
}
|
||||
|
||||
// ── Gauge label derivation ──────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
|
||||
Reference in New Issue
Block a user