fix(usage): stop 429s with an activity-dependent poll cadence + manual refresh
The usage monitor polled the undocumented OAuth usage endpoint every 60s and earned 429s. It now polls every 5 min while any task is Running and every 15 min while idle (usage_poll_interval_active_seconds / _idle_seconds, both clamped to >= 60; the old single usage_poll_interval_seconds key is gone). A 429 comes back as UsageRateLimitedException carrying Retry-After and adds exponential backoff on top, capped at 30 min and never shorter than the normal cadence; the strike count resets on the first success. The schedule arithmetic is the pure static UsagePollSchedule.NextDelay. Since the idle cadence is slow on purpose, WorkerHub.RefreshUsage drives UsageMonitorService.RefreshNowAsync behind a Refresh now button in the Usage Monitor modal: an out-of-band poll that pushes the loop's next-due time out so no double poll follows, with a 10s cooldown so click-spam can't earn a 429. Staleness now measures against the slower (idle) interval so an idle worker isn't flagged stale just for not polling.
This commit is contained in:
@@ -32,16 +32,25 @@ public sealed class UsageMonitorServiceTests : IDisposable
|
||||
Task.FromResult(new UsageGateDecision(false, null));
|
||||
}
|
||||
|
||||
private sealed class FakeRunningProbe : IRunningTaskProbe
|
||||
{
|
||||
public bool AnyRunning { get; set; }
|
||||
public Task<bool> AnyRunningAsync(CancellationToken ct = default) => Task.FromResult(AnyRunning);
|
||||
}
|
||||
|
||||
private static UsageSnapshot MakeSnapshot() => new(new UsageBucket(1, null), null, [], DateTime.UtcNow);
|
||||
|
||||
private (UsageMonitorService Service, UsageState State, CapturingHubContext Hub) CreateService(FakeClient client, WorkerConfig? cfg = null)
|
||||
private (UsageMonitorService Service, UsageState State, CapturingHubContext Hub) CreateService(
|
||||
FakeClient client, WorkerConfig? cfg = null, IRunningTaskProbe? probe = null)
|
||||
{
|
||||
var state = new UsageState();
|
||||
var config = cfg ?? new WorkerConfig();
|
||||
var builder = new UsageSnapshotBuilder(state, new FakeGate(), _db.CreateFactory(), config);
|
||||
var hubContext = new CapturingHubContext();
|
||||
var broadcaster = new HubBroadcaster(hubContext);
|
||||
var service = new UsageMonitorService(client, state, config, NullLogger<UsageMonitorService>.Instance, builder, broadcaster);
|
||||
var service = new UsageMonitorService(
|
||||
client, state, config, NullLogger<UsageMonitorService>.Instance, builder, broadcaster,
|
||||
probe ?? new FakeRunningProbe());
|
||||
return (service, state, hubContext);
|
||||
}
|
||||
|
||||
@@ -100,4 +109,94 @@ public sealed class UsageMonitorServiceTests : IDisposable
|
||||
var calls = hubContext.Proxy.Calls.Where(c => c.Method == "UsageUpdated").ToList();
|
||||
Assert.Equal(2, calls.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RefreshNowAsync_Polls_And_Returns_Fresh_Snapshot()
|
||||
{
|
||||
var client = new FakeClient();
|
||||
client.Results.Enqueue(MakeSnapshot);
|
||||
var (service, _, _) = CreateService(client);
|
||||
|
||||
var dto = await service.RefreshNowAsync();
|
||||
|
||||
Assert.Equal(1, client.CallCount);
|
||||
Assert.False(dto.IsStale);
|
||||
Assert.NotNull(dto.FiveHourPercent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RefreshNowAsync_WithinCooldown_ReusesLastPoll()
|
||||
{
|
||||
var client = new FakeClient();
|
||||
client.Results.Enqueue(MakeSnapshot);
|
||||
var (service, _, _) = CreateService(client);
|
||||
|
||||
await service.TickAsync(CancellationToken.None);
|
||||
var dto = await service.RefreshNowAsync();
|
||||
|
||||
// Second call would have thrown "no result queued" had it hit the client.
|
||||
Assert.Equal(1, client.CallCount);
|
||||
Assert.False(dto.IsStale);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RateLimit_Failure_Is_Recorded_As_Error()
|
||||
{
|
||||
var client = new FakeClient();
|
||||
client.Results.Enqueue(() => throw new UsageRateLimitedException(TimeSpan.FromSeconds(30)));
|
||||
var (service, state, _) = CreateService(client);
|
||||
|
||||
await service.TickAsync(CancellationToken.None);
|
||||
|
||||
Assert.NotNull(state.LastError);
|
||||
Assert.Contains("429", state.LastError);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class UsagePollScheduleTests
|
||||
{
|
||||
[Fact]
|
||||
public void Uses_active_interval_while_a_task_runs()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 0, null);
|
||||
Assert.Equal(TimeSpan.FromSeconds(300), delay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Uses_idle_interval_while_nothing_runs()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(false, 300, 900, 0, null);
|
||||
Assert.Equal(TimeSpan.FromSeconds(900), delay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Backs_off_exponentially_on_repeated_rate_limits()
|
||||
{
|
||||
var first = UsagePollSchedule.NextDelay(true, 300, 900, 1, null);
|
||||
var second = UsagePollSchedule.NextDelay(true, 300, 900, 2, null);
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(600), first);
|
||||
Assert.True(second > first);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Honours_retry_after_when_longer_than_the_base_interval()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 1, TimeSpan.FromSeconds(420));
|
||||
Assert.Equal(TimeSpan.FromSeconds(420), delay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Never_polls_sooner_than_the_base_interval_after_a_rate_limit()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 1, TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(TimeSpan.FromSeconds(300), delay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Caps_the_backoff()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(false, 300, 900, 4, TimeSpan.FromHours(4));
|
||||
Assert.Equal(UsagePollSchedule.MaxDelay, delay);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user