Merge branch 'claudedo/f359858ac98a439593e459df9c5d0a5d'

This commit is contained in:
mika kuns
2026-08-05 16:49:08 +02:00
26 changed files with 1169 additions and 43 deletions
+6 -1
View File
@@ -431,6 +431,8 @@
"title": "USAGE MONITOR",
"windowTitle": "Usage Monitor",
"noGauges": "Noch keine Nutzungslimits gemeldet.",
"refresh": "Jetzt aktualisieren",
"refreshHint": "Abruf alle 5 Min., während ein Task läuft, sonst alle 15 Min.",
"staleFormat": "Werte veraltet (Stand {0})",
"staleGateHint": "Das Gate greift in diesem Zustand nicht.",
"gateBlockedFormat": "Queue pausiert — {0}",
@@ -598,7 +600,10 @@
"settingsModal": { "workerOffline": "Worker offline — Einstellungen schreibgeschützt.", "saveFailed": "Speichern fehlgeschlagen: {0}" },
"onlineInbox": { "workerOffline": "Worker offline — Konfiguration kann nicht geladen werden.", "saved": "Konfiguration gespeichert.", "saveFailed": "Speichern fehlgeschlagen: {0}", "signedIn": "Erfolgreich angemeldet.", "signedInNoRole": "Angemeldet, aber diesem Konto fehlt die Rolle 'user' in Zitadel — die Online-Synchronisierung wird abgelehnt, bis die Rolle im ClaudeDo-Projekt zugewiesen wird.", "signInFailed": "Anmeldung fehlgeschlagen: {0}", "signedOut": "Abgemeldet.", "signOutFailed": "Abmeldung fehlgeschlagen: {0}" },
"weeklyReport": { "invalidRange": "Ungültiger Datumsbereich.", "generating": "Bericht wird erstellt…", "error": "Fehler: {0}" },
"usageMonitor": { "loadFailed": "Nutzungsdaten konnten nicht geladen werden: {0}" },
"usageMonitor": {
"loadFailed": "Nutzungsdaten konnten nicht geladen werden: {0}",
"refreshFailed": "Nutzung konnte nicht aktualisiert werden: {0}"
},
"filesTab": { "workerOffline": "Worker offline.", "noneBundled": "Keine Standard-Agenten mitgeliefert.", "allPresent": "Alle Standard-Agenten bereits vorhanden.", "restored": "{0} Standard-Agent(en) wiederhergestellt.", "restoreFailed": "Wiederherstellung fehlgeschlagen: {0}", "openFailed": "Öffnen fehlgeschlagen: {0}", "resetToDefault": "Auf den mitgelieferten Standard zurückgesetzt." },
"sessionSkillsTab": { "installed": "Installiert: {0}", "installFailed": "Installation fehlgeschlagen: {0}", "updated": "Aktualisiert.", "updateFailed": "Aktualisierung fehlgeschlagen: {0}", "removed": "Entfernt.", "removeFailed": "Entfernen fehlgeschlagen: {0}" },
"worktreesTab": { "workerOffline": "Worker offline.", "removed": "{0} Worktree(s) entfernt.", "blocked": "Zwangsentfernung nicht möglich: {0} Aufgabe(n) laufen noch. Brich sie zuerst ab.", "removedFrom": "{0} Worktree(s) von {1} Aufgabe(n) entfernt.", "cleanupFailed": "Aufräumen fehlgeschlagen: {0}", "resetFailed": "Zurücksetzen fehlgeschlagen: {0}" },
+6 -1
View File
@@ -431,6 +431,8 @@
"title": "USAGE MONITOR",
"windowTitle": "Usage Monitor",
"noGauges": "No usage limits reported yet.",
"refresh": "Refresh now",
"refreshHint": "Polled every 5 min while a task runs, otherwise every 15 min.",
"staleFormat": "Values stale (as of {0})",
"staleGateHint": "The gate does not apply while values are stale.",
"gateBlockedFormat": "Queue paused — {0}",
@@ -598,7 +600,10 @@
"settingsModal": { "workerOffline": "Worker offline — settings read-only.", "saveFailed": "Save failed: {0}" },
"onlineInbox": { "workerOffline": "Worker offline — cannot load config.", "saved": "Config saved.", "saveFailed": "Save failed: {0}", "signedIn": "Signed in successfully.", "signedInNoRole": "Signed in, but this account is missing the 'user' role in Zitadel — online sync will be rejected until the role is granted in the ClaudeDo project.", "signInFailed": "Sign-in failed: {0}", "signedOut": "Signed out.", "signOutFailed": "Sign-out failed: {0}" },
"weeklyReport": { "invalidRange": "Invalid date range.", "generating": "Generating report…", "error": "Error: {0}" },
"usageMonitor": { "loadFailed": "Couldn't load usage data: {0}" },
"usageMonitor": {
"loadFailed": "Couldn't load usage data: {0}",
"refreshFailed": "Couldn't refresh usage: {0}"
},
"filesTab": { "workerOffline": "Worker offline.", "noneBundled": "No default agents bundled.", "allPresent": "All default agents already present.", "restored": "Restored {0} default agent(s).", "restoreFailed": "Restore failed: {0}", "openFailed": "Open failed: {0}", "resetToDefault": "Reset to the bundled default." },
"sessionSkillsTab": { "installed": "Installed: {0}", "installFailed": "Install failed: {0}", "updated": "Updated.", "updateFailed": "Update failed: {0}", "removed": "Removed.", "removeFailed": "Remove failed: {0}" },
"worktreesTab": { "workerOffline": "Worker offline.", "removed": "Removed {0} worktree(s).", "blocked": "Cannot force-remove: {0} task(s) still running. Cancel them first.", "removedFrom": "Removed {0} worktree(s) from {1} task(s).", "cleanupFailed": "Cleanup failed: {0}", "resetFailed": "Reset failed: {0}" },
@@ -143,6 +143,9 @@ public interface IWorkerClient : INotifyPropertyChanged
/// <summary>Raised whenever the worker's usage poller ticks (success or failure).</summary>
event Action<UsageSnapshotDto>? UsageUpdatedEvent;
Task<UsageSnapshotDto?> GetUsageSnapshotAsync();
/// <summary>Forces an out-of-band usage poll on the worker and returns the fresh snapshot.</summary>
Task<UsageSnapshotDto?> RefreshUsageAsync();
Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to);
Task<IReadOnlyList<TaskUsageRowDto>> GetTaskUsageAsync(DateOnly from, DateOnly to);
}
+3
View File
@@ -582,6 +582,9 @@ public partial class WorkerClient : ObservableObject, IAsyncDisposable, IWorkerC
public Task<UsageSnapshotDto?> GetUsageSnapshotAsync()
=> TryInvokeAsync<UsageSnapshotDto>("GetUsageSnapshot");
public Task<UsageSnapshotDto?> RefreshUsageAsync()
=> TryInvokeAsync<UsageSnapshotDto>("RefreshUsage");
public async Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to)
=> await TryInvokeAsync<List<ModelUsageRowDto>>("GetModelUsage", from, to) ?? [];
@@ -40,6 +40,8 @@ public sealed partial class UsageMonitorModalViewModel : ViewModelBase
[NotifyPropertyChangedFor(nameof(ModelsEmpty))]
private bool _isBusy;
[ObservableProperty] private bool _isRefreshing;
[ObservableProperty]
[NotifyPropertyChangedFor(nameof(ModelsEmpty))]
private IReadOnlyList<ModelUsageDisplayRow> _modelRows = Array.Empty<ModelUsageDisplayRow>();
@@ -92,6 +94,28 @@ public sealed partial class UsageMonitorModalViewModel : ViewModelBase
private void OnUsageUpdated(UsageSnapshotDto snapshot) => Snapshot = snapshot;
/// <summary>
/// Manual refresh: the worker polls usage on a slow cadence (15 min idle / 5 min while a
/// task runs) to stay clear of the endpoint's 429s, so this is the way to get a number now.
/// </summary>
[RelayCommand]
private async Task RefreshAsync()
{
if (IsRefreshing) return;
IsRefreshing = true;
try
{
var snapshot = await _worker.RefreshUsageAsync();
if (snapshot is not null) Snapshot = snapshot;
await LoadUsageDataAsync();
}
catch (Exception ex)
{
ErrorReported?.Invoke(Loc.T("vm.usageMonitor.refreshFailed", ex.Message));
}
finally { IsRefreshing = false; }
}
private void ApplyPresetRange(int days)
{
var today = DateOnly.FromDateTime(DateTime.Today);
@@ -60,6 +60,18 @@
</Border>
</StackPanel>
<!-- Refresh: the worker polls on a slow cadence to avoid the endpoint's 429s -->
<StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Spacing="8"
Margin="20,12,20,0" VerticalAlignment="Center">
<Button Classes="btn" Content="{loc:Tr modals.usageMonitor.refresh}"
Command="{Binding RefreshCommand}"
IsEnabled="{Binding !IsRefreshing}"/>
<Ellipse Classes="spinner" Width="14" Height="14" VerticalAlignment="Center"
IsVisible="{Binding IsRefreshing}"/>
<TextBlock Classes="meta" VerticalAlignment="Center"
Text="{loc:Tr modals.usageMonitor.refreshHint}"/>
</StackPanel>
<!-- Gauges -->
<ItemsControl DockPanel.Dock="Top" Margin="20,12,20,0" ItemsSource="{Binding GaugeRows}">
<ItemsControl.ItemsPanel>
+14 -4
View File
@@ -44,9 +44,18 @@ public sealed class WorkerConfig
[JsonPropertyName("online_inbox")]
public OnlineInboxConfig OnlineInbox { get; set; } = new();
/// <summary>Poll interval for the OAuth usage monitor. Clamped to a minimum of 15s on load.</summary>
[JsonPropertyName("usage_poll_interval_seconds")]
public int UsagePollIntervalSeconds { get; set; } = 60;
/// <summary>
/// Usage-monitor poll interval while at least one task is Running. Clamped to a minimum
/// of 60s on load — the endpoint rate-limits (429) on tighter polling.
/// </summary>
[JsonPropertyName("usage_poll_interval_active_seconds")]
public int UsagePollIntervalActiveSeconds { get; set; } = 300;
/// <summary>
/// Usage-monitor poll interval while nothing is running. Clamped to a minimum of 60s on load.
/// </summary>
[JsonPropertyName("usage_poll_interval_idle_seconds")]
public int UsagePollIntervalIdleSeconds { get; set; } = 900;
public static string DefaultConfigPath =>
Path.Combine(Paths.AppDataRoot(), "worker.config.json");
@@ -75,7 +84,8 @@ public sealed class WorkerConfig
cfg.SandboxRoot = Paths.Expand(cfg.SandboxRoot);
cfg.LogRoot = Paths.Expand(cfg.LogRoot);
cfg.CentralWorktreeRoot = Paths.Expand(cfg.CentralWorktreeRoot);
cfg.UsagePollIntervalSeconds = Math.Max(15, cfg.UsagePollIntervalSeconds);
cfg.UsagePollIntervalActiveSeconds = Math.Max(60, cfg.UsagePollIntervalActiveSeconds);
cfg.UsagePollIntervalIdleSeconds = Math.Max(60, cfg.UsagePollIntervalIdleSeconds);
return cfg;
}
+16 -1
View File
@@ -189,6 +189,7 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
private readonly Data.Git.GitService? _git;
private readonly UsageSnapshotBuilder? _usageSnapshotBuilder;
private readonly ITranscriptUsageReader? _usageReader;
private readonly UsageMonitorService? _usageMonitor;
public WorkerHub(
QueueService queue,
@@ -220,7 +221,8 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
WorktreeManager? worktreeManager = null,
Data.Git.GitService? git = null,
UsageSnapshotBuilder? usageSnapshotBuilder = null,
ITranscriptUsageReader? usageReader = null)
ITranscriptUsageReader? usageReader = null,
UsageMonitorService? usageMonitor = null)
{
_queue = queue;
_waker = waker;
@@ -252,6 +254,7 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
_git = git;
_usageSnapshotBuilder = usageSnapshotBuilder;
_usageReader = usageReader;
_usageMonitor = usageMonitor;
}
// Persistence boundary for the session_skills JSON-array columns (task/list/global).
@@ -1049,6 +1052,18 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
return _usageSnapshotBuilder.BuildAsync(Context.ConnectionAborted);
});
/// <summary>
/// Manual "refresh now" for the usage monitor. Polls the endpoint out of band and returns the
/// fresh snapshot; a refresh inside the monitor's cooldown reuses the last poll's result
/// instead of risking a 429.
/// </summary>
public Task<UsageSnapshotDto> RefreshUsage() => HubGuard(() =>
{
if (_usageMonitor is null)
throw new InvalidOperationException("Usage monitor is not configured.");
return _usageMonitor.RefreshNowAsync(Context.ConnectionAborted);
});
public Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsage(DateOnly from, DateOnly to) => HubGuard(async () =>
{
if (_usageReader is null)
+4 -1
View File
@@ -209,7 +209,10 @@ builder.Services.AddHttpClient<IUsageClient, ClaudeOAuthUsageClient>(client =>
});
builder.Services.AddSingleton<IUsageGate, UsageGate>();
builder.Services.AddSingleton<UsageSnapshotBuilder>();
builder.Services.AddHostedService<UsageMonitorService>();
builder.Services.AddSingleton<IRunningTaskProbe, RunningTaskProbe>();
// Singleton + hosted service (same instance) so WorkerHub.RefreshUsage can drive a manual poll.
builder.Services.AddSingleton<UsageMonitorService>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<UsageMonitorService>());
// Loopback-only bind. Firewall is irrelevant for 127.0.0.1.
builder.WebHost.UseUrls($"http://127.0.0.1:{cfg.SignalRPort}");
@@ -1,3 +1,4 @@
using System.Net;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.Json.Nodes;
@@ -36,6 +37,8 @@ public sealed class ClaudeOAuthUsageClient : IUsageClient
request.Headers.Add("anthropic-beta", "oauth-2025-04-20");
using var response = await _http.SendAsync(request, ct);
if (response.StatusCode == HttpStatusCode.TooManyRequests)
throw new UsageRateLimitedException(ReadRetryAfter(response));
if (!response.IsSuccessStatusCode)
throw new InvalidOperationException($"Usage request failed with status {(int)response.StatusCode}.");
@@ -43,6 +46,27 @@ public sealed class ClaudeOAuthUsageClient : IUsageClient
return Parse(body);
}
/// <summary>
/// Reads the <c>Retry-After</c> header in either form (delta-seconds or HTTP-date).
/// A missing/past value returns null — the caller then uses its own backoff.
/// </summary>
internal static TimeSpan? ReadRetryAfter(HttpResponseMessage response)
{
var header = response.Headers.RetryAfter;
if (header is null) return null;
if (header.Delta is { } delta)
return delta > TimeSpan.Zero ? delta : null;
if (header.Date is { } date)
{
var remaining = date - DateTimeOffset.UtcNow;
return remaining > TimeSpan.Zero ? remaining : null;
}
return null;
}
private string ReadAccessToken()
{
if (!File.Exists(_credentialsPath))
@@ -162,3 +186,20 @@ public sealed class ClaudeOAuthUsageClient : IUsageClient
? dto
: null;
}
/// <summary>
/// The usage endpoint answered 429. Carries the server's <c>Retry-After</c> when it sent one so
/// <see cref="UsageMonitorService"/> can honour it instead of guessing a backoff.
/// </summary>
public sealed class UsageRateLimitedException : InvalidOperationException
{
public UsageRateLimitedException(TimeSpan? retryAfter)
: base(retryAfter is { } r
? $"Usage request was rate-limited (429); retry after {(int)r.TotalSeconds}s."
: "Usage request was rate-limited (429).")
{
RetryAfter = retryAfter;
}
public TimeSpan? RetryAfter { get; }
}
@@ -0,0 +1,10 @@
namespace ClaudeDo.Worker.Usage.Interfaces;
/// <summary>
/// Tells the usage monitor whether ClaudeDo is currently burning tokens, so it can poll
/// often while work is in flight and back off to a slow heartbeat while idle.
/// </summary>
public interface IRunningTaskProbe
{
Task<bool> AnyRunningAsync(CancellationToken ct = default);
}
@@ -0,0 +1,36 @@
using ClaudeDo.Data;
using ClaudeDo.Worker.Usage.Interfaces;
using Microsoft.EntityFrameworkCore;
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
namespace ClaudeDo.Worker.Usage;
/// <summary>
/// Answers "is anything running?" from the task table rather than the in-memory queue slots,
/// so override-slot runs, continued runs, and runs still marked Running after a worker restart
/// all count. A read failure reports idle — the usage monitor must never poll harder because
/// its activity probe broke.
/// </summary>
public sealed class RunningTaskProbe : IRunningTaskProbe
{
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
public RunningTaskProbe(IDbContextFactory<ClaudeDoDbContext> dbFactory) => _dbFactory = dbFactory;
public async Task<bool> AnyRunningAsync(CancellationToken ct = default)
{
try
{
await using var context = await _dbFactory.CreateDbContextAsync(ct);
return await context.Tasks.AnyAsync(t => t.Status == TaskStatus.Running, ct);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
throw;
}
catch
{
return false;
}
}
}
+134 -30
View File
@@ -5,25 +5,43 @@ using ClaudeDo.Worker.Usage.Interfaces;
namespace ClaudeDo.Worker.Usage;
/// <summary>
/// Polls <see cref="IUsageClient"/> on <see cref="WorkerConfig.UsagePollIntervalSeconds"/> and keeps
/// <see cref="UsageState"/> current. Polls once immediately at startup. A failure is logged as a
/// warning at most once per distinct error message, to avoid log spam on a persistent outage.
/// Broadcasts <see cref="HubBroadcaster.UsageUpdated"/> after every poll cycle, success or failure,
/// so the UI can reflect a stale/blocked state as soon as it happens.
/// Polls <see cref="IUsageClient"/> and keeps <see cref="UsageState"/> current. Polls once
/// immediately at startup, then on an **activity-dependent** interval: while any task is
/// Running it uses <see cref="WorkerConfig.UsagePollIntervalActiveSeconds"/>, otherwise the
/// slower <see cref="WorkerConfig.UsagePollIntervalIdleSeconds"/>. A 429 adds exponential
/// backoff on top (honouring <c>Retry-After</c> when the server sends one) — the endpoint is
/// undocumented and rate-limits aggressively. <see cref="RefreshNowAsync"/> gives the UI a
/// manual refresh that also resets the schedule, so the slow idle interval never leaves the
/// user staring at a stale number.
///
/// A failure is logged as a warning at most once per distinct error message, to avoid log spam
/// on a persistent outage. Broadcasts <see cref="HubBroadcaster.UsageUpdated"/> after every
/// poll cycle, success or failure, so the UI can reflect a stale/blocked state as soon as it
/// happens.
/// </summary>
public sealed class UsageMonitorService : BackgroundService
{
/// <summary>A manual refresh inside this window of the last poll reuses the last result.</summary>
internal static readonly TimeSpan ManualRefreshCooldown = TimeSpan.FromSeconds(10);
private readonly IUsageClient _client;
private readonly UsageState _state;
private readonly WorkerConfig _config;
private readonly ILogger<UsageMonitorService> _logger;
private readonly UsageSnapshotBuilder _snapshotBuilder;
private readonly HubBroadcaster _broadcaster;
private readonly IRunningTaskProbe _runningProbe;
// Serializes the background loop against a manual refresh so two polls never overlap.
private readonly SemaphoreSlim _pollLock = new(1, 1);
private string? _lastLoggedError;
private int _rateLimitStrikes;
private DateTime _lastPollUtc = DateTime.MinValue;
private DateTime _nextPollDueUtc = DateTime.MinValue;
public UsageMonitorService(
IUsageClient client, UsageState state, WorkerConfig config, ILogger<UsageMonitorService> logger,
UsageSnapshotBuilder snapshotBuilder, HubBroadcaster broadcaster)
UsageSnapshotBuilder snapshotBuilder, HubBroadcaster broadcaster, IRunningTaskProbe runningProbe)
{
_client = client;
_state = state;
@@ -31,49 +49,135 @@ public sealed class UsageMonitorService : BackgroundService
_logger = logger;
_snapshotBuilder = snapshotBuilder;
_broadcaster = broadcaster;
_runningProbe = runningProbe;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
await TickAsync(stoppingToken);
// Re-read the due time every iteration: a manual refresh pushes it out, which is
// how the loop avoids polling again right after the user hit refresh.
var wait = _nextPollDueUtc - DateTime.UtcNow;
if (wait > TimeSpan.Zero)
{
try
{
await Task.Delay(wait, stoppingToken);
}
catch (OperationCanceledException)
{
return;
}
continue;
}
try
{
await Task.Delay(TimeSpan.FromSeconds(_config.UsagePollIntervalSeconds), stoppingToken);
}
catch (OperationCanceledException)
{
return;
}
await TickAsync(stoppingToken);
}
}
internal async Task TickAsync(CancellationToken ct)
/// <summary>
/// Forces a poll now and returns the resulting snapshot DTO. Within
/// <see cref="ManualRefreshCooldown"/> of the last poll the API call is skipped and the
/// current state is returned instead, so click-spamming the refresh button can't earn a 429.
/// </summary>
public async Task<UsageSnapshotDto> RefreshNowAsync(CancellationToken ct = default)
{
await PollAsync(ct, ManualRefreshCooldown);
return await _snapshotBuilder.BuildAsync(ct);
}
internal Task TickAsync(CancellationToken ct) => PollAsync(ct, null);
private async Task PollAsync(CancellationToken ct, TimeSpan? skipIfPolledWithin)
{
await _pollLock.WaitAsync(ct);
TimeSpan? retryAfter = null;
try
{
var snapshot = await _client.GetUsageAsync(ct);
_state.ReportSuccess(snapshot);
_lastLoggedError = null;
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
_state.ReportFailure(ex.Message, DateTime.UtcNow);
// Checked under the lock so a manual refresh that queued behind a background poll
// sees that poll's timestamp and reuses its result instead of firing a second call.
if (skipIfPolledWithin is { } window && DateTime.UtcNow - _lastPollUtc < window)
return;
if (_lastLoggedError != ex.Message)
try
{
_logger.LogWarning(ex, "UsageMonitorService: failed to fetch usage: {Message}", ex.Message);
_lastLoggedError = ex.Message;
var snapshot = await _client.GetUsageAsync(ct);
_state.ReportSuccess(snapshot);
_lastLoggedError = null;
_rateLimitStrikes = 0;
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
throw;
}
catch (UsageRateLimitedException ex)
{
_rateLimitStrikes = Math.Min(_rateLimitStrikes + 1, UsagePollSchedule.MaxStrikes);
retryAfter = ex.RetryAfter;
RecordFailure(ex);
}
catch (Exception ex)
{
RecordFailure(ex);
}
_lastPollUtc = DateTime.UtcNow;
_nextPollDueUtc = _lastPollUtc + await NextDelayAsync(retryAfter, ct);
}
finally
{
_pollLock.Release();
}
var dto = await _snapshotBuilder.BuildAsync(ct);
await _broadcaster.UsageUpdated(dto);
}
private void RecordFailure(Exception ex)
{
_state.ReportFailure(ex.Message, DateTime.UtcNow);
if (_lastLoggedError == ex.Message) return;
_logger.LogWarning(ex, "UsageMonitorService: failed to fetch usage: {Message}", ex.Message);
_lastLoggedError = ex.Message;
}
private async Task<TimeSpan> NextDelayAsync(TimeSpan? retryAfter, CancellationToken ct)
{
var anyRunning = await _runningProbe.AnyRunningAsync(ct);
return UsagePollSchedule.NextDelay(
anyRunning,
_config.UsagePollIntervalActiveSeconds,
_config.UsagePollIntervalIdleSeconds,
_rateLimitStrikes,
retryAfter);
}
}
/// <summary>
/// Pure poll-interval arithmetic for <see cref="UsageMonitorService"/>: active-vs-idle base
/// interval plus 429 backoff. Kept static and side-effect-free so the schedule is testable
/// without a running background service.
/// </summary>
internal static class UsagePollSchedule
{
/// <summary>Strike count is capped so the exponent can't run away on a long outage.</summary>
internal const int MaxStrikes = 4;
/// <summary>Nothing ever waits longer than this, not even an absurd <c>Retry-After</c>.</summary>
internal static readonly TimeSpan MaxDelay = TimeSpan.FromMinutes(30);
internal static TimeSpan NextDelay(
bool anyTaskRunning, int activeSeconds, int idleSeconds, int rateLimitStrikes, TimeSpan? retryAfter)
{
var baseDelay = TimeSpan.FromSeconds(Math.Max(1, anyTaskRunning ? activeSeconds : idleSeconds));
if (rateLimitStrikes <= 0)
return baseDelay;
var backoff = retryAfter ?? baseDelay * Math.Pow(2, Math.Min(rateLimitStrikes, MaxStrikes));
// Never poll *sooner* than the normal cadence just because Retry-After was small.
if (backoff < baseDelay) backoff = baseDelay;
return backoff > MaxDelay ? MaxDelay : backoff;
}
}
@@ -39,7 +39,10 @@ public sealed class UsageSnapshotBuilder
var decision = await _gate.EvaluateAsync(ct);
var maxAge = TimeSpan.FromSeconds(_cfg.UsagePollIntervalSeconds * 3);
// Measured against the *slowest* cadence — the idle interval — so a genuinely idle
// worker on its 15-minute heartbeat isn't reported stale just for not polling.
var maxAge = TimeSpan.FromSeconds(
Math.Max(_cfg.UsagePollIntervalActiveSeconds, _cfg.UsagePollIntervalIdleSeconds) * 3);
var isStale = snapshot is null || lastError is not null || (DateTime.UtcNow - snapshot.FetchedAtUtc) > maxAge;
var limits = (snapshot?.Limits ?? Array.Empty<UsageLimitRow>())