fix(usage): stop 429s with an activity-dependent poll cadence + manual refresh
The usage monitor polled the undocumented OAuth usage endpoint every 60s and earned 429s. It now polls every 5 min while any task is Running and every 15 min while idle (usage_poll_interval_active_seconds / _idle_seconds, both clamped to >= 60; the old single usage_poll_interval_seconds key is gone). A 429 comes back as UsageRateLimitedException carrying Retry-After and adds exponential backoff on top, capped at 30 min and never shorter than the normal cadence; the strike count resets on the first success. The schedule arithmetic is the pure static UsagePollSchedule.NextDelay. Since the idle cadence is slow on purpose, WorkerHub.RefreshUsage drives UsageMonitorService.RefreshNowAsync behind a Refresh now button in the Usage Monitor modal: an out-of-band poll that pushes the loop's next-due time out so no double poll follows, with a 10s cooldown so click-spam can't earn a 429. Staleness now measures against the slower (idle) interval so an idle worker isn't flagged stale just for not polling.
This commit is contained in:
@@ -108,6 +108,20 @@ Offene Entscheidungen dazu:
|
||||
undokumentiert und kann sich ändern; bei Ausfall/Formatänderung ist das Gate wirkungslos
|
||||
(fail-open by design — kein Blocker, aber der Schutz fällt dann aus, ohne dass es auffällt).
|
||||
|
||||
### Nachtrag 2026-08-05: 429-Fix (Poll-Kadenz + Refresh-Button)
|
||||
|
||||
Der 60s-Poll lief in 429s. Neu: aktivitätsabhängige Kadenz (5 Min. solange ein Task `Running`
|
||||
ist, sonst 15 Min.), 429-Backoff mit `Retry-After`, und ein „Jetzt aktualisieren"-Button im
|
||||
Usage-Monitor-Modal (`RefreshUsage` → `UsageMonitorService.RefreshNowAsync`, 10s-Cooldown).
|
||||
Unit-Tests grün, **offen**:
|
||||
|
||||
- **Visueller Pass Refresh-Button** im Modal (Button + Spinner + Hinweiszeile, Dark/Light,
|
||||
en/de) — Teil des oben schon offenen Modal-Passes.
|
||||
- **E2E:** über ≥20 Min. mit und ohne laufenden Task beobachten, dass keine 429s mehr im
|
||||
Worker-Log auftauchen und die Pill trotzdem aktuell bleibt.
|
||||
- **Beachten:** die Pill wird jetzt erst nach 3× 15 Min. als `stale` markiert — ein echter
|
||||
Endpoint-Ausfall fällt vorher nur über `LastError` auf (der `IsStale` sofort setzt).
|
||||
|
||||
## Offene Verifikation (2026-08-05, Max-Turns-Ceiling)
|
||||
|
||||
Build + unit tests grün (`ResolveMaxTurns`-Klemmung, Repository-Backfill von `model_presets`,
|
||||
|
||||
@@ -431,6 +431,8 @@
|
||||
"title": "USAGE MONITOR",
|
||||
"windowTitle": "Usage Monitor",
|
||||
"noGauges": "Noch keine Nutzungslimits gemeldet.",
|
||||
"refresh": "Jetzt aktualisieren",
|
||||
"refreshHint": "Abruf alle 5 Min., während ein Task läuft, sonst alle 15 Min.",
|
||||
"staleFormat": "Werte veraltet (Stand {0})",
|
||||
"staleGateHint": "Das Gate greift in diesem Zustand nicht.",
|
||||
"gateBlockedFormat": "Queue pausiert — {0}",
|
||||
@@ -598,7 +600,10 @@
|
||||
"settingsModal": { "workerOffline": "Worker offline — Einstellungen schreibgeschützt.", "saveFailed": "Speichern fehlgeschlagen: {0}" },
|
||||
"onlineInbox": { "workerOffline": "Worker offline — Konfiguration kann nicht geladen werden.", "saved": "Konfiguration gespeichert.", "saveFailed": "Speichern fehlgeschlagen: {0}", "signedIn": "Erfolgreich angemeldet.", "signedInNoRole": "Angemeldet, aber diesem Konto fehlt die Rolle 'user' in Zitadel — die Online-Synchronisierung wird abgelehnt, bis die Rolle im ClaudeDo-Projekt zugewiesen wird.", "signInFailed": "Anmeldung fehlgeschlagen: {0}", "signedOut": "Abgemeldet.", "signOutFailed": "Abmeldung fehlgeschlagen: {0}" },
|
||||
"weeklyReport": { "invalidRange": "Ungültiger Datumsbereich.", "generating": "Bericht wird erstellt…", "error": "Fehler: {0}" },
|
||||
"usageMonitor": { "loadFailed": "Nutzungsdaten konnten nicht geladen werden: {0}" },
|
||||
"usageMonitor": {
|
||||
"loadFailed": "Nutzungsdaten konnten nicht geladen werden: {0}",
|
||||
"refreshFailed": "Nutzung konnte nicht aktualisiert werden: {0}"
|
||||
},
|
||||
"filesTab": { "workerOffline": "Worker offline.", "noneBundled": "Keine Standard-Agenten mitgeliefert.", "allPresent": "Alle Standard-Agenten bereits vorhanden.", "restored": "{0} Standard-Agent(en) wiederhergestellt.", "restoreFailed": "Wiederherstellung fehlgeschlagen: {0}", "openFailed": "Öffnen fehlgeschlagen: {0}", "resetToDefault": "Auf den mitgelieferten Standard zurückgesetzt." },
|
||||
"sessionSkillsTab": { "installed": "Installiert: {0}", "installFailed": "Installation fehlgeschlagen: {0}", "updated": "Aktualisiert.", "updateFailed": "Aktualisierung fehlgeschlagen: {0}", "removed": "Entfernt.", "removeFailed": "Entfernen fehlgeschlagen: {0}" },
|
||||
"worktreesTab": { "workerOffline": "Worker offline.", "removed": "{0} Worktree(s) entfernt.", "blocked": "Zwangsentfernung nicht möglich: {0} Aufgabe(n) laufen noch. Brich sie zuerst ab.", "removedFrom": "{0} Worktree(s) von {1} Aufgabe(n) entfernt.", "cleanupFailed": "Aufräumen fehlgeschlagen: {0}", "resetFailed": "Zurücksetzen fehlgeschlagen: {0}" },
|
||||
|
||||
@@ -431,6 +431,8 @@
|
||||
"title": "USAGE MONITOR",
|
||||
"windowTitle": "Usage Monitor",
|
||||
"noGauges": "No usage limits reported yet.",
|
||||
"refresh": "Refresh now",
|
||||
"refreshHint": "Polled every 5 min while a task runs, otherwise every 15 min.",
|
||||
"staleFormat": "Values stale (as of {0})",
|
||||
"staleGateHint": "The gate does not apply while values are stale.",
|
||||
"gateBlockedFormat": "Queue paused — {0}",
|
||||
@@ -598,7 +600,10 @@
|
||||
"settingsModal": { "workerOffline": "Worker offline — settings read-only.", "saveFailed": "Save failed: {0}" },
|
||||
"onlineInbox": { "workerOffline": "Worker offline — cannot load config.", "saved": "Config saved.", "saveFailed": "Save failed: {0}", "signedIn": "Signed in successfully.", "signedInNoRole": "Signed in, but this account is missing the 'user' role in Zitadel — online sync will be rejected until the role is granted in the ClaudeDo project.", "signInFailed": "Sign-in failed: {0}", "signedOut": "Signed out.", "signOutFailed": "Sign-out failed: {0}" },
|
||||
"weeklyReport": { "invalidRange": "Invalid date range.", "generating": "Generating report…", "error": "Error: {0}" },
|
||||
"usageMonitor": { "loadFailed": "Couldn't load usage data: {0}" },
|
||||
"usageMonitor": {
|
||||
"loadFailed": "Couldn't load usage data: {0}",
|
||||
"refreshFailed": "Couldn't refresh usage: {0}"
|
||||
},
|
||||
"filesTab": { "workerOffline": "Worker offline.", "noneBundled": "No default agents bundled.", "allPresent": "All default agents already present.", "restored": "Restored {0} default agent(s).", "restoreFailed": "Restore failed: {0}", "openFailed": "Open failed: {0}", "resetToDefault": "Reset to the bundled default." },
|
||||
"sessionSkillsTab": { "installed": "Installed: {0}", "installFailed": "Install failed: {0}", "updated": "Updated.", "updateFailed": "Update failed: {0}", "removed": "Removed.", "removeFailed": "Remove failed: {0}" },
|
||||
"worktreesTab": { "workerOffline": "Worker offline.", "removed": "Removed {0} worktree(s).", "blocked": "Cannot force-remove: {0} task(s) still running. Cancel them first.", "removedFrom": "Removed {0} worktree(s) from {1} task(s).", "cleanupFailed": "Cleanup failed: {0}", "resetFailed": "Reset failed: {0}" },
|
||||
|
||||
@@ -40,13 +40,13 @@ Design/ — Tokens.axaml (design tokens; merged before styles) + IslandStyle
|
||||
- **TaskRowViewModel** / **ListNavItemViewModel** — lightweight display VMs (task row: status, planning phase, parent/blocked links, roadblock count, computed `IsDraft`/`IsPlanned`/`IsChild`/`IsPlanningParent`/`CanRefine`; plus `IsManual` (→ MANUAL badge; suppresses `CanSendToQueue`/`CanRefine`/`CanOpenPlanningSession`) and `HasInteractiveSession` (→ accent "Interactive" chip instead of "Parked"; tapping it jumps to that Mission Control pane); list row: kind Smart/Virtual/User, count, icon/dot keys, drop hints, `IsManual`).
|
||||
- **NotesEditorViewModel** — day navigator + bullet CRUD for daily notes via `INotesApi`.
|
||||
- **UsagePillViewModel** — one shared instance backs the `UsagePill` control hosted in both the footer and the Mission Control header; loads via `GetUsageSnapshotAsync` and updates live off `IWorkerClient.UsageUpdatedEvent`; derives display text, tooltip, and dot state (normal/warn/stale/blocked, mutually exclusive priority blocked > stale > warn > normal) from the shared `UsageSnapshotDto`. `IsThrottled` (effective slots below configured, and not gate-blocked) adds a tooltip line naming the effective/configured slot count and the decisive bucket (`ThrottleBucket` on the DTO — `"five_hour"`/`"seven_day"`).
|
||||
- **Modal VMs** — `SettingsModalViewModel` (four tabs: General, Worktrees, Files prompt-paths, Prime Claude incl. `DailyPrepMaxTasks` + prime-schedule rows). General hosts the per-model preset table (`ModelPresets` → `ModelPresetRowViewModel`: effort + max turns per alias) which **replaced** the single global "Max turns" field, `ListSettingsModalViewModel` (name, working dir, commit type, "manual list" flag, `VerifyCommand` (optional post-merge verify gate, own field/section — not part of `AgentConfigEditorViewModel`), delete list; hosts shared `AgentConfigEditorViewModel` as `Agent` property (scope=List) — save delegates to `Agent.SaveAsync(verifyCommand)`, since both fields land in the same `list_config` row via one `UpdateListConfig` call and would otherwise clobber each other), `RepoImportModalViewModel` (bulk-create lists from git repos found under chosen parents; already-wired repos disabled), `WeeklyReportModalViewModel` (range pickers default "since last standup weekday → today", cached per range, markdown via MarkdownView), `MergeModalViewModel` (single-task merge form, called from the diff modal), `WorktreesOverviewModalViewModel` (global/per-list worktree rows, batch merge + state ops), `UnfinishedPlanningModalViewModel` (Resume/FinalizeNow/Discard for a draft planning session), `MergeHelperSelectionModalViewModel` ("Let Claude handle it": checkbox picker over one list's non-terminal, non-manual tasks, pre-ticks the actionable ones; list-scoped only — `Configure(listId, listName)`, no global scope. Opened from the list row's context menu, which is hidden when the list has no working dir; on confirm `ListsIslandViewModel` raises `LetClaudeHandleRequested` → shell → `MissionControlViewModel.OpenMergeHelperConPtySessionAsync`, which first calls `IWorkerClient.CreateMergeHelperTaskAsync` to create one new ClaudeDo task per run in that list — `Idle`/`IsManual=true` (never queued), title/description localized (`missionControl.mergeHelperTaskTitle`/`mergeHelperTaskDescriptionHeader`), `TaskEntity.HandlerBaseCommit` stamped to the list repo's current HEAD — then opens a **task-based** ConPTY tile for it (deduped by `TaskId` like `OpenConPtySessionAsync`, not `CreateAdHoc`) running the five-phase handler prompt. The handler still merges the tasks it handles itself; the host task never gets a worktree of its own, so "Submit for review" stamps `HandlerHeadCommit` instead of committing a worktree, and the detail pane's `MergeSectionViewModel` falls back to `HandlerBaseCommit`/`HandlerHeadCommit` over the list's working dir for its diff — see `TaskEntity` in `ClaudeDo.Data/CLAUDE.md`), `WorkerConnectionModalViewModel` (offline help), `AboutModalViewModel`, `LogVisualizerViewModel` (worker logs, last 30 min, all levels + a warn/error-only filter; loads via `GetRecentLogsAsync`), `UsageMonitorModalViewModel` (opened from the usage pill; renders one gauge per row in `UsageSnapshotDto.Limits` — **dynamic**, since the fixed `seven_day_opus`/`seven_day_sonnet`-style buckets the raw Anthropic API can return are plan-dependent and come back `null` on plans that don't have them, so a fixed gauge layout would break; also shows model usage (`GetModelUsageAsync`, ClaudeDo-vs-Other split per model) and top-task usage (`GetTaskUsageAsync`) tables over a 7d/30d preset or custom date range).
|
||||
- **Modal VMs** — `SettingsModalViewModel` (four tabs: General, Worktrees, Files prompt-paths, Prime Claude incl. `DailyPrepMaxTasks` + prime-schedule rows). General hosts the per-model preset table (`ModelPresets` → `ModelPresetRowViewModel`: effort + max turns per alias) which **replaced** the single global "Max turns" field, `ListSettingsModalViewModel` (name, working dir, commit type, "manual list" flag, `VerifyCommand` (optional post-merge verify gate, own field/section — not part of `AgentConfigEditorViewModel`), delete list; hosts shared `AgentConfigEditorViewModel` as `Agent` property (scope=List) — save delegates to `Agent.SaveAsync(verifyCommand)`, since both fields land in the same `list_config` row via one `UpdateListConfig` call and would otherwise clobber each other), `RepoImportModalViewModel` (bulk-create lists from git repos found under chosen parents; already-wired repos disabled), `WeeklyReportModalViewModel` (range pickers default "since last standup weekday → today", cached per range, markdown via MarkdownView), `MergeModalViewModel` (single-task merge form, called from the diff modal), `WorktreesOverviewModalViewModel` (global/per-list worktree rows, batch merge + state ops), `UnfinishedPlanningModalViewModel` (Resume/FinalizeNow/Discard for a draft planning session), `MergeHelperSelectionModalViewModel` ("Let Claude handle it": checkbox picker over one list's non-terminal, non-manual tasks, pre-ticks the actionable ones; list-scoped only — `Configure(listId, listName)`, no global scope. Opened from the list row's context menu, which is hidden when the list has no working dir; on confirm `ListsIslandViewModel` raises `LetClaudeHandleRequested` → shell → `MissionControlViewModel.OpenMergeHelperConPtySessionAsync`, which first calls `IWorkerClient.CreateMergeHelperTaskAsync` to create one new ClaudeDo task per run in that list — `Idle`/`IsManual=true` (never queued), title/description localized (`missionControl.mergeHelperTaskTitle`/`mergeHelperTaskDescriptionHeader`), `TaskEntity.HandlerBaseCommit` stamped to the list repo's current HEAD — then opens a **task-based** ConPTY tile for it (deduped by `TaskId` like `OpenConPtySessionAsync`, not `CreateAdHoc`) running the five-phase handler prompt. The handler still merges the tasks it handles itself; the host task never gets a worktree of its own, so "Submit for review" stamps `HandlerHeadCommit` instead of committing a worktree, and the detail pane's `MergeSectionViewModel` falls back to `HandlerBaseCommit`/`HandlerHeadCommit` over the list's working dir for its diff — see `TaskEntity` in `ClaudeDo.Data/CLAUDE.md`), `WorkerConnectionModalViewModel` (offline help), `AboutModalViewModel`, `LogVisualizerViewModel` (worker logs, last 30 min, all levels + a warn/error-only filter; loads via `GetRecentLogsAsync`), `UsageMonitorModalViewModel` (opened from the usage pill; renders one gauge per row in `UsageSnapshotDto.Limits` — **dynamic**, since the fixed `seven_day_opus`/`seven_day_sonnet`-style buckets the raw Anthropic API can return are plan-dependent and come back `null` on plans that don't have them, so a fixed gauge layout would break; also shows model usage (`GetModelUsageAsync`, ClaudeDo-vs-Other split per model) and top-task usage (`GetTaskUsageAsync`) tables over a 7d/30d preset or custom date range. A **Refresh now** button (`RefreshCommand`/`IsRefreshing`) drives `RefreshUsageAsync` — the worker deliberately polls slowly (5 min while a task runs, 15 min idle) to stay clear of the endpoint's 429s, so this is the only way to pull a current number on demand).
|
||||
- **Diff stack** — `UnifiedDiffParser` (static; parses `git diff` output into `DiffFileViewModel`s, detecting added/deleted/renamed/binary files and per-line numbers; `Flatten` injects file-header rows for a combined single-pane view). `DiffModels.cs` holds shared types: `DiffLineViewModel`, `DiffFileViewModel`, `DiffLineKind`, `DiffFileStatus`, `SubtaskDiffRow`, `DiffTreeNodeViewModel`, `DiffTree`. `DiffViewerViewModel` is a single unified read-only diff viewer with two modes: **Files** (dirty worktree / branch-vs-base / commit-range — loads via GitService, shows a folder file-tree on the left + per-file diff pane on the right, Merge button for live branch source) and **Planning** (per-subtask diffs via `GetPlanningAggregateAsync`, subtask list left + flat diff right, combined integration-branch toggle). The Merge button opens the merge form, which routes to `ConflictResolverViewModel` on conflict. `DiffLinesView` renders per-file diff content with binary/empty placeholders.
|
||||
- **Conflicts** — `ConflictResolverViewModel` (in-app **Rider-style 3-pane merge editor** for both single-task and planning unit-merge conflicts: single-task starts the conflict merge, parses each conflicted file into stable/conflict `MergeFileSegment`s via the worker's `GetMergeConflictDocuments`; exposes the active file's three reconstructed documents — `ActiveOursText` / `ActiveResultText` / `ActiveTheirsText` (from `MergeFile.OursText/ResultText/TheirsText`; Result seeds unresolved conflicts with Ours) — plus `ActiveFile`/`SelectFileCommand` (multi-file switcher), `Current`/`Next`/`Previous` (focused-conflict nav), a per-active-file `PositionText` readout, per-block `AcceptOurs/Theirs/Both/Base` + `MergeFile.Compose`, and `CanContinue` gated on every file resolved + no binary; writes each file via `WriteConflictResolution`, continue/abort; **planning mode** via `OpenForPlanningAsync(parentId, subtaskId)` loads the current subtask's mid-merge conflicts without re-starting the merge and routes continue/abort to `ContinuePlanningMerge`/`AbortPlanningMerge`, so a unit-merge conflict re-opens the editor per subtask via the `PlanningMergeConflict` broadcast). The view (`Views/Conflicts/ConflictResolverView`) shows the whole file in three **AvaloniaEdit** panes — MAIN/ours (read-only) | editable Result | INCOMING/theirs (read-only) — with TextMate highlighting by extension (theme `StyleInclude` in `App.axaml`); a code-behind `IBackgroundRenderer` tints each conflict block (unresolved/resolved) across panes, an `IReadOnlySectionProvider` + `TextAnchor` regions keep only conflict spans editable in Result (edits flow back to the block); each unresolved conflict starts EMPTY (a thin marker bar); the between-pane gutter controls **toggle** each side in/out of the result — `›`/`‹` add MAIN/INCOMING in click order (first pick on top), clicking again removes that side — so a conflict can take main, incoming, both, or neither; a `FilesSummary` readout shows how many files still have conflicts, and the three panes share a proportional synced vertical scroll. A conflict overview ruler right of the Result pane (`ConflictMap`) maps every conflict in the file proportionally (click a tick to jump) — handy for long files. Conflict block tints live in `Tokens.axaml` (`Merge*TintBrush`). The editor is reached from review **Approve** on conflict and from the **Merge** button in the Diff window (a conflicting `MergeTask` hands off to the resolver via `RequestConflictResolution`).
|
||||
|
||||
## Services
|
||||
|
||||
- **WorkerClient** / **IWorkerClient** — SignalR client connecting to `http://127.0.0.1:47821/hub`, auto-reconnect with exponential backoff. The surface tracks `WorkerHub` (see `src/ClaudeDo.Worker/CLAUDE.md` for the canonical method/event list); groups: task execution (RunNow/Cancel/Continue/Reset/SetTaskStatus), review (`ApproveReviewAsync(taskId, targetBranch) -> MergeResultDto`, reject-to-queue/idle, cancel review, `PreviewMergeAsync -> MergePreviewDto`), planning sessions (start/resume/discard/finalize, queue subtasks, pending draft count, refine), embedded ConPTY launch specs (`GetInteractiveLaunchSpecAsync`/`GetAdHocLaunchSpecAsync`), planning aggregate/integration-branch diffs, unit-merge continue/abort, single-task conflict resolving (start/get-conflict-documents/write-resolution/continue/abort), worktrees (overview, set state, force remove, cleanup, reset all), agents, app settings, lists/config, weekly report, daily notes, daily prep (`RunDailyPrepNowAsync`, `ClearMyDayAsync`, `GetLastPrepLogAsync`), prime schedules, recent worker logs (`GetRecentLogsAsync`), usage monitoring (`GetUsageSnapshotAsync -> UsageSnapshotDto?`, `GetModelUsageAsync(from, to)`, `GetTaskUsageAsync(from, to)`). Events mirror `HubBroadcaster` (task/worktree/list/run updates, prep events, planning-merge events, refine events, worker log, `UsageUpdatedEvent`). Lifecycle (`StartAsync`/`StopAsync`) and a few admin methods live only on the concrete `WorkerClient`.
|
||||
- **WorkerClient** / **IWorkerClient** — SignalR client connecting to `http://127.0.0.1:47821/hub`, auto-reconnect with exponential backoff. The surface tracks `WorkerHub` (see `src/ClaudeDo.Worker/CLAUDE.md` for the canonical method/event list); groups: task execution (RunNow/Cancel/Continue/Reset/SetTaskStatus), review (`ApproveReviewAsync(taskId, targetBranch) -> MergeResultDto`, reject-to-queue/idle, cancel review, `PreviewMergeAsync -> MergePreviewDto`), planning sessions (start/resume/discard/finalize, queue subtasks, pending draft count, refine), embedded ConPTY launch specs (`GetInteractiveLaunchSpecAsync`/`GetAdHocLaunchSpecAsync`), planning aggregate/integration-branch diffs, unit-merge continue/abort, single-task conflict resolving (start/get-conflict-documents/write-resolution/continue/abort), worktrees (overview, set state, force remove, cleanup, reset all), agents, app settings, lists/config, weekly report, daily notes, daily prep (`RunDailyPrepNowAsync`, `ClearMyDayAsync`, `GetLastPrepLogAsync`), prime schedules, recent worker logs (`GetRecentLogsAsync`), usage monitoring (`GetUsageSnapshotAsync -> UsageSnapshotDto?`, `RefreshUsageAsync -> UsageSnapshotDto?` (manual out-of-band poll), `GetModelUsageAsync(from, to)`, `GetTaskUsageAsync(from, to)`). Events mirror `HubBroadcaster` (task/worktree/list/run updates, prep events, planning-merge events, refine events, worker log, `UsageUpdatedEvent`). Lifecycle (`StartAsync`/`StopAsync`) and a few admin methods live only on the concrete `WorkerClient`.
|
||||
- **INotesApi** / **WorkerNotesApi** — daily-note CRUD (`ListAsync(day)`, `AddAsync`, `UpdateAsync`, `DeleteAsync`); UI DTO `DailyNoteDto(Id, Date, Text, SortOrder)`.
|
||||
- **IPrimeScheduleApi** — prime-schedule CRUD (`ListAsync`, `UpsertAsync`, `DeleteAsync`).
|
||||
- **UpdateCheckService** — polls releases, exposes `LastCheckStatus`/`LatestVersion`/`CheckNowAsync` (feeds the shell's update banner).
|
||||
|
||||
@@ -143,6 +143,9 @@ public interface IWorkerClient : INotifyPropertyChanged
|
||||
/// <summary>Raised whenever the worker's usage poller ticks (success or failure).</summary>
|
||||
event Action<UsageSnapshotDto>? UsageUpdatedEvent;
|
||||
Task<UsageSnapshotDto?> GetUsageSnapshotAsync();
|
||||
|
||||
/// <summary>Forces an out-of-band usage poll on the worker and returns the fresh snapshot.</summary>
|
||||
Task<UsageSnapshotDto?> RefreshUsageAsync();
|
||||
Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to);
|
||||
Task<IReadOnlyList<TaskUsageRowDto>> GetTaskUsageAsync(DateOnly from, DateOnly to);
|
||||
}
|
||||
|
||||
@@ -582,6 +582,9 @@ public partial class WorkerClient : ObservableObject, IAsyncDisposable, IWorkerC
|
||||
public Task<UsageSnapshotDto?> GetUsageSnapshotAsync()
|
||||
=> TryInvokeAsync<UsageSnapshotDto>("GetUsageSnapshot");
|
||||
|
||||
public Task<UsageSnapshotDto?> RefreshUsageAsync()
|
||||
=> TryInvokeAsync<UsageSnapshotDto>("RefreshUsage");
|
||||
|
||||
public async Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to)
|
||||
=> await TryInvokeAsync<List<ModelUsageRowDto>>("GetModelUsage", from, to) ?? [];
|
||||
|
||||
|
||||
@@ -40,6 +40,8 @@ public sealed partial class UsageMonitorModalViewModel : ViewModelBase
|
||||
[NotifyPropertyChangedFor(nameof(ModelsEmpty))]
|
||||
private bool _isBusy;
|
||||
|
||||
[ObservableProperty] private bool _isRefreshing;
|
||||
|
||||
[ObservableProperty]
|
||||
[NotifyPropertyChangedFor(nameof(ModelsEmpty))]
|
||||
private IReadOnlyList<ModelUsageDisplayRow> _modelRows = Array.Empty<ModelUsageDisplayRow>();
|
||||
@@ -92,6 +94,28 @@ public sealed partial class UsageMonitorModalViewModel : ViewModelBase
|
||||
|
||||
private void OnUsageUpdated(UsageSnapshotDto snapshot) => Snapshot = snapshot;
|
||||
|
||||
/// <summary>
|
||||
/// Manual refresh: the worker polls usage on a slow cadence (15 min idle / 5 min while a
|
||||
/// task runs) to stay clear of the endpoint's 429s, so this is the way to get a number now.
|
||||
/// </summary>
|
||||
[RelayCommand]
|
||||
private async Task RefreshAsync()
|
||||
{
|
||||
if (IsRefreshing) return;
|
||||
IsRefreshing = true;
|
||||
try
|
||||
{
|
||||
var snapshot = await _worker.RefreshUsageAsync();
|
||||
if (snapshot is not null) Snapshot = snapshot;
|
||||
await LoadUsageDataAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ErrorReported?.Invoke(Loc.T("vm.usageMonitor.refreshFailed", ex.Message));
|
||||
}
|
||||
finally { IsRefreshing = false; }
|
||||
}
|
||||
|
||||
private void ApplyPresetRange(int days)
|
||||
{
|
||||
var today = DateOnly.FromDateTime(DateTime.Today);
|
||||
|
||||
@@ -60,6 +60,18 @@
|
||||
</Border>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Refresh: the worker polls on a slow cadence to avoid the endpoint's 429s -->
|
||||
<StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Spacing="8"
|
||||
Margin="20,12,20,0" VerticalAlignment="Center">
|
||||
<Button Classes="btn" Content="{loc:Tr modals.usageMonitor.refresh}"
|
||||
Command="{Binding RefreshCommand}"
|
||||
IsEnabled="{Binding !IsRefreshing}"/>
|
||||
<Ellipse Classes="spinner" Width="14" Height="14" VerticalAlignment="Center"
|
||||
IsVisible="{Binding IsRefreshing}"/>
|
||||
<TextBlock Classes="meta" VerticalAlignment="Center"
|
||||
Text="{loc:Tr modals.usageMonitor.refreshHint}"/>
|
||||
</StackPanel>
|
||||
|
||||
<!-- Gauges -->
|
||||
<ItemsControl DockPanel.Dock="Top" Margin="20,12,20,0" ItemsSource="{Binding GaugeRows}">
|
||||
<ItemsControl.ItemsPanel>
|
||||
|
||||
@@ -21,7 +21,7 @@ Worker/
|
||||
Report/ — ClaudeHistoryReader, WeekReportPromptBuilder, WeekReportService; interfaces in Report/Interfaces/
|
||||
Prime/ — daily-prep ("Prime Claude"): PrimeScheduler (BackgroundService), PrimeRunner (runs the daily prep), DailyPrepPrompt (fixed prompt + CLI args + LogPath() helper), NextDueCalculator, PrimeScheduleSignal; interfaces in Prime/Interfaces/ (IPrimeRunner, IPrimeClock, IPrimeScheduleSignal, IPrimeBroadcaster)
|
||||
Online/ — optional Online Inbox sync: OnlineInboxConfig (config record), Dtos (RemoteList/RemoteTask/MirrorTask), IOnlineInboxApi, OnlineInboxApiClient (typed HttpClient, bearer auth, HTTPS guard), OnlineTokenStore (DPAPI refresh-token store, Windows-only), StaticTokenAuthProvider (default/test IOnlineAuthProvider), ZitadelAuthProvider (OIDC discovery + refresh-token flow), OnlineSyncService (BackgroundService: reconcile loop), OnlineBacklog (Idle-backlog filter/query); interface in Online/Interfaces/ (IOnlineAuthProvider)
|
||||
Usage/ — OAuth usage monitor: UsageModels (UsageBucket/UsageLimitRow/UsageSnapshot), ClaudeOAuthUsageClient (reads the access token Claude Code keeps fresh at `~/.claude/.credentials.json`, calls `GET https://api.anthropic.com/api/oauth/usage`; defensive parsing — missing/null buckets → null, missing `limits` → empty list; never logs the token), UsageState (threadsafe singleton; a failed poll never overwrites the last good snapshot, only sets `LastError`), UsageMonitorService (BackgroundService, polls on `usage_poll_interval_seconds`, one poll at startup, logs a failure at most once per distinct error message, broadcasts `HubBroadcaster.UsageUpdated` after every tick via `UsageSnapshotBuilder`), UsageSnapshotBuilder (builds the Hub-facing `UsageSnapshotDto` from `UsageState` + `IUsageGate` + `AppSettings` thresholds — the one place `WorkerHub.GetUsageSnapshot` and `UsageMonitorService` share the stale/threshold/gate logic), TranscriptUsageReader (aggregates Claude Code transcript token usage from `~/.claude/projects/**/*.jsonl` by date/model/scope (ClaudeDo vs Other), deduped by requestId, with a per-file length+mtime cache; `<synthetic>`-model lines are skipped everywhere, not real API calls; also exposes `ReadSessionTotalsAsync(sessionId)` — cumulative raw input/output/cache-read/cache-write totals for one session, located by `{sessionId}.jsonl`, used by `TaskRunner` to populate `task_runs`' per-run token fields), UsageGate (reads `UsageState` + `AppSettings.UsageGateFiveHourPct`/`UsageGateSevenDayPct`, returns a `UsageGateDecision(IsBlocked, Reason)`; `Utilization` from `UsageBucket` is already a 0–100 percent, compared directly against the threshold with `>=`; threshold `0` = that bucket never gates; fail-open — no snapshot yet, a failed last poll, or a settings-read error all resolve to not-blocked), UsageThrottle (pure static `EffectiveSlots(configuredSlots, fiveHourPct, sevenDayPct, softPct, hardPct, gateFiveHourPct, gateSevenDayPct)` — stages parallelism down ahead of the hard gate: whichever bucket is more utilized decides the stage, `0` for a threshold disables that stage, `>= softPct` caps at 2 slots, `>= hardPct` caps at 1, `>= either gate threshold` returns 0 — same 0 as `UsageGate`'s hard block, kept in sync since both read the same gate thresholds); interfaces in Usage/Interfaces/ (IUsageClient, ITranscriptUsageReader, IUsageGate)
|
||||
Usage/ — OAuth usage monitor: UsageModels (UsageBucket/UsageLimitRow/UsageSnapshot), ClaudeOAuthUsageClient (reads the access token Claude Code keeps fresh at `~/.claude/.credentials.json`, calls `GET https://api.anthropic.com/api/oauth/usage`; defensive parsing — missing/null buckets → null, missing `limits` → empty list; never logs the token; a 429 becomes `UsageRateLimitedException` carrying the parsed `Retry-After`, delta or HTTP-date, null when absent/past), UsageState (threadsafe singleton; a failed poll never overwrites the last good snapshot, only sets `LastError`), UsageMonitorService (singleton + BackgroundService, one poll at startup then an activity-dependent interval with 429 backoff, `RefreshNowAsync` for the UI's manual refresh, logs a failure at most once per distinct error message, broadcasts `HubBroadcaster.UsageUpdated` after every tick via `UsageSnapshotBuilder`) + UsagePollSchedule (pure static `NextDelay`, see below), RunningTaskProbe (`IRunningTaskProbe` — `Tasks.Any(Status == Running)`, so override-slot/continued/post-restart runs all count; a read failure reports idle so a broken probe can never make the monitor poll harder), UsageSnapshotBuilder (builds the Hub-facing `UsageSnapshotDto` from `UsageState` + `IUsageGate` + `AppSettings` thresholds — the one place `WorkerHub.GetUsageSnapshot` and `UsageMonitorService` share the stale/threshold/gate logic), TranscriptUsageReader (aggregates Claude Code transcript token usage from `~/.claude/projects/**/*.jsonl` by date/model/scope (ClaudeDo vs Other), deduped by requestId, with a per-file length+mtime cache; `<synthetic>`-model lines are skipped everywhere, not real API calls; also exposes `ReadSessionTotalsAsync(sessionId)` — cumulative raw input/output/cache-read/cache-write totals for one session, located by `{sessionId}.jsonl`, used by `TaskRunner` to populate `task_runs`' per-run token fields), UsageGate (reads `UsageState` + `AppSettings.UsageGateFiveHourPct`/`UsageGateSevenDayPct`, returns a `UsageGateDecision(IsBlocked, Reason)`; `Utilization` from `UsageBucket` is already a 0–100 percent, compared directly against the threshold with `>=`; threshold `0` = that bucket never gates; fail-open — no snapshot yet, a failed last poll, or a settings-read error all resolve to not-blocked), UsageThrottle (pure static `EffectiveSlots(configuredSlots, fiveHourPct, sevenDayPct, softPct, hardPct, gateFiveHourPct, gateSevenDayPct)` — stages parallelism down ahead of the hard gate: whichever bucket is more utilized decides the stage, `0` for a threshold disables that stage, `>= softPct` caps at 2 slots, `>= hardPct` caps at 1, `>= either gate threshold` returns 0 — same 0 as `UsageGate`'s hard block, kept in sync since both read the same gate thresholds); interfaces in Usage/Interfaces/ (IUsageClient, ITranscriptUsageReader, IUsageGate, IRunningTaskProbe)
|
||||
```
|
||||
|
||||
Interfaces (e.g. `IQueueWaker`, `IPrimeClock`, `ITaskStateService`) live in an `Interfaces/` subfolder within their area; the namespace stays the area namespace.
|
||||
@@ -65,8 +65,22 @@ Interfaces (e.g. `IQueueWaker`, `IPrimeClock`, `ITaskStateService`) live in an `
|
||||
Source: `GET https://api.anthropic.com/api/oauth/usage`, an **undocumented** Anthropic API,
|
||||
authenticated with the Bearer access token Claude Code itself keeps fresh at
|
||||
`~/.claude/.credentials.json` — ClaudeDo reads that token but never refreshes it and never
|
||||
logs it. `UsageMonitorService` polls on `usage_poll_interval_seconds` (default 60s, one poll
|
||||
at startup too) and broadcasts `HubBroadcaster.UsageUpdated` after every tick.
|
||||
logs it. The endpoint **rate-limits (429)**, so `UsageMonitorService` polls on an
|
||||
activity-dependent cadence: one poll at startup, then `usage_poll_interval_active_seconds`
|
||||
(default 300) while `IRunningTaskProbe` reports any task `Running`, otherwise
|
||||
`usage_poll_interval_idle_seconds` (default 900). A 429 (surfaced by the client as
|
||||
`UsageRateLimitedException`) adds exponential backoff on top — the server's `Retry-After` when
|
||||
it sends one, else `base × 2^strikes` — clamped to `UsagePollSchedule.MaxDelay` (30 min) and
|
||||
never shorter than the normal cadence; the strike count resets on the first success. The
|
||||
schedule arithmetic itself is the pure static `UsagePollSchedule.NextDelay`. Every tick
|
||||
broadcasts `HubBroadcaster.UsageUpdated`.
|
||||
|
||||
Because the idle cadence is deliberately slow, `WorkerHub.RefreshUsage` →
|
||||
`UsageMonitorService.RefreshNowAsync` gives the UI a manual "refresh now" (the Usage Monitor
|
||||
modal's button): it polls out of band, pushes the loop's next-due time out so no double poll
|
||||
follows, and within `ManualRefreshCooldown` (10 s) of the last poll reuses that poll's result
|
||||
instead of hitting the endpoint — click-spam can't earn a 429. A poll and a manual refresh are
|
||||
serialized against each other by a semaphore.
|
||||
|
||||
The gate (`IUsageGate`, thresholds `usage_gate_five_hour_pct`/`usage_gate_seven_day_pct`)
|
||||
pauses **only** the queue's slot-fill loop (new tasks don't start) once `five_hour >=
|
||||
@@ -220,7 +234,7 @@ Each CLI invocation is recorded in the `task_runs` table via `TaskRunRepository`
|
||||
- Agents/settings/lists: `GetAgents`, `RefreshAgents`, `RestoreDefaultAgents`, `GetAppSettings`, `UpdateAppSettings`, `UpdateList`, `UpdateListConfig`, `GetListConfig`, `UpdateTaskAgentSettings`
|
||||
- Reports/notes/prep: `GetWeekReport`, `GenerateWeekReport`, `GetDailyNotes`, `AddDailyNote`, `UpdateDailyNote`, `DeleteDailyNote`, `RunDailyPrepNow`, `ClearMyDay`, `GetLastPrepLog`, `ListPrimeSchedules`, `UpsertPrimeSchedule`, `DeletePrimeSchedule`
|
||||
- Diagnostics: `GetRecentLogs` (last 30 min of buffered log records, all levels, for the Log Visualizer overlay)
|
||||
- Usage: `GetUsageSnapshot() -> UsageSnapshotDto` (built by `UsageSnapshotBuilder` from `UsageState` + `IUsageGate` + `AppSettings` gate thresholds; percentages/limits/`FetchedAtUtc` null and `IsStale=true` when no snapshot has landed yet; `IsStale` also trips on a failed last poll or a snapshot older than 3× `usage_poll_interval_seconds`), `GetModelUsage(from, to) -> IReadOnlyList<ModelUsageRowDto>` (thin wrapper over `ITranscriptUsageReader.ReadAsync`), `GetTaskUsage(from, to) -> IReadOnlyList<TaskUsageRowDto>` (top consumers from `task_runs` joined to task/list, grouped per task — `Runs`/summed `TokensIn`/`TokensOut` (null token columns count as 0, never dropped), `Model` from that task's most recent run — sorted by total tokens descending, capped at 100)
|
||||
- Usage: `GetUsageSnapshot() -> UsageSnapshotDto` (built by `UsageSnapshotBuilder` from `UsageState` + `IUsageGate` + `AppSettings` gate thresholds; percentages/limits/`FetchedAtUtc` null and `IsStale=true` when no snapshot has landed yet; `IsStale` also trips on a failed last poll or a snapshot older than 3× the *slower* of the two poll intervals — measured against the idle cadence so an idle worker isn't flagged stale for simply not polling), `RefreshUsage() -> UsageSnapshotDto` (manual out-of-band poll via `UsageMonitorService.RefreshNowAsync`; cooldown-guarded), `GetModelUsage(from, to) -> IReadOnlyList<ModelUsageRowDto>` (thin wrapper over `ITranscriptUsageReader.ReadAsync`), `GetTaskUsage(from, to) -> IReadOnlyList<TaskUsageRowDto>` (top consumers from `task_runs` joined to task/list, grouped per task — `Runs`/summed `TokensIn`/`TokensOut` (null token columns count as 0, never dropped), `Model` from that task's most recent run — sorted by total tokens descending, capped at 100)
|
||||
|
||||
**HubBroadcaster** events: `TaskStarted`, `TaskFinished`, `TaskMessage`, `WorktreeUpdated`, `TaskUpdated`, `RunCreated`, `ListUpdated`, `WorkerLog`, `PrimeFired`, `PrepStarted`, `PrepLine`, `PrepFinished`, `PlanningMergeStarted`, `PlanningSubtaskMerged`, `PlanningMergeConflict`, `PlanningMergeAborted`, `PlanningCompleted`, `RefineStarted`, `RefineFinished`, `UsageUpdated` (carries the same `UsageSnapshotDto` as `GetUsageSnapshot`; `UsageMonitorService` fires it after every poll cycle, success or failure, via the shared `UsageSnapshotBuilder`)
|
||||
|
||||
@@ -240,7 +254,7 @@ Loaded from `~/.todo-app/worker.config.json`:
|
||||
- `poll_interval_seconds` (int, default 60)
|
||||
- `zitadel.authority`, `zitadel.client_id`, `zitadel.scopes` — used by `ZitadelAuthProvider` (OIDC discovery + refresh-token flow)
|
||||
- The refresh token is NOT in this file — stored encrypted via DPAPI at `~/.todo-app/online-inbox.token`
|
||||
- `usage_poll_interval_seconds` (default 60, clamped to a minimum of 15 on load) — poll interval for `UsageMonitorService`
|
||||
- `usage_poll_interval_active_seconds` (default 300) / `usage_poll_interval_idle_seconds` (default 900) — `UsageMonitorService`'s poll interval while at least one task is `Running` vs. while idle. Both clamped to a minimum of 60 on load; the endpoint answers 429 on tighter polling. Replaces the old single `usage_poll_interval_seconds` (silently ignored if still present in an existing config file).
|
||||
|
||||
Per-list config (`list_config` in DB) provides defaults for `model`, `system_prompt`, `agent_path`; tasks can override each individually. Task-generating MCP tools (`AddTask`, planning `CreateChildTask`, `SuggestImprovement`) accept an optional `model` (alias-validated via `ModelRegistry.NormalizeAlias` — `haiku`/`sonnet`/`opus`, blank = inherit) so Claude assigns the cheapest capable model at creation time; the planning/system/improvement prompts instruct it to do so (`ModelRegistry.ByCostAscending` = the cost order). Planning's `CreateChildTask` additionally accepts an optional `maxTurns` (positive int; `0`/negative rejected with `ArgumentException`, null = inherit list/global default) so the planner can raise the turn budget for a subtask it knows will run long; `SuggestImprovement`/`AddTask` do not expose it.
|
||||
|
||||
|
||||
@@ -44,9 +44,18 @@ public sealed class WorkerConfig
|
||||
[JsonPropertyName("online_inbox")]
|
||||
public OnlineInboxConfig OnlineInbox { get; set; } = new();
|
||||
|
||||
/// <summary>Poll interval for the OAuth usage monitor. Clamped to a minimum of 15s on load.</summary>
|
||||
[JsonPropertyName("usage_poll_interval_seconds")]
|
||||
public int UsagePollIntervalSeconds { get; set; } = 60;
|
||||
/// <summary>
|
||||
/// Usage-monitor poll interval while at least one task is Running. Clamped to a minimum
|
||||
/// of 60s on load — the endpoint rate-limits (429) on tighter polling.
|
||||
/// </summary>
|
||||
[JsonPropertyName("usage_poll_interval_active_seconds")]
|
||||
public int UsagePollIntervalActiveSeconds { get; set; } = 300;
|
||||
|
||||
/// <summary>
|
||||
/// Usage-monitor poll interval while nothing is running. Clamped to a minimum of 60s on load.
|
||||
/// </summary>
|
||||
[JsonPropertyName("usage_poll_interval_idle_seconds")]
|
||||
public int UsagePollIntervalIdleSeconds { get; set; } = 900;
|
||||
|
||||
public static string DefaultConfigPath =>
|
||||
Path.Combine(Paths.AppDataRoot(), "worker.config.json");
|
||||
@@ -75,7 +84,8 @@ public sealed class WorkerConfig
|
||||
cfg.SandboxRoot = Paths.Expand(cfg.SandboxRoot);
|
||||
cfg.LogRoot = Paths.Expand(cfg.LogRoot);
|
||||
cfg.CentralWorktreeRoot = Paths.Expand(cfg.CentralWorktreeRoot);
|
||||
cfg.UsagePollIntervalSeconds = Math.Max(15, cfg.UsagePollIntervalSeconds);
|
||||
cfg.UsagePollIntervalActiveSeconds = Math.Max(60, cfg.UsagePollIntervalActiveSeconds);
|
||||
cfg.UsagePollIntervalIdleSeconds = Math.Max(60, cfg.UsagePollIntervalIdleSeconds);
|
||||
|
||||
return cfg;
|
||||
}
|
||||
|
||||
@@ -189,6 +189,7 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
|
||||
private readonly Data.Git.GitService? _git;
|
||||
private readonly UsageSnapshotBuilder? _usageSnapshotBuilder;
|
||||
private readonly ITranscriptUsageReader? _usageReader;
|
||||
private readonly UsageMonitorService? _usageMonitor;
|
||||
|
||||
public WorkerHub(
|
||||
QueueService queue,
|
||||
@@ -220,7 +221,8 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
|
||||
WorktreeManager? worktreeManager = null,
|
||||
Data.Git.GitService? git = null,
|
||||
UsageSnapshotBuilder? usageSnapshotBuilder = null,
|
||||
ITranscriptUsageReader? usageReader = null)
|
||||
ITranscriptUsageReader? usageReader = null,
|
||||
UsageMonitorService? usageMonitor = null)
|
||||
{
|
||||
_queue = queue;
|
||||
_waker = waker;
|
||||
@@ -252,6 +254,7 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
|
||||
_git = git;
|
||||
_usageSnapshotBuilder = usageSnapshotBuilder;
|
||||
_usageReader = usageReader;
|
||||
_usageMonitor = usageMonitor;
|
||||
}
|
||||
|
||||
// Persistence boundary for the session_skills JSON-array columns (task/list/global).
|
||||
@@ -1049,6 +1052,18 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
|
||||
return _usageSnapshotBuilder.BuildAsync(Context.ConnectionAborted);
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
/// Manual "refresh now" for the usage monitor. Polls the endpoint out of band and returns the
|
||||
/// fresh snapshot; a refresh inside the monitor's cooldown reuses the last poll's result
|
||||
/// instead of risking a 429.
|
||||
/// </summary>
|
||||
public Task<UsageSnapshotDto> RefreshUsage() => HubGuard(() =>
|
||||
{
|
||||
if (_usageMonitor is null)
|
||||
throw new InvalidOperationException("Usage monitor is not configured.");
|
||||
return _usageMonitor.RefreshNowAsync(Context.ConnectionAborted);
|
||||
});
|
||||
|
||||
public Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsage(DateOnly from, DateOnly to) => HubGuard(async () =>
|
||||
{
|
||||
if (_usageReader is null)
|
||||
|
||||
@@ -209,7 +209,10 @@ builder.Services.AddHttpClient<IUsageClient, ClaudeOAuthUsageClient>(client =>
|
||||
});
|
||||
builder.Services.AddSingleton<IUsageGate, UsageGate>();
|
||||
builder.Services.AddSingleton<UsageSnapshotBuilder>();
|
||||
builder.Services.AddHostedService<UsageMonitorService>();
|
||||
builder.Services.AddSingleton<IRunningTaskProbe, RunningTaskProbe>();
|
||||
// Singleton + hosted service (same instance) so WorkerHub.RefreshUsage can drive a manual poll.
|
||||
builder.Services.AddSingleton<UsageMonitorService>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<UsageMonitorService>());
|
||||
|
||||
// Loopback-only bind. Firewall is irrelevant for 127.0.0.1.
|
||||
builder.WebHost.UseUrls($"http://127.0.0.1:{cfg.SignalRPort}");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
@@ -36,6 +37,8 @@ public sealed class ClaudeOAuthUsageClient : IUsageClient
|
||||
request.Headers.Add("anthropic-beta", "oauth-2025-04-20");
|
||||
|
||||
using var response = await _http.SendAsync(request, ct);
|
||||
if (response.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
throw new UsageRateLimitedException(ReadRetryAfter(response));
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new InvalidOperationException($"Usage request failed with status {(int)response.StatusCode}.");
|
||||
|
||||
@@ -43,6 +46,27 @@ public sealed class ClaudeOAuthUsageClient : IUsageClient
|
||||
return Parse(body);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the <c>Retry-After</c> header in either form (delta-seconds or HTTP-date).
|
||||
/// A missing/past value returns null — the caller then uses its own backoff.
|
||||
/// </summary>
|
||||
internal static TimeSpan? ReadRetryAfter(HttpResponseMessage response)
|
||||
{
|
||||
var header = response.Headers.RetryAfter;
|
||||
if (header is null) return null;
|
||||
|
||||
if (header.Delta is { } delta)
|
||||
return delta > TimeSpan.Zero ? delta : null;
|
||||
|
||||
if (header.Date is { } date)
|
||||
{
|
||||
var remaining = date - DateTimeOffset.UtcNow;
|
||||
return remaining > TimeSpan.Zero ? remaining : null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private string ReadAccessToken()
|
||||
{
|
||||
if (!File.Exists(_credentialsPath))
|
||||
@@ -162,3 +186,20 @@ public sealed class ClaudeOAuthUsageClient : IUsageClient
|
||||
? dto
|
||||
: null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The usage endpoint answered 429. Carries the server's <c>Retry-After</c> when it sent one so
|
||||
/// <see cref="UsageMonitorService"/> can honour it instead of guessing a backoff.
|
||||
/// </summary>
|
||||
public sealed class UsageRateLimitedException : InvalidOperationException
|
||||
{
|
||||
public UsageRateLimitedException(TimeSpan? retryAfter)
|
||||
: base(retryAfter is { } r
|
||||
? $"Usage request was rate-limited (429); retry after {(int)r.TotalSeconds}s."
|
||||
: "Usage request was rate-limited (429).")
|
||||
{
|
||||
RetryAfter = retryAfter;
|
||||
}
|
||||
|
||||
public TimeSpan? RetryAfter { get; }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace ClaudeDo.Worker.Usage.Interfaces;
|
||||
|
||||
/// <summary>
|
||||
/// Tells the usage monitor whether ClaudeDo is currently burning tokens, so it can poll
|
||||
/// often while work is in flight and back off to a slow heartbeat while idle.
|
||||
/// </summary>
|
||||
public interface IRunningTaskProbe
|
||||
{
|
||||
Task<bool> AnyRunningAsync(CancellationToken ct = default);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using ClaudeDo.Data;
|
||||
using ClaudeDo.Worker.Usage.Interfaces;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
|
||||
|
||||
namespace ClaudeDo.Worker.Usage;
|
||||
|
||||
/// <summary>
|
||||
/// Answers "is anything running?" from the task table rather than the in-memory queue slots,
|
||||
/// so override-slot runs, continued runs, and runs still marked Running after a worker restart
|
||||
/// all count. A read failure reports idle — the usage monitor must never poll harder because
|
||||
/// its activity probe broke.
|
||||
/// </summary>
|
||||
public sealed class RunningTaskProbe : IRunningTaskProbe
|
||||
{
|
||||
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||
|
||||
public RunningTaskProbe(IDbContextFactory<ClaudeDoDbContext> dbFactory) => _dbFactory = dbFactory;
|
||||
|
||||
public async Task<bool> AnyRunningAsync(CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var context = await _dbFactory.CreateDbContextAsync(ct);
|
||||
return await context.Tasks.AnyAsync(t => t.Status == TaskStatus.Running, ct);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,25 +5,43 @@ using ClaudeDo.Worker.Usage.Interfaces;
|
||||
namespace ClaudeDo.Worker.Usage;
|
||||
|
||||
/// <summary>
|
||||
/// Polls <see cref="IUsageClient"/> on <see cref="WorkerConfig.UsagePollIntervalSeconds"/> and keeps
|
||||
/// <see cref="UsageState"/> current. Polls once immediately at startup. A failure is logged as a
|
||||
/// warning at most once per distinct error message, to avoid log spam on a persistent outage.
|
||||
/// Broadcasts <see cref="HubBroadcaster.UsageUpdated"/> after every poll cycle, success or failure,
|
||||
/// so the UI can reflect a stale/blocked state as soon as it happens.
|
||||
/// Polls <see cref="IUsageClient"/> and keeps <see cref="UsageState"/> current. Polls once
|
||||
/// immediately at startup, then on an **activity-dependent** interval: while any task is
|
||||
/// Running it uses <see cref="WorkerConfig.UsagePollIntervalActiveSeconds"/>, otherwise the
|
||||
/// slower <see cref="WorkerConfig.UsagePollIntervalIdleSeconds"/>. A 429 adds exponential
|
||||
/// backoff on top (honouring <c>Retry-After</c> when the server sends one) — the endpoint is
|
||||
/// undocumented and rate-limits aggressively. <see cref="RefreshNowAsync"/> gives the UI a
|
||||
/// manual refresh that also resets the schedule, so the slow idle interval never leaves the
|
||||
/// user staring at a stale number.
|
||||
///
|
||||
/// A failure is logged as a warning at most once per distinct error message, to avoid log spam
|
||||
/// on a persistent outage. Broadcasts <see cref="HubBroadcaster.UsageUpdated"/> after every
|
||||
/// poll cycle, success or failure, so the UI can reflect a stale/blocked state as soon as it
|
||||
/// happens.
|
||||
/// </summary>
|
||||
public sealed class UsageMonitorService : BackgroundService
|
||||
{
|
||||
/// <summary>A manual refresh inside this window of the last poll reuses the last result.</summary>
|
||||
internal static readonly TimeSpan ManualRefreshCooldown = TimeSpan.FromSeconds(10);
|
||||
|
||||
private readonly IUsageClient _client;
|
||||
private readonly UsageState _state;
|
||||
private readonly WorkerConfig _config;
|
||||
private readonly ILogger<UsageMonitorService> _logger;
|
||||
private readonly UsageSnapshotBuilder _snapshotBuilder;
|
||||
private readonly HubBroadcaster _broadcaster;
|
||||
private readonly IRunningTaskProbe _runningProbe;
|
||||
|
||||
// Serializes the background loop against a manual refresh so two polls never overlap.
|
||||
private readonly SemaphoreSlim _pollLock = new(1, 1);
|
||||
private string? _lastLoggedError;
|
||||
private int _rateLimitStrikes;
|
||||
private DateTime _lastPollUtc = DateTime.MinValue;
|
||||
private DateTime _nextPollDueUtc = DateTime.MinValue;
|
||||
|
||||
public UsageMonitorService(
|
||||
IUsageClient client, UsageState state, WorkerConfig config, ILogger<UsageMonitorService> logger,
|
||||
UsageSnapshotBuilder snapshotBuilder, HubBroadcaster broadcaster)
|
||||
UsageSnapshotBuilder snapshotBuilder, HubBroadcaster broadcaster, IRunningTaskProbe runningProbe)
|
||||
{
|
||||
_client = client;
|
||||
_state = state;
|
||||
@@ -31,49 +49,135 @@ public sealed class UsageMonitorService : BackgroundService
|
||||
_logger = logger;
|
||||
_snapshotBuilder = snapshotBuilder;
|
||||
_broadcaster = broadcaster;
|
||||
_runningProbe = runningProbe;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
await TickAsync(stoppingToken);
|
||||
// Re-read the due time every iteration: a manual refresh pushes it out, which is
|
||||
// how the loop avoids polling again right after the user hit refresh.
|
||||
var wait = _nextPollDueUtc - DateTime.UtcNow;
|
||||
if (wait > TimeSpan.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(wait, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(_config.UsagePollIntervalSeconds), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
await TickAsync(stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task TickAsync(CancellationToken ct)
|
||||
/// <summary>
|
||||
/// Forces a poll now and returns the resulting snapshot DTO. Within
|
||||
/// <see cref="ManualRefreshCooldown"/> of the last poll the API call is skipped and the
|
||||
/// current state is returned instead, so click-spamming the refresh button can't earn a 429.
|
||||
/// </summary>
|
||||
public async Task<UsageSnapshotDto> RefreshNowAsync(CancellationToken ct = default)
|
||||
{
|
||||
await PollAsync(ct, ManualRefreshCooldown);
|
||||
return await _snapshotBuilder.BuildAsync(ct);
|
||||
}
|
||||
|
||||
internal Task TickAsync(CancellationToken ct) => PollAsync(ct, null);
|
||||
|
||||
private async Task PollAsync(CancellationToken ct, TimeSpan? skipIfPolledWithin)
|
||||
{
|
||||
await _pollLock.WaitAsync(ct);
|
||||
TimeSpan? retryAfter = null;
|
||||
try
|
||||
{
|
||||
var snapshot = await _client.GetUsageAsync(ct);
|
||||
_state.ReportSuccess(snapshot);
|
||||
_lastLoggedError = null;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_state.ReportFailure(ex.Message, DateTime.UtcNow);
|
||||
// Checked under the lock so a manual refresh that queued behind a background poll
|
||||
// sees that poll's timestamp and reuses its result instead of firing a second call.
|
||||
if (skipIfPolledWithin is { } window && DateTime.UtcNow - _lastPollUtc < window)
|
||||
return;
|
||||
|
||||
if (_lastLoggedError != ex.Message)
|
||||
try
|
||||
{
|
||||
_logger.LogWarning(ex, "UsageMonitorService: failed to fetch usage: {Message}", ex.Message);
|
||||
_lastLoggedError = ex.Message;
|
||||
var snapshot = await _client.GetUsageAsync(ct);
|
||||
_state.ReportSuccess(snapshot);
|
||||
_lastLoggedError = null;
|
||||
_rateLimitStrikes = 0;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (UsageRateLimitedException ex)
|
||||
{
|
||||
_rateLimitStrikes = Math.Min(_rateLimitStrikes + 1, UsagePollSchedule.MaxStrikes);
|
||||
retryAfter = ex.RetryAfter;
|
||||
RecordFailure(ex);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
}
|
||||
|
||||
_lastPollUtc = DateTime.UtcNow;
|
||||
_nextPollDueUtc = _lastPollUtc + await NextDelayAsync(retryAfter, ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_pollLock.Release();
|
||||
}
|
||||
|
||||
var dto = await _snapshotBuilder.BuildAsync(ct);
|
||||
await _broadcaster.UsageUpdated(dto);
|
||||
}
|
||||
|
||||
private void RecordFailure(Exception ex)
|
||||
{
|
||||
_state.ReportFailure(ex.Message, DateTime.UtcNow);
|
||||
|
||||
if (_lastLoggedError == ex.Message) return;
|
||||
_logger.LogWarning(ex, "UsageMonitorService: failed to fetch usage: {Message}", ex.Message);
|
||||
_lastLoggedError = ex.Message;
|
||||
}
|
||||
|
||||
private async Task<TimeSpan> NextDelayAsync(TimeSpan? retryAfter, CancellationToken ct)
|
||||
{
|
||||
var anyRunning = await _runningProbe.AnyRunningAsync(ct);
|
||||
return UsagePollSchedule.NextDelay(
|
||||
anyRunning,
|
||||
_config.UsagePollIntervalActiveSeconds,
|
||||
_config.UsagePollIntervalIdleSeconds,
|
||||
_rateLimitStrikes,
|
||||
retryAfter);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure poll-interval arithmetic for <see cref="UsageMonitorService"/>: active-vs-idle base
|
||||
/// interval plus 429 backoff. Kept static and side-effect-free so the schedule is testable
|
||||
/// without a running background service.
|
||||
/// </summary>
|
||||
internal static class UsagePollSchedule
|
||||
{
|
||||
/// <summary>Strike count is capped so the exponent can't run away on a long outage.</summary>
|
||||
internal const int MaxStrikes = 4;
|
||||
|
||||
/// <summary>Nothing ever waits longer than this, not even an absurd <c>Retry-After</c>.</summary>
|
||||
internal static readonly TimeSpan MaxDelay = TimeSpan.FromMinutes(30);
|
||||
|
||||
internal static TimeSpan NextDelay(
|
||||
bool anyTaskRunning, int activeSeconds, int idleSeconds, int rateLimitStrikes, TimeSpan? retryAfter)
|
||||
{
|
||||
var baseDelay = TimeSpan.FromSeconds(Math.Max(1, anyTaskRunning ? activeSeconds : idleSeconds));
|
||||
if (rateLimitStrikes <= 0)
|
||||
return baseDelay;
|
||||
|
||||
var backoff = retryAfter ?? baseDelay * Math.Pow(2, Math.Min(rateLimitStrikes, MaxStrikes));
|
||||
// Never poll *sooner* than the normal cadence just because Retry-After was small.
|
||||
if (backoff < baseDelay) backoff = baseDelay;
|
||||
return backoff > MaxDelay ? MaxDelay : backoff;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,7 +39,10 @@ public sealed class UsageSnapshotBuilder
|
||||
|
||||
var decision = await _gate.EvaluateAsync(ct);
|
||||
|
||||
var maxAge = TimeSpan.FromSeconds(_cfg.UsagePollIntervalSeconds * 3);
|
||||
// Measured against the *slowest* cadence — the idle interval — so a genuinely idle
|
||||
// worker on its 15-minute heartbeat isn't reported stale just for not polling.
|
||||
var maxAge = TimeSpan.FromSeconds(
|
||||
Math.Max(_cfg.UsagePollIntervalActiveSeconds, _cfg.UsagePollIntervalIdleSeconds) * 3);
|
||||
var isStale = snapshot is null || lastError is not null || (DateTime.UtcNow - snapshot.FetchedAtUtc) > maxAge;
|
||||
|
||||
var limits = (snapshot?.Limits ?? Array.Empty<UsageLimitRow>())
|
||||
|
||||
@@ -153,6 +153,7 @@ public abstract class StubWorkerClient : IWorkerClient
|
||||
public virtual Task ClearOnlineInboxAuthAsync() => Task.CompletedTask;
|
||||
|
||||
public virtual Task<UsageSnapshotDto?> GetUsageSnapshotAsync() => Task.FromResult<UsageSnapshotDto?>(null);
|
||||
public virtual Task<UsageSnapshotDto?> RefreshUsageAsync() => Task.FromResult<UsageSnapshotDto?>(null);
|
||||
public virtual Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to)
|
||||
=> Task.FromResult<IReadOnlyList<ModelUsageRowDto>>(Array.Empty<ModelUsageRowDto>());
|
||||
public virtual Task<IReadOnlyList<TaskUsageRowDto>> GetTaskUsageAsync(DateOnly from, DateOnly to)
|
||||
|
||||
@@ -29,8 +29,19 @@ public class UsageMonitorModalViewModelTests
|
||||
public int ModelUsageCalls;
|
||||
public int TaskUsageCalls;
|
||||
|
||||
public UsageSnapshotDto? RefreshedSnapshot;
|
||||
public int RefreshCalls;
|
||||
public Exception? RefreshThrows;
|
||||
|
||||
public override Task<UsageSnapshotDto?> GetUsageSnapshotAsync() => Task.FromResult(Snapshot);
|
||||
|
||||
public override Task<UsageSnapshotDto?> RefreshUsageAsync()
|
||||
{
|
||||
RefreshCalls++;
|
||||
if (RefreshThrows is not null) throw RefreshThrows;
|
||||
return Task.FromResult(RefreshedSnapshot);
|
||||
}
|
||||
|
||||
public override Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to)
|
||||
{
|
||||
ModelUsageCalls++;
|
||||
@@ -68,6 +79,60 @@ public class UsageMonitorModalViewModelTests
|
||||
isGateBlocked, gateReason, fetchedAtUtc ?? DateTime.UtcNow, isStale, lastError,
|
||||
configuredSlots, effectiveSlots, throttleBucket);
|
||||
|
||||
// ── Manual refresh ──────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_ReplacesSnapshotAndReloadsTables()
|
||||
{
|
||||
var worker = new FakeWorker
|
||||
{
|
||||
Snapshot = Snapshot(new[] { Limit("session") }),
|
||||
RefreshedSnapshot = Snapshot(new[] { Limit("session"), Limit("weekly_all") }),
|
||||
};
|
||||
var vm = new UsageMonitorModalViewModel(worker);
|
||||
await vm.LoadAsync();
|
||||
var callsAfterLoad = worker.ModelUsageCalls;
|
||||
|
||||
await vm.RefreshCommand.ExecuteAsync(null);
|
||||
|
||||
Assert.Equal(1, worker.RefreshCalls);
|
||||
Assert.Equal(2, vm.GaugeRows.Count);
|
||||
Assert.Equal(callsAfterLoad + 1, worker.ModelUsageCalls);
|
||||
Assert.False(vm.IsRefreshing);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_NullResult_KeepsPreviousSnapshot()
|
||||
{
|
||||
var worker = new FakeWorker { Snapshot = Snapshot(new[] { Limit("session") }) };
|
||||
var vm = new UsageMonitorModalViewModel(worker);
|
||||
await vm.LoadAsync();
|
||||
|
||||
await vm.RefreshCommand.ExecuteAsync(null);
|
||||
|
||||
Assert.Single(vm.GaugeRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_Failure_ReportsErrorAndClearsBusyFlag()
|
||||
{
|
||||
var worker = new FakeWorker
|
||||
{
|
||||
Snapshot = Snapshot(new[] { Limit("session") }),
|
||||
RefreshThrows = new InvalidOperationException("worker offline"),
|
||||
};
|
||||
var vm = new UsageMonitorModalViewModel(worker);
|
||||
await vm.LoadAsync();
|
||||
string? reported = null;
|
||||
vm.ErrorReported += m => reported = m;
|
||||
|
||||
await vm.RefreshCommand.ExecuteAsync(null);
|
||||
|
||||
Assert.NotNull(reported);
|
||||
Assert.Contains("worker offline", reported);
|
||||
Assert.False(vm.IsRefreshing);
|
||||
}
|
||||
|
||||
// ── Gauge label derivation ──────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -145,6 +145,7 @@ sealed class FakeWorkerClient : IWorkerClient
|
||||
public IReadOnlyList<ActiveTask> GetActiveTasks() => System.Array.Empty<ActiveTask>();
|
||||
|
||||
public Task<UsageSnapshotDto?> GetUsageSnapshotAsync() => Task.FromResult<UsageSnapshotDto?>(null);
|
||||
public Task<UsageSnapshotDto?> RefreshUsageAsync() => Task.FromResult<UsageSnapshotDto?>(null);
|
||||
public Task<IReadOnlyList<ModelUsageRowDto>> GetModelUsageAsync(DateOnly from, DateOnly to)
|
||||
=> Task.FromResult<IReadOnlyList<ModelUsageRowDto>>(System.Array.Empty<ModelUsageRowDto>());
|
||||
public Task<IReadOnlyList<TaskUsageRowDto>> GetTaskUsageAsync(DateOnly from, DateOnly to)
|
||||
|
||||
@@ -44,6 +44,7 @@ public sealed class ClaudeOAuthUsageClientTests : IDisposable
|
||||
public List<HttpRequestMessage> Requests { get; } = new();
|
||||
public HttpStatusCode ResponseStatus { get; set; } = HttpStatusCode.OK;
|
||||
public string ResponseBody { get; set; } = "{}";
|
||||
public RetryConditionHeaderValue? RetryAfter { get; set; }
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||
{
|
||||
@@ -52,6 +53,7 @@ public sealed class ClaudeOAuthUsageClientTests : IDisposable
|
||||
{
|
||||
Content = new StringContent(ResponseBody, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
if (RetryAfter is not null) resp.Headers.RetryAfter = RetryAfter;
|
||||
return Task.FromResult(resp);
|
||||
}
|
||||
}
|
||||
@@ -179,4 +181,43 @@ public sealed class ClaudeOAuthUsageClientTests : IDisposable
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => client.GetUsageAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUsageAsync_429_ThrowsRateLimitedWithRetryAfterDelta()
|
||||
{
|
||||
WriteCredentials();
|
||||
var (client, handler) = Build();
|
||||
handler.ResponseStatus = HttpStatusCode.TooManyRequests;
|
||||
handler.RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(45));
|
||||
|
||||
var ex = await Assert.ThrowsAsync<UsageRateLimitedException>(() => client.GetUsageAsync());
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(45), ex.RetryAfter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUsageAsync_429_WithoutRetryAfter_HasNullRetryAfter()
|
||||
{
|
||||
WriteCredentials();
|
||||
var (client, handler) = Build();
|
||||
handler.ResponseStatus = HttpStatusCode.TooManyRequests;
|
||||
|
||||
var ex = await Assert.ThrowsAsync<UsageRateLimitedException>(() => client.GetUsageAsync());
|
||||
|
||||
Assert.Null(ex.RetryAfter);
|
||||
Assert.Contains("429", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetUsageAsync_429_WithPastRetryAfterDate_IgnoresIt()
|
||||
{
|
||||
WriteCredentials();
|
||||
var (client, handler) = Build();
|
||||
handler.ResponseStatus = HttpStatusCode.TooManyRequests;
|
||||
handler.RetryAfter = new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddMinutes(-5));
|
||||
|
||||
var ex = await Assert.ThrowsAsync<UsageRateLimitedException>(() => client.GetUsageAsync());
|
||||
|
||||
Assert.Null(ex.RetryAfter);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using ClaudeDo.Data.Models;
|
||||
using ClaudeDo.Worker.Tests.Infrastructure;
|
||||
using ClaudeDo.Worker.Usage;
|
||||
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
|
||||
|
||||
namespace ClaudeDo.Worker.Tests.Usage;
|
||||
|
||||
public sealed class RunningTaskProbeTests : IDisposable
|
||||
{
|
||||
private readonly DbFixture _db = new();
|
||||
|
||||
public void Dispose() => _db.Dispose();
|
||||
|
||||
private async Task SeedTaskAsync(TaskStatus status)
|
||||
{
|
||||
await using var context = _db.CreateFactory().CreateDbContext();
|
||||
var list = new ListEntity { Id = Guid.NewGuid().ToString(), Name = "L", CreatedAt = DateTime.UtcNow };
|
||||
context.Lists.Add(list);
|
||||
context.Tasks.Add(new TaskEntity
|
||||
{
|
||||
Id = Guid.NewGuid().ToString(),
|
||||
ListId = list.Id,
|
||||
Title = $"task-{status}",
|
||||
Status = status,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
});
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnyRunningAsync_NoTasks_ReturnsFalse()
|
||||
{
|
||||
var probe = new RunningTaskProbe(_db.CreateFactory());
|
||||
|
||||
Assert.False(await probe.AnyRunningAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnyRunningAsync_QueuedTaskOnly_ReturnsFalse()
|
||||
{
|
||||
await SeedTaskAsync(TaskStatus.Queued);
|
||||
var probe = new RunningTaskProbe(_db.CreateFactory());
|
||||
|
||||
Assert.False(await probe.AnyRunningAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnyRunningAsync_RunningTask_ReturnsTrue()
|
||||
{
|
||||
await SeedTaskAsync(TaskStatus.Idle);
|
||||
await SeedTaskAsync(TaskStatus.Running);
|
||||
var probe = new RunningTaskProbe(_db.CreateFactory());
|
||||
|
||||
Assert.True(await probe.AnyRunningAsync());
|
||||
}
|
||||
}
|
||||
@@ -32,16 +32,25 @@ public sealed class UsageMonitorServiceTests : IDisposable
|
||||
Task.FromResult(new UsageGateDecision(false, null));
|
||||
}
|
||||
|
||||
private sealed class FakeRunningProbe : IRunningTaskProbe
|
||||
{
|
||||
public bool AnyRunning { get; set; }
|
||||
public Task<bool> AnyRunningAsync(CancellationToken ct = default) => Task.FromResult(AnyRunning);
|
||||
}
|
||||
|
||||
private static UsageSnapshot MakeSnapshot() => new(new UsageBucket(1, null), null, [], DateTime.UtcNow);
|
||||
|
||||
private (UsageMonitorService Service, UsageState State, CapturingHubContext Hub) CreateService(FakeClient client, WorkerConfig? cfg = null)
|
||||
private (UsageMonitorService Service, UsageState State, CapturingHubContext Hub) CreateService(
|
||||
FakeClient client, WorkerConfig? cfg = null, IRunningTaskProbe? probe = null)
|
||||
{
|
||||
var state = new UsageState();
|
||||
var config = cfg ?? new WorkerConfig();
|
||||
var builder = new UsageSnapshotBuilder(state, new FakeGate(), _db.CreateFactory(), config);
|
||||
var hubContext = new CapturingHubContext();
|
||||
var broadcaster = new HubBroadcaster(hubContext);
|
||||
var service = new UsageMonitorService(client, state, config, NullLogger<UsageMonitorService>.Instance, builder, broadcaster);
|
||||
var service = new UsageMonitorService(
|
||||
client, state, config, NullLogger<UsageMonitorService>.Instance, builder, broadcaster,
|
||||
probe ?? new FakeRunningProbe());
|
||||
return (service, state, hubContext);
|
||||
}
|
||||
|
||||
@@ -100,4 +109,94 @@ public sealed class UsageMonitorServiceTests : IDisposable
|
||||
var calls = hubContext.Proxy.Calls.Where(c => c.Method == "UsageUpdated").ToList();
|
||||
Assert.Equal(2, calls.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RefreshNowAsync_Polls_And_Returns_Fresh_Snapshot()
|
||||
{
|
||||
var client = new FakeClient();
|
||||
client.Results.Enqueue(MakeSnapshot);
|
||||
var (service, _, _) = CreateService(client);
|
||||
|
||||
var dto = await service.RefreshNowAsync();
|
||||
|
||||
Assert.Equal(1, client.CallCount);
|
||||
Assert.False(dto.IsStale);
|
||||
Assert.NotNull(dto.FiveHourPercent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RefreshNowAsync_WithinCooldown_ReusesLastPoll()
|
||||
{
|
||||
var client = new FakeClient();
|
||||
client.Results.Enqueue(MakeSnapshot);
|
||||
var (service, _, _) = CreateService(client);
|
||||
|
||||
await service.TickAsync(CancellationToken.None);
|
||||
var dto = await service.RefreshNowAsync();
|
||||
|
||||
// Second call would have thrown "no result queued" had it hit the client.
|
||||
Assert.Equal(1, client.CallCount);
|
||||
Assert.False(dto.IsStale);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RateLimit_Failure_Is_Recorded_As_Error()
|
||||
{
|
||||
var client = new FakeClient();
|
||||
client.Results.Enqueue(() => throw new UsageRateLimitedException(TimeSpan.FromSeconds(30)));
|
||||
var (service, state, _) = CreateService(client);
|
||||
|
||||
await service.TickAsync(CancellationToken.None);
|
||||
|
||||
Assert.NotNull(state.LastError);
|
||||
Assert.Contains("429", state.LastError);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class UsagePollScheduleTests
|
||||
{
|
||||
[Fact]
|
||||
public void Uses_active_interval_while_a_task_runs()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 0, null);
|
||||
Assert.Equal(TimeSpan.FromSeconds(300), delay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Uses_idle_interval_while_nothing_runs()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(false, 300, 900, 0, null);
|
||||
Assert.Equal(TimeSpan.FromSeconds(900), delay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Backs_off_exponentially_on_repeated_rate_limits()
|
||||
{
|
||||
var first = UsagePollSchedule.NextDelay(true, 300, 900, 1, null);
|
||||
var second = UsagePollSchedule.NextDelay(true, 300, 900, 2, null);
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(600), first);
|
||||
Assert.True(second > first);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Honours_retry_after_when_longer_than_the_base_interval()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 1, TimeSpan.FromSeconds(420));
|
||||
Assert.Equal(TimeSpan.FromSeconds(420), delay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Never_polls_sooner_than_the_base_interval_after_a_rate_limit()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(true, 300, 900, 1, TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(TimeSpan.FromSeconds(300), delay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Caps_the_backoff()
|
||||
{
|
||||
var delay = UsagePollSchedule.NextDelay(false, 300, 900, 4, TimeSpan.FromHours(4));
|
||||
Assert.Equal(UsagePollSchedule.MaxDelay, delay);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ public sealed class UsageSnapshotBuilderTests : IDisposable
|
||||
public async Task Snapshot_older_than_4x_poll_interval_is_stale()
|
||||
{
|
||||
await SetThresholdsAsync(80, 90);
|
||||
var cfg = new WorkerConfig { UsagePollIntervalSeconds = 60 };
|
||||
var cfg = new WorkerConfig { UsagePollIntervalActiveSeconds = 60, UsagePollIntervalIdleSeconds = 60 };
|
||||
var state = new UsageState();
|
||||
state.ReportSuccess(new UsageSnapshot(
|
||||
new UsageBucket(10, null), null, Array.Empty<UsageLimitRow>(),
|
||||
@@ -113,7 +113,7 @@ public sealed class UsageSnapshotBuilderTests : IDisposable
|
||||
public async Task Fresh_snapshot_is_not_stale()
|
||||
{
|
||||
await SetThresholdsAsync(80, 90);
|
||||
var cfg = new WorkerConfig { UsagePollIntervalSeconds = 60 };
|
||||
var cfg = new WorkerConfig { UsagePollIntervalActiveSeconds = 60, UsagePollIntervalIdleSeconds = 60 };
|
||||
var state = new UsageState();
|
||||
state.ReportSuccess(new UsageSnapshot(
|
||||
new UsageBucket(10, null), null, Array.Empty<UsageLimitRow>(),
|
||||
|
||||
Reference in New Issue
Block a user