diff --git a/src/ClaudeDo.Localization/locales/de.json b/src/ClaudeDo.Localization/locales/de.json index 653cd727..83462c7d 100644 --- a/src/ClaudeDo.Localization/locales/de.json +++ b/src/ClaudeDo.Localization/locales/de.json @@ -338,6 +338,7 @@ "closeSession": "Sitzung beenden", "conptyLaunchFailed": "Interaktive Sitzung konnte nicht geöffnet werden: {0}", "conptyStarting": "Sitzung wird gestartet…", + "usageWarningNotDelivered": "Die Listen-Handler-Sitzung konnte nicht vor dem Usage-Limit gewarnt werden.", "mergeHelperTitle": "Merge-Helfer", "mergeHelperHandoffTitleSuffix": " (Übergabe)", "mergeHelperTaskTitle": "Listen-Handler: {0}", diff --git a/src/ClaudeDo.Localization/locales/en.json b/src/ClaudeDo.Localization/locales/en.json index baf7c3c1..866828da 100644 --- a/src/ClaudeDo.Localization/locales/en.json +++ b/src/ClaudeDo.Localization/locales/en.json @@ -338,6 +338,7 @@ "closeSession": "Kill session", "conptyLaunchFailed": "Couldn't open interactive session: {0}", "conptyStarting": "Starting session…", + "usageWarningNotDelivered": "Couldn't warn the list-handler session about the usage limit.", "mergeHelperTitle": "Merge Helper", "mergeHelperHandoffTitleSuffix": " (Handoff)", "mergeHelperTaskTitle": "List handler: {0}", diff --git a/src/ClaudeDo.Ui/Services/Interfaces/IWorkerClient.cs b/src/ClaudeDo.Ui/Services/Interfaces/IWorkerClient.cs index e952a3c3..9242ed42 100644 --- a/src/ClaudeDo.Ui/Services/Interfaces/IWorkerClient.cs +++ b/src/ClaudeDo.Ui/Services/Interfaces/IWorkerClient.cs @@ -30,6 +30,11 @@ public interface IWorkerClient : INotifyPropertyChanged /// task to carry out nextPhase ("wait" | "merge" | "wait_final" | "merge_final"). event Action, string>? HandoffRequestedEvent; + /// The 5h usage window is running out under a running list-handler session: + /// (handlerTaskId, message). Mission Control types the message into that task's ConPTY pane so + /// the session can hand off cleanly; no pane for the task means nothing happens. + event Action? HandlerUsageWarningEvent; + /// An agent added a note through the MCP tool — an open notes editor reloads. event Action? NotesUpdatedEvent; diff --git a/src/ClaudeDo.Ui/Services/PtyTerminalSession.cs b/src/ClaudeDo.Ui/Services/PtyTerminalSession.cs index e9f6b43f..adf631ee 100644 --- a/src/ClaudeDo.Ui/Services/PtyTerminalSession.cs +++ b/src/ClaudeDo.Ui/Services/PtyTerminalSession.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Iciclecreek.Terminal; @@ -93,6 +94,56 @@ public sealed class PtyTerminalSession : IDisposable ProcessExited?.Invoke(this, e.ExitCode); } + /// + /// Types into the running session and submits it, exactly as if the user + /// had typed it — used to warn a list-handler session before the usage limit + /// (HandlerUsageWarning). Verified against a real `claude` TUI, including while it was + /// mid-turn (the message is queued and acted on when the turn ends). + /// + /// Three non-obvious details, all established by that spike: + /// • exposes no public write path at all — its only public paste is + /// TerminalView.PasteAsync(), which takes no argument and reads the clipboard. The + /// private SendToPtyAsync on the inner view is the write seam, hence the reflection. + /// • The text and the Enter must be TWO writes. A single text + "\r" chunk lands in + /// claude's composer but is never submitted — a chunk carrying an embedded CR reads as pasted + /// text, not as a keypress. + /// • The Enter bytes come from the emulator (GenerateKeyInput) rather than a literal + /// "\r", because claude turns Win32 input mode on and the emulator owns that encoding. + /// + /// Best-effort by design: a package upgrade that renames the private member costs a warning that + /// never arrives, never a crashed Mission Control. + /// + public async Task SendLineAsync(string text) + { + if (_control is not { } control || !IsRunning || string.IsNullOrEmpty(text)) return false; + try + { + await SendRawAsync(control, text); + await Task.Delay(SubmitDelay); + await SendRawAsync(control, control.Terminal.GenerateKeyInput(XTerm.Input.Key.Enter, XTerm.Input.KeyModifiers.None)); + return true; + } + catch (Exception) + { + return false; + } + } + + // Long enough that the two writes can't be read as one chunk by the child's stdin reader. + private static readonly TimeSpan SubmitDelay = TimeSpan.FromMilliseconds(150); + + private static readonly FieldInfo? s_terminalViewField = + typeof(TerminalControl).GetField("_terminalView", BindingFlags.Instance | BindingFlags.NonPublic); + private static readonly MethodInfo? s_sendToPty = + typeof(TerminalView).GetMethod("SendToPtyAsync", BindingFlags.Instance | BindingFlags.NonPublic); + + private static Task SendRawAsync(TerminalControl control, string data) + { + if (s_terminalViewField?.GetValue(control) is not TerminalView view || s_sendToPty is null) + throw new InvalidOperationException("Iciclecreek.Avalonia.Terminal no longer exposes a PTY write seam."); + return (Task)s_sendToPty.Invoke(view, new object[] { data, CancellationToken.None })!; + } + public void Kill() { try { _control?.Kill(); } diff --git a/src/ClaudeDo.Ui/Services/WorkerClient.cs b/src/ClaudeDo.Ui/Services/WorkerClient.cs index f97140e5..9ef17bf8 100644 --- a/src/ClaudeDo.Ui/Services/WorkerClient.cs +++ b/src/ClaudeDo.Ui/Services/WorkerClient.cs @@ -49,6 +49,7 @@ public partial class WorkerClient : ObservableObject, IAsyncDisposable, IWorkerC public event Action? TaskQuestionAskedEvent; public event Action? TaskQuestionResolvedEvent; public event Action, string>? HandoffRequestedEvent; + public event Action? HandlerUsageWarningEvent; public event Action? ConnectionRestoredEvent; public event Action? WorktreeUpdatedEvent; public event Action? ListUpdatedEvent; @@ -159,6 +160,11 @@ public partial class WorkerClient : ObservableObject, IAsyncDisposable, IWorkerC Dispatcher.UIThread.Post(() => HandoffRequestedEvent?.Invoke(taskId, survivingTaskIds, nextPhase)); }); + _hub.On("HandlerUsageWarning", (taskId, message) => + { + Dispatcher.UIThread.Post(() => HandlerUsageWarningEvent?.Invoke(taskId, message)); + }); + _hub.On("WorktreeUpdated", taskId => { Dispatcher.UIThread.Post(() => WorktreeUpdatedEvent?.Invoke(taskId)); diff --git a/src/ClaudeDo.Ui/ViewModels/InteractiveTerminalViewModel.cs b/src/ClaudeDo.Ui/ViewModels/InteractiveTerminalViewModel.cs index 3468aa2d..ca72e00c 100644 --- a/src/ClaudeDo.Ui/ViewModels/InteractiveTerminalViewModel.cs +++ b/src/ClaudeDo.Ui/ViewModels/InteractiveTerminalViewModel.cs @@ -131,6 +131,11 @@ public sealed partial class InteractiveTerminalViewModel : ViewModelBase, IDispo StartError = message; } + /// Types a line into the live session and submits it — see + /// . False when there is no running process to + /// type into (or the write seam is gone). + public Task SendLineAsync(string text) => _session.SendLineAsync(text); + public void Kill() => _session.Kill(); public void Dispose() diff --git a/src/ClaudeDo.Ui/ViewModels/MissionControl/ConPtyPaneViewModel.cs b/src/ClaudeDo.Ui/ViewModels/MissionControl/ConPtyPaneViewModel.cs index 8fe2455d..e930850d 100644 --- a/src/ClaudeDo.Ui/ViewModels/MissionControl/ConPtyPaneViewModel.cs +++ b/src/ClaudeDo.Ui/ViewModels/MissionControl/ConPtyPaneViewModel.cs @@ -123,6 +123,10 @@ public sealed partial class ConPtyPaneViewModel : ViewModelBase, IDisposable } } + /// Types a message into this pane's live session and submits it (the worker's + /// pre-usage-limit warning). False when the session isn't running. + public System.Threading.Tasks.Task SendLineAsync(string text) => Terminal.SendLineAsync(text); + [RelayCommand] private void Close() => CloseRequested?.Invoke(this); diff --git a/src/ClaudeDo.Ui/ViewModels/MissionControlViewModel.cs b/src/ClaudeDo.Ui/ViewModels/MissionControlViewModel.cs index 45c210b1..d293a869 100644 --- a/src/ClaudeDo.Ui/ViewModels/MissionControlViewModel.cs +++ b/src/ClaudeDo.Ui/ViewModels/MissionControlViewModel.cs @@ -21,6 +21,7 @@ public sealed partial class MissionControlViewModel : ViewModelBase, IDisposable private readonly Action _onTaskUpdated; private readonly Action _onConnectionRestored; private readonly Action, string> _onHandoffRequested; + private readonly Action _onHandlerUsageWarning; // Embedded ConPTY sessions (task-based only) — a manual cockpit detached from the // review/merge/status machinery. @@ -98,6 +99,9 @@ public sealed partial class MissionControlViewModel : ViewModelBase, IDisposable _onHandoffRequested = (taskId, survivingTaskIds, nextPhase) => { _ = OpenMergeHelperHandoffConPtySessionAsync(taskId, survivingTaskIds, nextPhase); }; _worker.HandoffRequestedEvent += _onHandoffRequested; + _onHandlerUsageWarning = (taskId, message) => { _ = WarnHandlerSessionAsync(taskId, message); }; + _worker.HandlerUsageWarningEvent += _onHandlerUsageWarning; + _ = RefreshQueueAsync(); _ = RefreshSessionTargetsAsync(); } @@ -406,6 +410,19 @@ public sealed partial class MissionControlViewModel : ViewModelBase, IDisposable () => DescribeAsync(() => _worker.GetMergeHelperHandoffLaunchSpecAsync(taskId, survivingTaskIds, nextPhase)))); } + // The 5h usage window is about to run out under this handler session: type the worker's warning + // into its pane so the session can hand off cleanly instead of dying mid-merge. The UI process + // owns the terminal, so this is the only place that can reach it at all. No pane (the tile was + // closed, or this is a second app instance) = nothing to warn, and that is not an error. + internal async System.Threading.Tasks.Task WarnHandlerSessionAsync(string taskId, string message) + { + if (string.IsNullOrEmpty(taskId) || string.IsNullOrEmpty(message)) return; + if (ConPtySessions.FirstOrDefault(s => s.TaskId == taskId) is not { } pane) return; + + if (!await pane.SendLineAsync(message)) + ErrorReported?.Invoke(Loc.T("missionControl.usageWarningNotDelivered")); + } + // Wires a freshly built pane and shows it immediately — the pane resolves its own launch spec, // so the tile is on screen (spinner running) while the worker is still preparing the worktree. private void AddConPtyPane(ConPtyPaneViewModel pane) @@ -519,6 +536,7 @@ public sealed partial class MissionControlViewModel : ViewModelBase, IDisposable _worker.TaskUpdatedEvent -= _onTaskUpdated; _worker.ConnectionRestoredEvent -= _onConnectionRestored; _worker.HandoffRequestedEvent -= _onHandoffRequested; + _worker.HandlerUsageWarningEvent -= _onHandlerUsageWarning; ConPtySessions.CollectionChanged -= OnConPtySessionsChanged; Panes.CollectionChanged -= OnPanesChanged; foreach (var c in ConPtySessions.ToList()) diff --git a/src/ClaudeDo.Worker/External/HandoffMcpTools.cs b/src/ClaudeDo.Worker/External/HandoffMcpTools.cs index 9a90de9d..19693bf3 100644 --- a/src/ClaudeDo.Worker/External/HandoffMcpTools.cs +++ b/src/ClaudeDo.Worker/External/HandoffMcpTools.cs @@ -11,7 +11,11 @@ namespace ClaudeDo.Worker.External; // Coerced=true means nextPhase asked for a non-final phase but the server-side round cap // (HandoffRoundTracker) forced it to its "_final" counterpart instead -- NextPhase already // reflects the effective (post-coercion) value in that case. -public sealed record HandoffListHandlerResult(bool Requested, string TaskId, int Number, int SurvivingCount, string NextPhase, bool Coerced = false); +// DeferredUntil set means the next session was NOT opened yet: the 5h usage window is exhausted and +// the handoff is parked until it resets (HandlerUsageGuard). +public sealed record HandoffListHandlerResult( + bool Requested, string TaskId, int Number, int SurvivingCount, string NextPhase, bool Coerced = false, + DateTimeOffset? DeferredUntil = null); public sealed record HandoffStatusResult(string TaskId, int Number, int Round, int Remaining, bool Final); @@ -22,17 +26,20 @@ public sealed class HandoffMcpTools private readonly HubBroadcaster _broadcaster; private readonly HandoffRoundTracker _rounds; private readonly IDbContextFactory _dbFactory; + private readonly HandlerUsageGuard _usageGuard; public HandoffMcpTools( TaskRepository tasks, HubBroadcaster broadcaster, HandoffRoundTracker rounds, - IDbContextFactory dbFactory) + IDbContextFactory dbFactory, + HandlerUsageGuard usageGuard) { _tasks = tasks; _broadcaster = broadcaster; _rounds = rounds; _dbFactory = dbFactory; + _usageGuard = usageGuard; } [McpServerTool, Description( @@ -47,8 +54,10 @@ public sealed class HandoffMcpTools "4 by default, i.e. two " + "full wait/merge cycles) for a given handler task: once reached, a \"wait\"/\"merge\" request is " + "silently coerced to its \"_final\" counterpart instead (check the result's coerced field), and " + - "any handoff at all after a merge_final round for this task throws. The current tile stays " + - "open; you must end your own turn immediately after calling this.")] + "any handoff at all after a merge_final round for this task throws. If the 5h usage window is " + + "already exhausted the next session is NOT opened right away -- the handoff is parked until " + + "the window resets and the result carries deferredUntil; say so in your summary. The current " + + "tile stays open; you must end your own turn immediately after calling this either way.")] public async Task HandoffListHandler( [Description("This session's own handler task id.")] string taskId, [Description("The tasks that made it past dedupe, in the order to run them.")] IReadOnlyList survivingTaskIds, @@ -77,8 +86,15 @@ public sealed class HandoffMcpTools if (round.Rejected) throw new InvalidOperationException(round.Error); - await _broadcaster.HandoffRequested(taskId, survivingTaskIds, round.EffectivePhase); - return new HandoffListHandlerResult(true, taskId, task.Number, survivingTaskIds.Count, round.EffectivePhase, round.Coerced); + // The round is registered either way (the budget is spent by asking, not by the tile + // opening); only the broadcast waits when the 5h window is exhausted. + var deferredUntil = await _usageGuard.TryDeferHandoffAsync( + taskId, survivingTaskIds, round.EffectivePhase, cancellationToken); + if (deferredUntil is null) + await _broadcaster.HandoffRequested(taskId, survivingTaskIds, round.EffectivePhase); + + return new HandoffListHandlerResult( + true, taskId, task.Number, survivingTaskIds.Count, round.EffectivePhase, round.Coerced, deferredUntil); } [McpServerTool, Description( diff --git a/src/ClaudeDo.Worker/Hub/HubBroadcaster.cs b/src/ClaudeDo.Worker/Hub/HubBroadcaster.cs index 4c8430bd..151020a8 100644 --- a/src/ClaudeDo.Worker/Hub/HubBroadcaster.cs +++ b/src/ClaudeDo.Worker/Hub/HubBroadcaster.cs @@ -37,6 +37,12 @@ public sealed class HubBroadcaster : IPrimeBroadcaster, IRefineBroadcaster public Task HandoffRequested(string taskId, IReadOnlyList survivingTaskIds, string nextPhase) => _hub.Clients.All.SendAsync("HandoffRequested", taskId, survivingTaskIds, nextPhase); + // The 5h usage window is about to run out under a running list-handler session (HandlerUsageGuard). + // Mission Control types `message` straight into that task's ConPTY pane, so the session can hand + // off cleanly instead of dying mid-merge. No pane for the task = nothing happens. + public Task HandlerUsageWarning(string handlerTaskId, string message) => + _hub.Clients.All.SendAsync("HandlerUsageWarning", handlerTaskId, message); + /// An agent wrote a note through MCP — the open notes editor reloads. public Task NotesUpdated() => _hub.Clients.All.SendAsync("NotesUpdated"); diff --git a/src/ClaudeDo.Worker/Program.cs b/src/ClaudeDo.Worker/Program.cs index 5ef1b631..db59a5f9 100644 --- a/src/ClaudeDo.Worker/Program.cs +++ b/src/ClaudeDo.Worker/Program.cs @@ -97,6 +97,8 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -339,6 +341,7 @@ if (cfg.ExternalMcpPort > 0) Share(); Share(); Share(); + Share(); externalBuilder.Services.AddScoped(sp => sp.GetRequiredService>().CreateDbContext()); diff --git a/src/ClaudeDo.Worker/Runner/HandlerRestartCoordinator.cs b/src/ClaudeDo.Worker/Runner/HandlerRestartCoordinator.cs index dca9c6d9..62c7d60e 100644 --- a/src/ClaudeDo.Worker/Runner/HandlerRestartCoordinator.cs +++ b/src/ClaudeDo.Worker/Runner/HandlerRestartCoordinator.cs @@ -20,17 +20,14 @@ namespace ClaudeDo.Worker.Runner; /// /// Fails CLOSED, unlike the usage gate: every missing signal means "don't relaunch". An interactive /// session that pops back up at 3am uninvited is worse than one that stays dead. +/// +/// It does not wait here. The restart is parked in and fired +/// by — the same slot a session that handed off cleanly before the +/// limit uses, so exactly one of the two reopens a round. Parking it also survives a worker restart +/// during the hours-long wait, which an in-process Task.Delay did not. /// public sealed class HandlerRestartCoordinator { - // The corroborating signal that this exit really was the usage limit. A ConPTY session gives us - // nothing but an exit code -- there is no stdout to match against, so TaskRunner's - // IsUsageLimitMessage regex (which reads the CLI's own "session limit reached" text) has nothing - // to read here. The 5h utilization at the moment of death is the one independent witness we do - // have, and it is pinned at 100% when the limit is what stopped the session. A crash, a manual - // Ctrl-C, or a bad launch at 40% utilization is therefore NOT restarted. - internal const int UsageLimitProofPct = 95; - private readonly IDbContextFactory _dbFactory; private readonly UsageState _usageState; private readonly HandlerRoundState _rounds; @@ -51,14 +48,10 @@ public sealed class HandlerRestartCoordinator _logger = logger; } - /// Test seams — a real 5h wait is not a test. - internal Func NowUtc { get; set; } = () => DateTimeOffset.UtcNow; - internal Func Delay { get; set; } = (d, ct) => Task.Delay(d, ct); - - /// Handles a ConPTY session's process exit. Returns true when a restart was claimed and - /// carried out (the returned task completes only after the wait, so a caller that wants - /// fire-and-forget must not await it). Any non-handler task, clean exit, non-usage-limit death, - /// PowerUse-off list, or already-used round is a silent no-op. + /// Handles a ConPTY session's process exit. Returns true when a restart was parked for + /// the reset (HandlerUsageGuard fires it). Any non-handler task, clean exit, non-usage-limit + /// death, PowerUse-off list, already-used round, or round that already has a handoff parked is a + /// silent no-op. public async Task OnSessionExitedAsync(string taskId, int exitCode, CancellationToken ct = default) { if (exitCode == 0 || string.IsNullOrEmpty(taskId)) return false; @@ -71,7 +64,6 @@ public sealed class HandlerRestartCoordinator var snapshot = _usageState.Snapshot; if (snapshot?.FiveHour is not { } fiveHour) return false; - if (fiveHour.Utilization < UsageLimitProofPct) return false; if (fiveHour.ResetsAt is not { } resetsAt) return false; int number; @@ -85,8 +77,18 @@ public sealed class HandlerRestartCoordinator // Both PowerUse keys, exactly as ComposeHandlerPromptAsync gates the PowerUse prompt // addendum: the global switch alone must not make an interactive run on some unrelated // list restart itself overnight. - if (!(await new AppSettingsRepository(ctx).GetAsync(ct)).PowerUseEnabled) return false; + var settings = await new AppSettingsRepository(ctx).GetAsync(ct); + if (!settings.PowerUseEnabled) return false; if ((await new ListRepository(ctx).GetConfigAsync(task.ListId, ct))?.PowerUseAllowed != true) return false; + + // The corroborating signal that this exit really was the usage limit. A ConPTY session + // gives us nothing but an exit code -- there is no stdout to match against, so + // TaskRunner's IsUsageLimitMessage regex (which reads the CLI's own "session limit + // reached" text) has nothing to read here. The 5h utilization at the moment of death is + // the one independent witness we do have. Same threshold the warning and the withheld + // handoff use (PowerUseHandoffPct) so all three agree on where "at the limit" starts; + // a crash or a manual Ctrl-C well below it is NOT restarted. + if (fiveHour.Utilization < Math.Clamp(settings.PowerUseHandoffPct, 50, 99)) return false; } catch (Exception ex) { @@ -94,36 +96,16 @@ public sealed class HandlerRestartCoordinator return false; } - // One restart per round, claimed before the wait -- a restart that dies again lands back - // here and must stop, not relaunch. - if (!_rounds.TryClaimAutoRestart(taskId)) return false; + // One restart per round, and never on top of a handoff the session itself already parked. + // The budget is NOT spent here -- HandlerUsageGuard spends it when the broadcast goes out, + // so a worker that dies during the wait doesn't leave the round marked "restart used". + var dueAt = resetsAt + HandlerUsageGuard.ResetGrace; + if (!_rounds.TryScheduleAutoRestart(taskId, dueAt, round.Phase, round.TaskIds)) return false; - try - { - await _broadcaster.WorkerLog( - $"List handler #{number} died at the usage limit (5h {fiveHour.Utilization:0}%) — " + - $"restarting phase {round.Phase} after the reset at {resetsAt.ToLocalTime():HH:mm}", - WorkerLogLevel.Warn, DateTime.UtcNow); - - var wait = resetsAt - NowUtc(); - if (wait > TimeSpan.Zero) await Delay(wait, ct); - - await _broadcaster.HandoffRequested(taskId, round.TaskIds, round.Phase); - await _broadcaster.WorkerLog( - $"Auto-restarting list handler #{number} at phase {round.Phase} after usage-limit reset", - WorkerLogLevel.Info, DateTime.UtcNow); - return true; - } - catch (OperationCanceledException) - { - return false; - } - catch (Exception ex) - { - // Callers fire-and-forget this (WorkerHub.ReportInteractiveSessionExited), so an - // unobserved throw here would be invisible. - _logger.LogError(ex, "HandlerRestartCoordinator: auto-restart of handler task {TaskId} failed", taskId); - return false; - } + await _broadcaster.WorkerLog( + $"List handler #{number} died at the usage limit (5h {fiveHour.Utilization:0}%) — " + + $"restarting phase {round.Phase} after the reset, at {dueAt.ToLocalTime():HH:mm}", + WorkerLogLevel.Warn, DateTime.UtcNow); + return true; } } diff --git a/src/ClaudeDo.Worker/Runner/HandlerRoundState.cs b/src/ClaudeDo.Worker/Runner/HandlerRoundState.cs index 55e261bb..c6e64a85 100644 --- a/src/ClaudeDo.Worker/Runner/HandlerRoundState.cs +++ b/src/ClaudeDo.Worker/Runner/HandlerRoundState.cs @@ -9,17 +9,26 @@ namespace ClaudeDo.Worker.Runner; /// HandoffRequested broadcast that is the only other place these ids exist. /// A value, or "triage" for the initial session /// (which has no handoff spec and therefore cannot be reopened — see HandlerRestartCoordinator). -/// Claimed once per round by HandlerRestartCoordinator, so a handler -/// that dies again after its auto-restart stays dead instead of relaunching all night. -/// Reserved for the "hand off cleanly before the usage limit" -/// follow-up, which parks a handoff until the 5h reset and needs to survive a worker restart. +/// Spent once per round, and only once the deferred +/// HandoffRequested broadcast has actually gone out — a restart that is merely *scheduled* +/// must not count, or a worker that dies during the hours-long wait leaves the round permanently +/// marked "restart used" without one ever having happened. +/// Due time of a handoff parked until the 5h usage window resets. +/// One slot per round, shared by both producers: a session that handed off cleanly just before the +/// limit (HandlerUsageGuard) and a session that died at it (HandlerRestartCoordinator) — so the two +/// can never reopen the same round twice. Fired by HandlerUsageGuard, which re-picks it up after a +/// worker restart because it lives in this file rather than in a timer. public sealed record HandlerRound( string HandlerTaskId, string Phase, IReadOnlyList TaskIds, DateTime StartedAtUtc, bool AutoRestartUsed = false, - DateTimeOffset? DeferredHandoffAt = null); + DateTimeOffset? DeferredHandoffAt = null, + string? DeferredPhase = null, + IReadOnlyList? DeferredTaskIds = null, + bool DeferredIsAutoRestart = false, + bool UsageWarningSent = false); /// /// File-backed store for , one entry per handler task. Deliberately a @@ -74,22 +83,95 @@ public sealed class HandlerRoundState } } - /// Takes this round's single auto-restart budget. False when there is no round for the - /// task or its budget is already spent — the same "never twice" discipline - /// TaskRepository.TryClaimUsageLimitAutoContinueAsync enforces for headless runs. - public bool TryClaimAutoRestart(string handlerTaskId) + public IReadOnlyList All() + { + lock (_lock) + return Rounds().Values.ToList(); + } + + /// Claims this round's single "warn the session before the usage limit" shot. True only + /// the first time, so a monitor that keeps ticking above the threshold does not retype the same + /// warning into the session every 30 seconds. + public bool TryMarkUsageWarned(string handlerTaskId) { lock (_lock) { var rounds = Rounds(); - if (!rounds.TryGetValue(handlerTaskId, out var round) || round.AutoRestartUsed) return false; + if (!rounds.TryGetValue(handlerTaskId, out var round) || round.UsageWarningSent) return false; - rounds[handlerTaskId] = round with { AutoRestartUsed = true }; + rounds[handlerTaskId] = round with { UsageWarningSent = true }; Persist(rounds); return true; } } + /// Parks a handoff the running session asked for until . + /// Overwrites any existing slot: the live session's own request is the most recent word on what + /// the next round should be. + public void SetDeferredHandoff( + string handlerTaskId, DateTimeOffset dueAt, string phase, IReadOnlyList taskIds) + => WriteDeferred(handlerTaskId, dueAt, phase, taskIds, isAutoRestart: false); + + /// Schedules this round's single auto-restart (the session died at the limit). False + /// when there is no round, the restart budget is already spent, or a handoff is *already* parked + /// for this round — the deferred handoff and the auto-restart are one trigger, not two. + /// is deliberately NOT set here; only + /// spends it, once the broadcast really went out. + public bool TryScheduleAutoRestart( + string handlerTaskId, DateTimeOffset dueAt, string phase, IReadOnlyList taskIds) + { + lock (_lock) // Monitor is reentrant — WriteDeferred takes the same lock. + { + var rounds = Rounds(); + if (!rounds.TryGetValue(handlerTaskId, out var round)) return false; + if (round.AutoRestartUsed || round.DeferredHandoffAt is not null) return false; + + WriteDeferred(handlerTaskId, dueAt, phase, taskIds, isAutoRestart: true); + return true; + } + } + + /// Clears the parked handoff. marks the round's + /// restart budget as used — pass the slot's own , + /// so a clean pre-limit handoff never costs the next session its restart. + public void ClearDeferredHandoff(string handlerTaskId, bool spendAutoRestart) + { + lock (_lock) + { + var rounds = Rounds(); + if (!rounds.TryGetValue(handlerTaskId, out var round)) return; + + rounds[handlerTaskId] = round with + { + DeferredHandoffAt = null, + DeferredPhase = null, + DeferredTaskIds = null, + DeferredIsAutoRestart = false, + AutoRestartUsed = round.AutoRestartUsed || spendAutoRestart, + }; + Persist(rounds); + } + } + + private void WriteDeferred( + string handlerTaskId, DateTimeOffset dueAt, string phase, IReadOnlyList taskIds, bool isAutoRestart) + { + lock (_lock) + { + var rounds = Rounds(); + if (!rounds.TryGetValue(handlerTaskId, out var round)) return; + + rounds[handlerTaskId] = round with + { + DeferredHandoffAt = dueAt, + DeferredPhase = phase, + DeferredTaskIds = taskIds.ToList(), + DeferredIsAutoRestart = isAutoRestart, + }; + Persist(rounds); + } + } + public void Remove(string handlerTaskId) { lock (_lock) diff --git a/src/ClaudeDo.Worker/Runner/HandlerUsageGuard.cs b/src/ClaudeDo.Worker/Runner/HandlerUsageGuard.cs new file mode 100644 index 00000000..49d75e92 --- /dev/null +++ b/src/ClaudeDo.Worker/Runner/HandlerUsageGuard.cs @@ -0,0 +1,207 @@ +using ClaudeDo.Data; +using ClaudeDo.Data.Models; +using ClaudeDo.Data.Repositories; +using ClaudeDo.Worker.External; +using ClaudeDo.Worker.Hub; +using ClaudeDo.Worker.Usage; +using Microsoft.EntityFrameworkCore; + +namespace ClaudeDo.Worker.Runner; + +/// +/// Hands a PowerUse list-handler round over cleanly across the 5h usage limit, instead of letting it +/// die mid-merge: +/// +/// 1. Warn — once per round, as soon as the 5h window crosses +/// AppSettings.PowerUseHandoffPct, a message is typed straight into the live ConPTY session +/// (HandlerUsageWarning → Mission Control) telling it to finish what it is doing and call +/// handoff_list_handler itself. A session that hands off on its own leaves no half-merged +/// tree behind; an auto-restart after the fact cannot clean one up. +/// 2. Withhold — the resulting handoff is not broadcast immediately (the next session would +/// open straight into the same exhausted window) but parked in +/// until the window resets. Same slot the auto-restart of a session that *did* die uses +/// (), so the two can never reopen one round twice. +/// 3. Fire — this service's tick is also what releases a parked handoff. Deliberately a poll +/// over the persisted state rather than a timer: a worker restart mid-wait re-picks the parked +/// handoff up on the next tick, and an overdue one goes out immediately, for free. +/// +public sealed class HandlerUsageGuard : BackgroundService +{ + internal static readonly TimeSpan TickInterval = TimeSpan.FromSeconds(30); + + // Small cushion past ResetsAt: the reported reset time is the server's, and reopening a session + // one second early buys nothing but another dead tile. + internal static readonly TimeSpan ResetGrace = TimeSpan.FromSeconds(60); + + private readonly IDbContextFactory _dbFactory; + private readonly UsageState _usageState; + private readonly HandlerRoundState _rounds; + private readonly HandoffRoundTracker _handoffRounds; + private readonly HubBroadcaster _broadcaster; + private readonly ILogger _logger; + + public HandlerUsageGuard( + IDbContextFactory dbFactory, + UsageState usageState, + HandlerRoundState rounds, + HandoffRoundTracker handoffRounds, + HubBroadcaster broadcaster, + ILogger logger) + { + _dbFactory = dbFactory; + _usageState = usageState; + _rounds = rounds; + _handoffRounds = handoffRounds; + _broadcaster = broadcaster; + _logger = logger; + } + + /// Test seam — a real 5h wait is not a test. + internal Func NowUtc { get; set; } = () => DateTimeOffset.UtcNow; + + /// PowerUse verdict for one handler task: both PowerUse switches on, a 5h bucket + /// available, plus the configured threshold and the task's display number. + internal sealed record Policy(int HandoffPct, UsageBucket FiveHour, int Number); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await TickAsync(stoppingToken); + await Task.Delay(TickInterval, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "HandlerUsageGuard: tick failed"); + await Task.Delay(TickInterval, stoppingToken); + } + } + } + + internal async Task TickAsync(CancellationToken ct) + { + foreach (var round in _rounds.All()) + { + await FireIfDueAsync(round); + await WarnIfNearLimitAsync(round, ct); + } + } + + /// Called by handoff_list_handler. Returns the time the handoff was parked until + /// when it must be withheld, or null when it should go out now (PowerUse off for this list, well + /// below the threshold, or no reset time to wait for). + public async Task TryDeferHandoffAsync( + string handlerTaskId, IReadOnlyList survivingTaskIds, string phase, CancellationToken ct) + { + var policy = await EvaluateAsync(handlerTaskId, ct); + if (policy is null) return null; + + // The warning flag matters on its own: utilization can dip back under the threshold between + // the warning and the session actually finishing its merge, and a handoff produced *because* + // of the warning must still wait for the reset. + var warned = _rounds.Get(handlerTaskId)?.UsageWarningSent == true; + if (policy.FiveHour.Utilization < policy.HandoffPct && !warned) return null; + + if (policy.FiveHour.ResetsAt is not { } resetsAt) + { + _logger.LogWarning( + "HandlerUsageGuard: 5h window at {Pct}% has no reset time; handing off {TaskId} immediately", + policy.FiveHour.Utilization, handlerTaskId); + return null; + } + + var dueAt = resetsAt + ResetGrace; + _rounds.SetDeferredHandoff(handlerTaskId, dueAt, phase, survivingTaskIds); + await _broadcaster.WorkerLog( + $"List handler #{policy.Number}: handoff to phase {phase} withheld until " + + $"{dueAt.ToLocalTime():HH:mm} (5h window at {policy.FiveHour.Utilization:0}%)", + WorkerLogLevel.Warn, DateTime.UtcNow); + return dueAt; + } + + private async Task FireIfDueAsync(HandlerRound round) + { + if (round.DeferredHandoffAt is not { } dueAt || dueAt > NowUtc()) return; + + if (round.DeferredPhase is not { Length: > 0 } phase || round.DeferredTaskIds is not { Count: > 0 } taskIds) + { + _rounds.ClearDeferredHandoff(round.HandlerTaskId, spendAutoRestart: false); + return; + } + + // Cleared BEFORE the broadcast so a throw on the way out can't leave a slot that re-fires on + // every tick; the restart budget is spent here too, which is the point of the whole + // rework -- it is now only spent once a handoff really went out. + _rounds.ClearDeferredHandoff(round.HandlerTaskId, spendAutoRestart: round.DeferredIsAutoRestart); + + await _broadcaster.HandoffRequested(round.HandlerTaskId, taskIds, phase); + await _broadcaster.WorkerLog( + $"Reopening list handler at phase {phase} after the usage-limit reset", + WorkerLogLevel.Info, DateTime.UtcNow); + } + + private async Task WarnIfNearLimitAsync(HandlerRound round, CancellationToken ct) + { + if (round.UsageWarningSent) return; + // A sealed chain has no further handoff to ask for -- telling it to call + // handoff_list_handler would only earn the session an exception. + if (_handoffRounds.GetState(round.HandlerTaskId).Sealed) return; + + var policy = await EvaluateAsync(round.HandlerTaskId, ct); + if (policy is null || policy.FiveHour.Utilization < policy.HandoffPct) return; + if (!_rounds.TryMarkUsageWarned(round.HandlerTaskId)) return; + + await _broadcaster.HandlerUsageWarning( + round.HandlerTaskId, BuildWarning(round.HandlerTaskId, round.Phase, policy.FiveHour.Utilization)); + await _broadcaster.WorkerLog( + $"List handler #{policy.Number}: warned the session at 5h {policy.FiveHour.Utilization:0}% " + + $"(handoff threshold {policy.HandoffPct}%)", + WorkerLogLevel.Warn, DateTime.UtcNow); + } + + // Plain ASCII on purpose: this is typed character-by-character into a PTY, not rendered by us. + internal static string BuildWarning(string handlerTaskId, string phase, double utilization) => + $"USAGE LIMIT IMMINENT ({utilization:0}% of the 5h window). Finish or abort the merge you are " + + "in - never leave conflict markers in the tree - then call " + + $"handoff_list_handler(taskId: \"{handlerTaskId}\", survivingTaskIds: , " + + $"nextPhase: \"{NextPhaseFor(phase)}\") right now. Do not start anything new. " + + "The next session opens automatically after the limit resets."; + + // The session should resume where it is, so the warning names its own phase back. The opening + // triage round is the exception: "triage" is not a handoff phase, and what follows triage is + // always the wait phase. + private static string NextPhaseFor(string phase) => + MergeHelperPhase.All.Contains(phase, StringComparer.Ordinal) ? phase : MergeHelperPhase.Wait; + + /// Both PowerUse switches, exactly as ComposeHandlerPromptAsync and + /// gate them — the global toggle alone must not make an + /// unrelated list's handler session get typed into. Null = leave this round alone. + private async Task EvaluateAsync(string handlerTaskId, CancellationToken ct) + { + if (_usageState.Snapshot?.FiveHour is not { } fiveHour) return null; + + try + { + await using var ctx = await _dbFactory.CreateDbContextAsync(ct); + var task = await new TaskRepository(ctx).GetByIdAsync(handlerTaskId, ct); + if (task is null) return null; + + var settings = await new AppSettingsRepository(ctx).GetAsync(ct); + if (!settings.PowerUseEnabled) return null; + if ((await new ListRepository(ctx).GetConfigAsync(task.ListId, ct))?.PowerUseAllowed != true) return null; + + return new Policy(Math.Clamp(settings.PowerUseHandoffPct, 50, 99), fiveHour, task.Number); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "HandlerUsageGuard: could not evaluate handler task {TaskId}", handlerTaskId); + return null; + } + } +} diff --git a/tests/ClaudeDo.Ui.Tests/StubWorkerClient.cs b/tests/ClaudeDo.Ui.Tests/StubWorkerClient.cs index 382ac582..fb9e200c 100644 --- a/tests/ClaudeDo.Ui.Tests/StubWorkerClient.cs +++ b/tests/ClaudeDo.Ui.Tests/StubWorkerClient.cs @@ -26,6 +26,7 @@ public abstract class StubWorkerClient : IWorkerClient public event Action? TaskQuestionAskedEvent; public event Action? TaskQuestionResolvedEvent; public event Action, string>? HandoffRequestedEvent; + public event Action? HandlerUsageWarningEvent; public event Action? NotesUpdatedEvent; public event Action? PrepStartedEvent; public event Action? PrepLineEvent; @@ -57,6 +58,7 @@ public abstract class StubWorkerClient : IWorkerClient public void RaiseTaskQuestionAsked(string taskId, string questionId, string question) => TaskQuestionAskedEvent?.Invoke(taskId, questionId, question); public void RaiseTaskQuestionResolved(string taskId, string questionId) => TaskQuestionResolvedEvent?.Invoke(taskId, questionId); public void RaiseHandoffRequested(string taskId, IReadOnlyList survivingTaskIds, string nextPhase = "wait") => HandoffRequestedEvent?.Invoke(taskId, survivingTaskIds, nextPhase); + public void RaiseHandlerUsageWarning(string taskId, string message) => HandlerUsageWarningEvent?.Invoke(taskId, message); public void RaisePlanningMergeConflict(string planningTaskId, string subtaskId, IReadOnlyList files, bool externallyDriven) => PlanningMergeConflictEvent?.Invoke(planningTaskId, subtaskId, files, externallyDriven); public void RaiseOperationProgress(string opKey, string phase, int current, int total) => OperationProgressEvent?.Invoke(opKey, phase, current, total); diff --git a/tests/ClaudeDo.Ui.Tests/ViewModels/MissionControlViewModelTests.cs b/tests/ClaudeDo.Ui.Tests/ViewModels/MissionControlViewModelTests.cs index 557e1b29..c3d3f9b9 100644 --- a/tests/ClaudeDo.Ui.Tests/ViewModels/MissionControlViewModelTests.cs +++ b/tests/ClaudeDo.Ui.Tests/ViewModels/MissionControlViewModelTests.cs @@ -922,6 +922,38 @@ public class MissionControlViewModelTests : IDisposable pane.Terminal.OnSessionProcessExited(null, exitCode); } + // The worker's pre-usage-limit warning is typed into the pane for that handler task. There is no + // ConPTY process in a headless test, so SendLineAsync reports "not delivered" -- what this pins + // is the routing: the right pane is picked, and a warning for a task with no pane open is a + // silent no-op rather than an error in the footer. + [Fact] + public async Task HandlerUsageWarning_WithNoPaneForTheTask_IsSilentlyIgnored() + { + var worker = new FakeWorker(); + using var vm = BuildVm(worker); + string? error = null; + vm.ErrorReported += msg => error = msg; + + await vm.WarnHandlerSessionAsync("handler-nobody-opened", "USAGE LIMIT IMMINENT"); + + Assert.Null(error); + } + + [Fact] + public async Task HandlerUsageWarning_UndeliverableToItsPane_SurfacesAnError() + { + var worker = new FakeWorker(); + using var vm = BuildVm(worker); + await vm.OpenMergeHelperHandoffConPtySessionAsync("handler-1", new[] { "t1" }, "merge"); + Assert.Single(vm.ConPtySessions); + string? error = null; + vm.ErrorReported += msg => error = msg; + + await vm.WarnHandlerSessionAsync("handler-1", "USAGE LIMIT IMMINENT"); + + Assert.NotNull(error); + } + [Fact] public void ToggleLayoutCommand_FlipsIsFocusMode() { diff --git a/tests/ClaudeDo.Worker.Tests/External/HandoffMcpToolsTests.cs b/tests/ClaudeDo.Worker.Tests/External/HandoffMcpToolsTests.cs index 22ea003a..dbb1cb23 100644 --- a/tests/ClaudeDo.Worker.Tests/External/HandoffMcpToolsTests.cs +++ b/tests/ClaudeDo.Worker.Tests/External/HandoffMcpToolsTests.cs @@ -3,8 +3,11 @@ using ClaudeDo.Data.Models; using ClaudeDo.Data.Repositories; using ClaudeDo.Worker.External; using ClaudeDo.Worker.Hub; +using ClaudeDo.Worker.Runner; using ClaudeDo.Worker.Tests.Infrastructure; +using ClaudeDo.Worker.Usage; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; using TaskStatus = ClaudeDo.Data.Models.TaskStatus; namespace ClaudeDo.Worker.Tests.External; @@ -16,18 +19,35 @@ public sealed class HandoffMcpToolsTests : IDisposable private readonly TaskRepository _tasks; private readonly ListRepository _lists; private readonly CapturingHubContext _hubContext = new(); + private readonly string _tempDir; public HandoffMcpToolsTests() { _ctx = _db.CreateContext(); _tasks = new TaskRepository(_ctx); _lists = new ListRepository(_ctx); + _tempDir = Path.Combine(Path.GetTempPath(), $"cd_handoffmcp_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); } - public void Dispose() { _ctx.Dispose(); _db.Dispose(); } + public void Dispose() + { + _ctx.Dispose(); + _db.Dispose(); + try { Directory.Delete(_tempDir, true); } catch { } + } - private HandoffMcpTools BuildSut(HandoffRoundTracker? tracker = null) => - new(_tasks, new HubBroadcaster(_hubContext), tracker ?? new HandoffRoundTracker(), _db.CreateFactory()); + // No usage snapshot -> HandlerUsageGuard never withholds, which is the behaviour every test + // below assumes. The withholding path has its own tests in HandlerUsageGuardTests. + private HandoffMcpTools BuildSut(HandoffRoundTracker? tracker = null) + { + var rounds = tracker ?? new HandoffRoundTracker(); + var guard = new HandlerUsageGuard( + _db.CreateFactory(), new UsageState(), + new HandlerRoundState(Path.Combine(_tempDir, "handler-rounds.json")), rounds, + new HubBroadcaster(_hubContext), NullLogger.Instance); + return new HandoffMcpTools(_tasks, new HubBroadcaster(_hubContext), rounds, _db.CreateFactory(), guard); + } /// The cap the SUT reads comes from app_settings, so a test that wants a different cap has to /// write it there. GetAsync is AsNoTracking, hence the separate tracked query. diff --git a/tests/ClaudeDo.Worker.Tests/Runner/HandlerRestartCoordinatorTests.cs b/tests/ClaudeDo.Worker.Tests/Runner/HandlerRestartCoordinatorTests.cs index 72f5bec2..86df6f77 100644 --- a/tests/ClaudeDo.Worker.Tests/Runner/HandlerRestartCoordinatorTests.cs +++ b/tests/ClaudeDo.Worker.Tests/Runner/HandlerRestartCoordinatorTests.cs @@ -31,17 +31,18 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } } - private readonly List _waited = new(); + private static readonly DateTimeOffset DueAt = ResetsAt + HandlerUsageGuard.ResetGrace; - private HandlerRestartCoordinator BuildCoordinator(UsageState usage) - { - var coordinator = new HandlerRestartCoordinator( - _db.CreateFactory(), usage, _rounds, new HubBroadcaster(_hub), + private HandlerRestartCoordinator BuildCoordinator(UsageState usage) => + new(_db.CreateFactory(), usage, _rounds, new HubBroadcaster(_hub), NullLogger.Instance); - coordinator.NowUtc = () => ResetsAt.AddHours(-3); - coordinator.Delay = (d, _) => { _waited.Add(d); return Task.CompletedTask; }; - return coordinator; - } + + /// The coordinator only parks the restart; this is what actually fires it. `now` decides whether + /// the parked handoff is already due, so a 5h wait costs a test nothing. + private HandlerUsageGuard BuildGuard(UsageState usage, DateTimeOffset now) => + new(_db.CreateFactory(), usage, _rounds, new HandoffRoundTracker(), new HubBroadcaster(_hub), + NullLogger.Instance) + { NowUtc = () => now }; /// The one signal a ConPTY death gives the worker: the 5h window pinned at the limit. private static UsageState AtUsageLimit(double percent = 100) => @@ -91,16 +92,45 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable { var taskId = await SeedHandlerTaskAsync(); _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a", "b" }, DateTime.UtcNow)); - var coordinator = BuildCoordinator(AtUsageLimit()); - Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1)); + Assert.True(await BuildCoordinator(AtUsageLimit()).OnSessionExitedAsync(taskId, 1)); + + // Parked, not relaunched straight into the same exhausted window. + Assert.Empty(HandoffCalls); + Assert.Equal(DueAt, _rounds.Get(taskId)!.DeferredHandoffAt); + + await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default); var call = Assert.Single(HandoffCalls); Assert.Equal(taskId, call.Args[0]); Assert.Equal(new[] { "a", "b" }, (IReadOnlyList)call.Args[1]!); Assert.Equal(MergeHelperPhase.Merge, call.Args[2]); - // Waited out the window instead of relaunching straight into the same limit. - Assert.Equal(new[] { TimeSpan.FromHours(3) }, _waited); + } + + // The #272 review finding: the restart budget used to be spent when the wait STARTED, so a + // worker that died during the hours-long wait left the round marked "restart used" without one + // ever having happened -- and the session stayed dead forever. + [Fact] + public async Task ParkedRestart_SurvivesAWorkerRestart_AndOnlySpendsTheBudgetOnceItFires() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + Assert.True(await BuildCoordinator(AtUsageLimit()).OnSessionExitedAsync(taskId, 1)); + + // Worker restart: a fresh state object re-reads the same file, nothing else carries over. + var afterRestart = new HandlerRoundState(Path.Combine(_tempDir, "handler-rounds.json")); + Assert.False(afterRestart.Get(taskId)!.AutoRestartUsed); + Assert.Equal(DueAt, afterRestart.Get(taskId)!.DeferredHandoffAt); + + // Still parked while the window is open... + await BuildGuard(AtUsageLimit(), DueAt.AddMinutes(-5)).TickAsync(default); + Assert.Empty(HandoffCalls); + + // ...and sent as soon as it is due, which is also when the budget is spent. + await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default); + Assert.Single(HandoffCalls); + Assert.True(_rounds.Get(taskId)!.AutoRestartUsed); + Assert.Null(_rounds.Get(taskId)!.DeferredHandoffAt); } // ── acceptance 2: at most one auto-restart per round ─────────────────────────────────────── @@ -112,9 +142,25 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); var coordinator = BuildCoordinator(AtUsageLimit()); + Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1)); + await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default); + + Assert.False(await coordinator.OnSessionExitedAsync(taskId, 1)); + Assert.Single(HandoffCalls); + } + + // A second death while the first restart is still parked must not overwrite or duplicate it. + [Fact] + public async Task SecondDeathWhileTheRestartIsStillParked_ChangesNothing() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + var coordinator = BuildCoordinator(AtUsageLimit()); + Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1)); Assert.False(await coordinator.OnSessionExitedAsync(taskId, 1)); + await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default); Assert.Single(HandoffCalls); } @@ -128,6 +174,7 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable var coordinator = BuildCoordinator(AtUsageLimit()); Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1)); + await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default); // The relaunched session's launch spec writes the round again, same phase. _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); @@ -176,11 +223,14 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable tracker.Register(taskId, MergeHelperPhase.Merge, 4); // the run's real handoff into this phase using var ctx = _db.CreateContext(); + var usage = AtUsageLimit(); var tools = new HandoffMcpTools( - new TaskRepository(ctx), new HubBroadcaster(_hub), tracker, _db.CreateFactory()); + new TaskRepository(ctx), new HubBroadcaster(_hub), tracker, _db.CreateFactory(), + BuildGuard(usage, ResetsAt)); var before = await tools.GetHandoffStatus(taskId); - Assert.True(await BuildCoordinator(AtUsageLimit()).OnSessionExitedAsync(taskId, 1)); + Assert.True(await BuildCoordinator(usage).OnSessionExitedAsync(taskId, 1)); + await BuildGuard(usage, DueAt).TickAsync(default); var after = await tools.GetHandoffStatus(taskId); Assert.Equal(before.Round, after.Round); diff --git a/tests/ClaudeDo.Worker.Tests/Runner/HandlerUsageGuardTests.cs b/tests/ClaudeDo.Worker.Tests/Runner/HandlerUsageGuardTests.cs new file mode 100644 index 00000000..beb8f973 --- /dev/null +++ b/tests/ClaudeDo.Worker.Tests/Runner/HandlerUsageGuardTests.cs @@ -0,0 +1,277 @@ +using ClaudeDo.Data.Models; +using ClaudeDo.Data.Repositories; +using ClaudeDo.Worker.External; +using ClaudeDo.Worker.Hub; +using ClaudeDo.Worker.Runner; +using ClaudeDo.Worker.Tests.Infrastructure; +using ClaudeDo.Worker.Usage; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; + +namespace ClaudeDo.Worker.Tests.Runner; + +/// Handing a PowerUse list-handler round over cleanly across the 5h usage limit: warn the live +/// session once, then withhold the handoff it produces until the window resets. The UI half (the +/// warning actually reaching the ConPTY session) lives in Ui.Tests' MissionControlViewModel tests. +public sealed class HandlerUsageGuardTests : IDisposable +{ + private readonly DbFixture _db = new(); + private readonly string _tempDir; + private readonly CapturingHubContext _hub = new(); + private readonly HandlerRoundState _rounds; + private readonly HandoffRoundTracker _handoffRounds = new(); + + private static readonly DateTimeOffset ResetsAt = new(2026, 9, 3, 2, 0, 0, TimeSpan.Zero); + private static readonly DateTimeOffset DueAt = ResetsAt + HandlerUsageGuard.ResetGrace; + private static readonly DateTimeOffset Now = ResetsAt.AddHours(-3); + + public HandlerUsageGuardTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"cd_usageguard_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + _rounds = new HandlerRoundState(RoundsPath); + } + + private string RoundsPath => Path.Combine(_tempDir, "handler-rounds.json"); + + public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } } + + private static UsageState Usage(double percent, DateTimeOffset? resetsAt) => + Snapshot(new UsageBucket(percent, resetsAt)); + + private static UsageState Snapshot(UsageBucket? fiveHour) + { + var state = new UsageState(); + state.ReportSuccess(new UsageSnapshot(fiveHour, null, Array.Empty(), DateTime.UtcNow)); + return state; + } + + private HandlerUsageGuard BuildGuard(UsageState usage, DateTimeOffset? now = null, HandlerRoundState? rounds = null) => + new(_db.CreateFactory(), usage, rounds ?? _rounds, _handoffRounds, new HubBroadcaster(_hub), + NullLogger.Instance) + { NowUtc = () => now ?? Now }; + + private int _nextNumber = 1; + + private async Task SeedHandlerTaskAsync(bool powerUseEnabled = true, bool powerUseAllowed = true) + { + var listId = Guid.NewGuid().ToString(); + var taskId = Guid.NewGuid().ToString(); + using (var ctx = _db.CreateContext()) + { + ctx.Lists.Add(new ListEntity { Id = listId, Name = "L", WorkingDir = _tempDir, CreatedAt = DateTime.UtcNow }); + ctx.ListConfigs.Add(new ListConfigEntity { ListId = listId, PowerUseAllowed = powerUseAllowed }); + ctx.Tasks.Add(new TaskEntity + { + Id = taskId, ListId = listId, Title = "Handler", Number = _nextNumber++, + IsManual = true, CreatedAt = DateTime.UtcNow, + }); + await ctx.SaveChangesAsync(); + + // Written straight to the row: AppSettingsRepository.UpdateAsync deliberately doesn't + // copy PowerUseEnabled (the usage monitor owns that toggle, not the settings modal). + await new AppSettingsRepository(ctx).GetAsync(); + var settings = await ctx.AppSettings.FirstAsync(s => s.Id == AppSettingsEntity.SingletonId); + settings.PowerUseEnabled = powerUseEnabled; + await ctx.SaveChangesAsync(); + } + return taskId; + } + + private IReadOnlyList Calls(string method) => + _hub.Proxy.Calls.Where(c => c.Method == method).ToList(); + + private IReadOnlyList Warnings => Calls("HandlerUsageWarning"); + private IReadOnlyList HandoffCalls => Calls("HandoffRequested"); + + // ── acceptance 1: exactly one warning per round ──────────────────────────────────────────── + + [Fact] + public async Task AtTheThreshold_WarnsTheSessionExactlyOncePerRound() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + var guard = BuildGuard(Usage(95, ResetsAt)); + + await guard.TickAsync(default); + await guard.TickAsync(default); + await BuildGuard(Usage(99, ResetsAt)).TickAsync(default); + + var warning = Assert.Single(Warnings); + Assert.Equal(taskId, warning.Args[0]); + var message = (string)warning.Args[1]!; + Assert.Contains("USAGE LIMIT IMMINENT", message); + // Names the session's own phase back, so it resumes where it is. + Assert.Contains($"nextPhase: \"{MergeHelperPhase.Merge}\"", message); + Assert.Contains(taskId, message); + } + + [Fact] + public async Task BelowTheThreshold_NoWarning() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + + await BuildGuard(Usage(94, ResetsAt)).TickAsync(default); + + Assert.Empty(Warnings); + } + + // The triage round has no "triage" handoff phase -- what follows it is always the wait phase. + [Fact] + public async Task TriageRound_IsToldToHandOffToTheWaitPhase() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound( + taskId, InteractiveLaunchSpecService.TriagePhase, new[] { "a" }, DateTime.UtcNow)); + + await BuildGuard(Usage(96, ResetsAt)).TickAsync(default); + + Assert.Contains($"nextPhase: \"{MergeHelperPhase.Wait}\"", (string)Assert.Single(Warnings).Args[1]!); + } + + // A sealed chain can't hand off at all -- telling it to would only earn the session an exception. + [Fact] + public async Task SealedChain_IsNotWarned() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.MergeFinal, new[] { "a" }, DateTime.UtcNow)); + _handoffRounds.Register(taskId, MergeHelperPhase.MergeFinal); + + await BuildGuard(Usage(99, ResetsAt)).TickAsync(default); + + Assert.Empty(Warnings); + } + + // ── acceptance 5: PowerUse gates both halves ─────────────────────────────────────────────── + + [Theory] + [InlineData(false, true)] + [InlineData(true, false)] + public async Task PowerUseOff_NeitherWarnsNorWithholds(bool powerUseEnabled, bool powerUseAllowed) + { + var taskId = await SeedHandlerTaskAsync(powerUseEnabled, powerUseAllowed); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + var guard = BuildGuard(Usage(99, ResetsAt)); + + await guard.TickAsync(default); + + Assert.Empty(Warnings); + Assert.Null(await guard.TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default)); + Assert.Null(_rounds.Get(taskId)!.DeferredHandoffAt); + } + + // ── acceptance 2 + 3: withholding the handoff ────────────────────────────────────────────── + + [Fact] + public async Task BelowTheThreshold_HandoffIsNotWithheld() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + + var deferred = await BuildGuard(Usage(94, ResetsAt)) + .TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default); + + Assert.Null(deferred); + Assert.Null(_rounds.Get(taskId)!.DeferredHandoffAt); + } + + [Fact] + public async Task AtTheThreshold_HandoffIsWithheldUntilTheResetPlusGrace() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Wait, new[] { "a" }, DateTime.UtcNow)); + var guard = BuildGuard(Usage(95, ResetsAt)); + + var deferred = await guard.TryDeferHandoffAsync(taskId, new[] { "a", "b" }, MergeHelperPhase.Merge, default); + + Assert.Equal(DueAt, deferred); + Assert.Empty(HandoffCalls); + + // Still parked before it is due, sent afterwards -- with the handed-off phase and task set, + // not the round's own. + await BuildGuard(Usage(95, ResetsAt), DueAt.AddMinutes(-1)).TickAsync(default); + Assert.Empty(HandoffCalls); + + await BuildGuard(Usage(95, ResetsAt), DueAt).TickAsync(default); + var call = Assert.Single(HandoffCalls); + Assert.Equal(taskId, call.Args[0]); + Assert.Equal(new[] { "a", "b" }, (IReadOnlyList)call.Args[1]!); + Assert.Equal(MergeHelperPhase.Merge, call.Args[2]); + // A clean handoff must not cost the next session its auto-restart budget. + Assert.False(_rounds.Get(taskId)!.AutoRestartUsed); + } + + // Utilization can dip back below the threshold while the session finishes its merge; the handoff + // that the warning ASKED for still has to wait for the reset. + [Fact] + public async Task AfterAWarning_HandoffIsWithheldEvenIfUtilizationDropped() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + await BuildGuard(Usage(97, ResetsAt)).TickAsync(default); + Assert.Single(Warnings); + + var deferred = await BuildGuard(Usage(80, ResetsAt)) + .TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default); + + Assert.Equal(DueAt, deferred); + } + + // No reset time = nothing to wait for. Sending now beats parking a handoff that never fires. + [Fact] + public async Task NoResetTime_HandsOffImmediately() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + + var deferred = await BuildGuard(Usage(99, null)) + .TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default); + + Assert.Null(deferred); + Assert.Null(_rounds.Get(taskId)!.DeferredHandoffAt); + } + + // ── acceptance 4: a parked handoff survives a worker restart ─────────────────────────────── + + [Fact] + public async Task WorkerRestart_ReschedulesAParkedHandoff_AndSendsAnOverdueOneAtOnce() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + await BuildGuard(Usage(99, ResetsAt)) + .TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default); + + // Worker restart: nothing but the file carries over. + var afterRestart = new HandlerRoundState(RoundsPath); + Assert.Equal(DueAt, afterRestart.Get(taskId)!.DeferredHandoffAt); + + // Not due yet -> still parked. + await BuildGuard(Usage(99, ResetsAt), Now, afterRestart).TickAsync(default); + Assert.Empty(HandoffCalls); + + // Long overdue (the worker was down past the reset) -> goes out on the first tick. + await BuildGuard(Usage(99, ResetsAt), DueAt.AddHours(2), afterRestart).TickAsync(default); + Assert.Single(HandoffCalls); + Assert.Null(afterRestart.Get(taskId)!.DeferredHandoffAt); + } + + // One trigger, one state: a parked handoff blocks the auto-restart of the same round, so the + // round can never be reopened twice. + [Fact] + public async Task AParkedHandoff_BlocksTheAutoRestartOfTheSameRound() + { + var taskId = await SeedHandlerTaskAsync(); + _rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow)); + var usage = Usage(99, ResetsAt); + await BuildGuard(usage).TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default); + + var coordinator = new HandlerRestartCoordinator( + _db.CreateFactory(), usage, _rounds, new HubBroadcaster(_hub), + NullLogger.Instance); + Assert.False(await coordinator.OnSessionExitedAsync(taskId, 1)); + + await BuildGuard(usage, DueAt).TickAsync(default); + Assert.Single(HandoffCalls); + } +}