Merge branch 'claudedo/b346a534f0fc4635bd7614a00ab2a20e'

src/ClaudeDo.Worker/Runner/HandlerRestartCoordinator.cs

@@ -0,0 +1,129 @@
+using ClaudeDo.Data;
+using ClaudeDo.Data.Models;
+using ClaudeDo.Data.Repositories;
+using ClaudeDo.Worker.Hub;
+using ClaudeDo.Worker.Usage;
+using Microsoft.EntityFrameworkCore;
+
+namespace ClaudeDo.Worker.Runner;
+
+/// <summary>
+/// Reopens a list-handler ConPTY session that died on the Anthropic usage limit, once the 5h window
+/// resets — the interactive counterpart to <c>UsageLimitAutoContinueCoordinator</c>, which only ever
+/// sees headless task runs. Deliberately a sibling and not an extension of it: a ConPTY session has
+/// no <c>task_runs</c> row, no <c>FailureReason</c>, and is resumed by relaunching a phase rather
+/// than by <c>ContinueTask</c> through the override slot.
+///
+/// Driven by the UI (<c>WorkerHub.ReportInteractiveSessionExited</c>) because the exit code only
+/// exists in the process that owns the terminal. Everything else — was it the usage limit, is
+/// PowerUse on, which phase and task set, when does the window reset — is decided here.
+///
+/// Fails CLOSED, unlike the usage gate: every missing signal means "don't relaunch". An interactive
+/// session that pops back up at 3am uninvited is worse than one that stays dead.
+/// </summary>
+public sealed class HandlerRestartCoordinator
+{
+    // The corroborating signal that this exit really was the usage limit. A ConPTY session gives us
+    // nothing but an exit code -- there is no stdout to match against, so TaskRunner's
+    // IsUsageLimitMessage regex (which reads the CLI's own "session limit reached" text) has nothing
+    // to read here. The 5h utilization at the moment of death is the one independent witness we do
+    // have, and it is pinned at 100% when the limit is what stopped the session. A crash, a manual
+    // Ctrl-C, or a bad launch at 40% utilization is therefore NOT restarted.
+    internal const int UsageLimitProofPct = 95;
+
+    private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
+    private readonly UsageState _usageState;
+    private readonly HandlerRoundState _rounds;
+    private readonly HubBroadcaster _broadcaster;
+    private readonly ILogger<HandlerRestartCoordinator> _logger;
+
+    public HandlerRestartCoordinator(
+        IDbContextFactory<ClaudeDoDbContext> dbFactory,
+        UsageState usageState,
+        HandlerRoundState rounds,
+        HubBroadcaster broadcaster,
+        ILogger<HandlerRestartCoordinator> logger)
+    {
+        _dbFactory = dbFactory;
+        _usageState = usageState;
+        _rounds = rounds;
+        _broadcaster = broadcaster;
+        _logger = logger;
+    }
+
+    /// <summary>Test seams — a real 5h wait is not a test.</summary>
+    internal Func<DateTimeOffset> NowUtc { get; set; } = () => DateTimeOffset.UtcNow;
+    internal Func<TimeSpan, CancellationToken, Task> Delay { get; set; } = (d, ct) => Task.Delay(d, ct);
+
+    /// <summary>Handles a ConPTY session's process exit. Returns true when a restart was claimed and
+    /// carried out (the returned task completes only after the wait, so a caller that wants
+    /// fire-and-forget must not await it). Any non-handler task, clean exit, non-usage-limit death,
+    /// PowerUse-off list, or already-used round is a silent no-op.</summary>
+    public async Task<bool> OnSessionExitedAsync(string taskId, int exitCode, CancellationToken ct = default)
+    {
+        if (exitCode == 0 || string.IsNullOrEmpty(taskId)) return false;
+
+        var round = _rounds.Get(taskId);
+        // Not a list-handler session at all (a plain interactive task pane), or the initial triage
+        // session, which has no handoff launch spec to reopen -- BuildForMergeHelperAsync needs the
+        // list id and the original selection, and its dedupe/enhance work is not idempotent.
+        if (round is null || !MergeHelperPhase.All.Contains(round.Phase, StringComparer.Ordinal)) return false;
+
+        var snapshot = _usageState.Snapshot;
+        if (snapshot?.FiveHour is not { } fiveHour) return false;
+        if (fiveHour.Utilization < UsageLimitProofPct) return false;
+        if (fiveHour.ResetsAt is not { } resetsAt) return false;
+
+        int number;
+        try
+        {
+            await using var ctx = await _dbFactory.CreateDbContextAsync(ct);
+            var task = await new TaskRepository(ctx).GetByIdAsync(taskId, ct);
+            if (task is null) return false;
+            number = task.Number;
+
+            // Both PowerUse keys, exactly as ComposeHandlerPromptAsync gates the PowerUse prompt
+            // addendum: the global switch alone must not make an interactive run on some unrelated
+            // list restart itself overnight.
+            if (!(await new AppSettingsRepository(ctx).GetAsync(ct)).PowerUseEnabled) return false;
+            if ((await new ListRepository(ctx).GetConfigAsync(task.ListId, ct))?.PowerUseAllowed != true) return false;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "HandlerRestartCoordinator: could not evaluate handler task {TaskId}; not restarting", taskId);
+            return false;
+        }
+
+        // One restart per round, claimed before the wait -- a restart that dies again lands back
+        // here and must stop, not relaunch.
+        if (!_rounds.TryClaimAutoRestart(taskId)) return false;
+
+        try
+        {
+            await _broadcaster.WorkerLog(
+                $"List handler #{number} died at the usage limit (5h {fiveHour.Utilization:0}%) — " +
+                $"restarting phase {round.Phase} after the reset at {resetsAt.ToLocalTime():HH:mm}",
+                WorkerLogLevel.Warn, DateTime.UtcNow);
+
+            var wait = resetsAt - NowUtc();
+            if (wait > TimeSpan.Zero) await Delay(wait, ct);
+
+            await _broadcaster.HandoffRequested(taskId, round.TaskIds, round.Phase);
+            await _broadcaster.WorkerLog(
+                $"Auto-restarting list handler #{number} at phase {round.Phase} after usage-limit reset",
+                WorkerLogLevel.Info, DateTime.UtcNow);
+            return true;
+        }
+        catch (OperationCanceledException)
+        {
+            return false;
+        }
+        catch (Exception ex)
+        {
+            // Callers fire-and-forget this (WorkerHub.ReportInteractiveSessionExited), so an
+            // unobserved throw here would be invisible.
+            _logger.LogError(ex, "HandlerRestartCoordinator: auto-restart of handler task {TaskId} failed", taskId);
+            return false;
+        }
+    }
+}

src/ClaudeDo.Worker/Runner/HandlerRoundState.cs

@@ -0,0 +1,138 @@
+using System.Text.Json;
+using ClaudeDo.Data;
+
+namespace ClaudeDo.Worker.Runner;
+
+/// <summary>One list-handler round: the phase a session is running and the exact task set it was
+/// handed. Written every time a handler session is built, so an outside observer (a crashed tile,
+/// a deferred handoff) can reopen the SAME phase with the SAME set without the in-flight
+/// <c>HandoffRequested</c> broadcast that is the only other place these ids exist.</summary>
+/// <param name="Phase">A <see cref="MergeHelperPhase"/> value, or "triage" for the initial session
+/// (which has no handoff spec and therefore cannot be reopened — see HandlerRestartCoordinator).</param>
+/// <param name="AutoRestartUsed">Claimed once per round by HandlerRestartCoordinator, so a handler
+/// that dies again after its auto-restart stays dead instead of relaunching all night.</param>
+/// <param name="DeferredHandoffAt">Reserved for the "hand off cleanly before the usage limit"
+/// follow-up, which parks a handoff until the 5h reset and needs to survive a worker restart.</param>
+public sealed record HandlerRound(
+    string HandlerTaskId,
+    string Phase,
+    IReadOnlyList<string> TaskIds,
+    DateTime StartedAtUtc,
+    bool AutoRestartUsed = false,
+    DateTimeOffset? DeferredHandoffAt = null);
+
+/// <summary>
+/// File-backed store for <see cref="HandlerRound"/>, one entry per handler task. Deliberately a
+/// JSON file under <c>~/.claudeDo/</c> (same Load/Save shape as <c>ClaudeDo.Ui.AppSettings</c>)
+/// rather than a DB column: rounds are process-lifetime scratch state that no query joins against,
+/// and an EF migration here would collide with the one the settings work owns this round.
+/// Every read/write goes through one lock and re-serializes the whole file — there are at most a
+/// handful of concurrent handler runs, so nothing here needs to be clever.
+/// </summary>
+public sealed class HandlerRoundState
+{
+    // Entries older than this are dropped on the next save. A round that has been open for a week
+    // is not coming back, and without a sweep the file only ever grows.
+    private static readonly TimeSpan Retention = TimeSpan.FromDays(7);
+
+    private readonly string _path;
+    private readonly object _lock = new();
+    private Dictionary<string, HandlerRound>? _rounds;
+
+    public HandlerRoundState() : this(Path.Combine(Paths.AppDataRoot(), "handler-rounds.json"))
+    {
+    }
+
+    /// <summary>Test seam — point the store at a temp file instead of the live app data root.</summary>
+    internal HandlerRoundState(string path) => _path = path;
+
+    public HandlerRound? Get(string handlerTaskId)
+    {
+        lock (_lock)
+            return Rounds().GetValueOrDefault(handlerTaskId);
+    }
+
+    /// <summary>Overwrites this handler task's round. <see cref="HandlerRound.AutoRestartUsed"/> is
+    /// carried over when the phase is unchanged: the auto-restart itself rebuilds the same phase's
+    /// launch spec, which lands right back here, and a naive overwrite would hand the round a fresh
+    /// restart budget every time — exactly the all-night relaunch loop the claim exists to prevent.
+    /// A genuinely new phase is a new round and starts with a fresh budget.</summary>
+    public void Save(HandlerRound round)
+    {
+        lock (_lock)
+        {
+            var rounds = Rounds();
+            if (rounds.TryGetValue(round.HandlerTaskId, out var previous)
+                && string.Equals(previous.Phase, round.Phase, StringComparison.Ordinal)
+                && previous.AutoRestartUsed)
+            {
+                round = round with { AutoRestartUsed = true };
+            }
+
+            rounds[round.HandlerTaskId] = round;
+            Persist(rounds);
+        }
+    }
+
+    /// <summary>Takes this round's single auto-restart budget. False when there is no round for the
+    /// task or its budget is already spent — the same "never twice" discipline
+    /// <c>TaskRepository.TryClaimUsageLimitAutoContinueAsync</c> enforces for headless runs.</summary>
+    public bool TryClaimAutoRestart(string handlerTaskId)
+    {
+        lock (_lock)
+        {
+            var rounds = Rounds();
+            if (!rounds.TryGetValue(handlerTaskId, out var round) || round.AutoRestartUsed) return false;
+
+            rounds[handlerTaskId] = round with { AutoRestartUsed = true };
+            Persist(rounds);
+            return true;
+        }
+    }
+
+    public void Remove(string handlerTaskId)
+    {
+        lock (_lock)
+        {
+            var rounds = Rounds();
+            if (rounds.Remove(handlerTaskId)) Persist(rounds);
+        }
+    }
+
+    // Callers hold _lock.
+    private Dictionary<string, HandlerRound> Rounds()
+    {
+        if (_rounds is not null) return _rounds;
+        try
+        {
+            if (File.Exists(_path))
+            {
+                _rounds = JsonSerializer.Deserialize<Dictionary<string, HandlerRound>>(
+                    File.ReadAllText(_path), new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
+            }
+        }
+        catch
+        {
+            // A corrupt/unreadable file must never block a handler launch — start empty.
+        }
+        return _rounds ??= new Dictionary<string, HandlerRound>(StringComparer.Ordinal);
+    }
+
+    private void Persist(Dictionary<string, HandlerRound> rounds)
+    {
+        var cutoff = DateTime.UtcNow - Retention;
+        foreach (var stale in rounds.Where(kv => kv.Value.StartedAtUtc < cutoff).Select(kv => kv.Key).ToList())
+            rounds.Remove(stale);
+
+        try
+        {
+            var dir = Path.GetDirectoryName(_path);
+            if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
+            File.WriteAllText(_path, JsonSerializer.Serialize(rounds, new JsonSerializerOptions { WriteIndented = true }));
+        }
+        catch
+        {
+            // Losing persistence costs an auto-restart across a worker restart, never a launch.
+        }
+    }
+}

src/ClaudeDo.Worker/Runner/InteractiveLaunchSpecService.cs

@@ -44,6 +44,7 @@ public sealed class InteractiveLaunchSpecService
     private readonly GitService _git;
     private readonly string _claudePath;
     private readonly HandoffRoundTracker _handoffRounds;
+    private readonly HandlerRoundState _roundState;
 
     // Local wall clock, injectable so the PowerUse time window is testable without sleeping.
     private readonly Func<DateTime> _now;
@@ -56,6 +57,7 @@ public sealed class InteractiveLaunchSpecService
         GitService git,
         WorkerConfig cfg,
         HandoffRoundTracker handoffRounds,
+        HandlerRoundState roundState,
         Func<DateTime>? now = null)
     {
         _now = now ?? (() => DateTime.Now);
@@ -66,6 +68,7 @@ public sealed class InteractiveLaunchSpecService
         _git = git;
         _claudePath = cfg.ClaudeBin;
         _handoffRounds = handoffRounds;
+        _roundState = roundState;
     }
 
     /// <summary>Builds a LaunchSpec for opening an interactive session in a task's worktree.
@@ -347,9 +350,20 @@ public sealed class InteractiveLaunchSpecService
             ["MCP_TOOL_TIMEOUT"] = McpToolTimeoutMs,
         };
 
+        // The opening round. Recorded so anything watching a handler run knows which round is live
+        // (the deferred-handoff follow-up needs that), but marked "triage" rather than a
+        // MergeHelperPhase value on purpose: there is no handoff spec for this phase, so
+        // HandlerRestartCoordinator will not reopen it.
+        _roundState.Save(new HandlerRound(handlerTaskId, TriagePhase, taskIds, DateTime.UtcNow));
+
         return new LaunchSpec(repoDir, resolvedClaude, args, env);
     }
 
+    /// <summary>The <see cref="HandlerRound.Phase"/> marker for the initial (dedupe/enhance) handler
+    /// session. Deliberately not a <see cref="MergeHelperPhase"/> value — that set is exactly the
+    /// phases <see cref="BuildForMergeHelperHandoffAsync"/> can rebuild.</summary>
+    public const string TriagePhase = "triage";
+
     // Builds the LaunchSpec for the fresh session a merge-helper run hands off to for the given
     // phase -- SAME handler task id as the run that called handoff_list_handler, so
     // HandlerBaseCommit/HandlerHeadCommit and the review range stay untouched; this never creates
@@ -454,6 +468,11 @@ public sealed class InteractiveLaunchSpecService
             ["MCP_TOOL_TIMEOUT"] = McpToolTimeoutMs,
         };
 
+        // Persist the round that is about to start. Phase + task set only exist in the transient
+        // HandoffRequested broadcast otherwise, so without this a dead tile cannot be reopened at
+        // the same phase with the same set (see HandlerRestartCoordinator).
+        _roundState.Save(new HandlerRound(taskId, nextPhase, survivingTaskIds, DateTime.UtcNow));
+
         return new LaunchSpec(repoDir, resolvedClaude, args, env);
     }