feat(worker): Handler-Session vor dem Usage-Limit warnen und Handoff bis zum Reset zurueckhalten
Eine PowerUse-Handler-Session soll am 5h-Limit nicht mehr mitten im Merge sterben, sondern vorher selbst sauber uebergeben: - HandlerUsageGuard (neu) warnt die laufende Session genau einmal pro Runde, sobald FiveHour.Utilization den Schwellwert PowerUseHandoffPct erreicht. Die Warnung wird per HandlerUsageWarning-Broadcast an Mission Control geschickt und dort in das Pane der Handler-TaskId getippt. - handoff_list_handler broadcastet den Handoff dann nicht sofort, sondern parkt ihn bis ResetsAt + 60 s; das Ergebnis traegt deferredUntil. Ohne ResetsAt wird wie bisher sofort gesendet. - Der geparkte Handoff liegt im HandlerRoundState (handler-rounds.json) und wird vom Guard-Tick ausgeloest, also auch nach einem Worker-Neustart wieder eingeplant bzw. sofort gesendet, wenn er ueberfaellig ist. - Review-Befund aus #272: HandlerRestartCoordinator wartet nicht mehr selbst bis ResetsAt, sondern parkt den Neustart im selben Slot. AutoRestartUsed wird erst gesetzt, wenn der Broadcast wirklich rausging - ein Worker-Absturz waehrend der Wartezeit laesst die Runde nicht mehr dauerhaft tot zurueck. Zurueckgehaltener Handoff und Auto-Neustart teilen sich einen Slot, also oeffnet nie beides dieselbe Runde. UsageLimitProofPct entfaellt zugunsten von PowerUseHandoffPct. - PtyTerminalSession.SendLineAsync schreibt in die laufende Session. Gegen eine echte claude-Session verifiziert: Text und Enter muessen zwei getrennte Writes sein, die Enter-Bytes kommen von Terminal.GenerateKeyInput.
This commit is contained in:
@@ -338,6 +338,7 @@
|
|||||||
"closeSession": "Sitzung beenden",
|
"closeSession": "Sitzung beenden",
|
||||||
"conptyLaunchFailed": "Interaktive Sitzung konnte nicht geöffnet werden: {0}",
|
"conptyLaunchFailed": "Interaktive Sitzung konnte nicht geöffnet werden: {0}",
|
||||||
"conptyStarting": "Sitzung wird gestartet…",
|
"conptyStarting": "Sitzung wird gestartet…",
|
||||||
|
"usageWarningNotDelivered": "Die Listen-Handler-Sitzung konnte nicht vor dem Usage-Limit gewarnt werden.",
|
||||||
"mergeHelperTitle": "Merge-Helfer",
|
"mergeHelperTitle": "Merge-Helfer",
|
||||||
"mergeHelperHandoffTitleSuffix": " (Übergabe)",
|
"mergeHelperHandoffTitleSuffix": " (Übergabe)",
|
||||||
"mergeHelperTaskTitle": "Listen-Handler: {0}",
|
"mergeHelperTaskTitle": "Listen-Handler: {0}",
|
||||||
|
|||||||
@@ -338,6 +338,7 @@
|
|||||||
"closeSession": "Kill session",
|
"closeSession": "Kill session",
|
||||||
"conptyLaunchFailed": "Couldn't open interactive session: {0}",
|
"conptyLaunchFailed": "Couldn't open interactive session: {0}",
|
||||||
"conptyStarting": "Starting session…",
|
"conptyStarting": "Starting session…",
|
||||||
|
"usageWarningNotDelivered": "Couldn't warn the list-handler session about the usage limit.",
|
||||||
"mergeHelperTitle": "Merge Helper",
|
"mergeHelperTitle": "Merge Helper",
|
||||||
"mergeHelperHandoffTitleSuffix": " (Handoff)",
|
"mergeHelperHandoffTitleSuffix": " (Handoff)",
|
||||||
"mergeHelperTaskTitle": "List handler: {0}",
|
"mergeHelperTaskTitle": "List handler: {0}",
|
||||||
|
|||||||
@@ -30,6 +30,11 @@ public interface IWorkerClient : INotifyPropertyChanged
|
|||||||
/// task to carry out nextPhase ("wait" | "merge" | "wait_final" | "merge_final").</summary>
|
/// task to carry out nextPhase ("wait" | "merge" | "wait_final" | "merge_final").</summary>
|
||||||
event Action<string, IReadOnlyList<string>, string>? HandoffRequestedEvent;
|
event Action<string, IReadOnlyList<string>, string>? HandoffRequestedEvent;
|
||||||
|
|
||||||
|
/// <summary>The 5h usage window is running out under a running list-handler session:
|
||||||
|
/// (handlerTaskId, message). Mission Control types the message into that task's ConPTY pane so
|
||||||
|
/// the session can hand off cleanly; no pane for the task means nothing happens.</summary>
|
||||||
|
event Action<string, string>? HandlerUsageWarningEvent;
|
||||||
|
|
||||||
/// <summary>An agent added a note through the MCP tool — an open notes editor reloads.</summary>
|
/// <summary>An agent added a note through the MCP tool — an open notes editor reloads.</summary>
|
||||||
event Action? NotesUpdatedEvent;
|
event Action? NotesUpdatedEvent;
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Reflection;
|
||||||
using System.Threading;
|
using System.Threading;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using Iciclecreek.Terminal;
|
using Iciclecreek.Terminal;
|
||||||
@@ -93,6 +94,56 @@ public sealed class PtyTerminalSession : IDisposable
|
|||||||
ProcessExited?.Invoke(this, e.ExitCode);
|
ProcessExited?.Invoke(this, e.ExitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Types <paramref name="text"/> into the running session and submits it, exactly as if the user
|
||||||
|
/// had typed it — used to warn a list-handler session before the usage limit
|
||||||
|
/// (<c>HandlerUsageWarning</c>). Verified against a real `claude` TUI, including while it was
|
||||||
|
/// mid-turn (the message is queued and acted on when the turn ends).
|
||||||
|
///
|
||||||
|
/// Three non-obvious details, all established by that spike:
|
||||||
|
/// • <see cref="TerminalControl"/> exposes no public write path at all — its only public paste is
|
||||||
|
/// <c>TerminalView.PasteAsync()</c>, which takes no argument and reads the clipboard. The
|
||||||
|
/// private <c>SendToPtyAsync</c> on the inner view is the write seam, hence the reflection.
|
||||||
|
/// • The text and the Enter must be TWO writes. A single <c>text + "\r"</c> chunk lands in
|
||||||
|
/// claude's composer but is never submitted — a chunk carrying an embedded CR reads as pasted
|
||||||
|
/// text, not as a keypress.
|
||||||
|
/// • The Enter bytes come from the emulator (<c>GenerateKeyInput</c>) rather than a literal
|
||||||
|
/// "\r", because claude turns Win32 input mode on and the emulator owns that encoding.
|
||||||
|
///
|
||||||
|
/// Best-effort by design: a package upgrade that renames the private member costs a warning that
|
||||||
|
/// never arrives, never a crashed Mission Control.
|
||||||
|
/// </summary>
|
||||||
|
public async Task<bool> SendLineAsync(string text)
|
||||||
|
{
|
||||||
|
if (_control is not { } control || !IsRunning || string.IsNullOrEmpty(text)) return false;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await SendRawAsync(control, text);
|
||||||
|
await Task.Delay(SubmitDelay);
|
||||||
|
await SendRawAsync(control, control.Terminal.GenerateKeyInput(XTerm.Input.Key.Enter, XTerm.Input.KeyModifiers.None));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (Exception)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Long enough that the two writes can't be read as one chunk by the child's stdin reader.
|
||||||
|
private static readonly TimeSpan SubmitDelay = TimeSpan.FromMilliseconds(150);
|
||||||
|
|
||||||
|
private static readonly FieldInfo? s_terminalViewField =
|
||||||
|
typeof(TerminalControl).GetField("_terminalView", BindingFlags.Instance | BindingFlags.NonPublic);
|
||||||
|
private static readonly MethodInfo? s_sendToPty =
|
||||||
|
typeof(TerminalView).GetMethod("SendToPtyAsync", BindingFlags.Instance | BindingFlags.NonPublic);
|
||||||
|
|
||||||
|
private static Task SendRawAsync(TerminalControl control, string data)
|
||||||
|
{
|
||||||
|
if (s_terminalViewField?.GetValue(control) is not TerminalView view || s_sendToPty is null)
|
||||||
|
throw new InvalidOperationException("Iciclecreek.Avalonia.Terminal no longer exposes a PTY write seam.");
|
||||||
|
return (Task)s_sendToPty.Invoke(view, new object[] { data, CancellationToken.None })!;
|
||||||
|
}
|
||||||
|
|
||||||
public void Kill()
|
public void Kill()
|
||||||
{
|
{
|
||||||
try { _control?.Kill(); }
|
try { _control?.Kill(); }
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ public partial class WorkerClient : ObservableObject, IAsyncDisposable, IWorkerC
|
|||||||
public event Action<string, string, string>? TaskQuestionAskedEvent;
|
public event Action<string, string, string>? TaskQuestionAskedEvent;
|
||||||
public event Action<string, string>? TaskQuestionResolvedEvent;
|
public event Action<string, string>? TaskQuestionResolvedEvent;
|
||||||
public event Action<string, IReadOnlyList<string>, string>? HandoffRequestedEvent;
|
public event Action<string, IReadOnlyList<string>, string>? HandoffRequestedEvent;
|
||||||
|
public event Action<string, string>? HandlerUsageWarningEvent;
|
||||||
public event Action? ConnectionRestoredEvent;
|
public event Action? ConnectionRestoredEvent;
|
||||||
public event Action<string>? WorktreeUpdatedEvent;
|
public event Action<string>? WorktreeUpdatedEvent;
|
||||||
public event Action<string>? ListUpdatedEvent;
|
public event Action<string>? ListUpdatedEvent;
|
||||||
@@ -159,6 +160,11 @@ public partial class WorkerClient : ObservableObject, IAsyncDisposable, IWorkerC
|
|||||||
Dispatcher.UIThread.Post(() => HandoffRequestedEvent?.Invoke(taskId, survivingTaskIds, nextPhase));
|
Dispatcher.UIThread.Post(() => HandoffRequestedEvent?.Invoke(taskId, survivingTaskIds, nextPhase));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
_hub.On<string, string>("HandlerUsageWarning", (taskId, message) =>
|
||||||
|
{
|
||||||
|
Dispatcher.UIThread.Post(() => HandlerUsageWarningEvent?.Invoke(taskId, message));
|
||||||
|
});
|
||||||
|
|
||||||
_hub.On<string>("WorktreeUpdated", taskId =>
|
_hub.On<string>("WorktreeUpdated", taskId =>
|
||||||
{
|
{
|
||||||
Dispatcher.UIThread.Post(() => WorktreeUpdatedEvent?.Invoke(taskId));
|
Dispatcher.UIThread.Post(() => WorktreeUpdatedEvent?.Invoke(taskId));
|
||||||
|
|||||||
@@ -131,6 +131,11 @@ public sealed partial class InteractiveTerminalViewModel : ViewModelBase, IDispo
|
|||||||
StartError = message;
|
StartError = message;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Types a line into the live session and submits it — see
|
||||||
|
/// <see cref="PtyTerminalSession.SendLineAsync"/>. False when there is no running process to
|
||||||
|
/// type into (or the write seam is gone).</summary>
|
||||||
|
public Task<bool> SendLineAsync(string text) => _session.SendLineAsync(text);
|
||||||
|
|
||||||
public void Kill() => _session.Kill();
|
public void Kill() => _session.Kill();
|
||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
|
|||||||
@@ -123,6 +123,10 @@ public sealed partial class ConPtyPaneViewModel : ViewModelBase, IDisposable
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Types a message into this pane's live session and submits it (the worker's
|
||||||
|
/// pre-usage-limit warning). False when the session isn't running.</summary>
|
||||||
|
public System.Threading.Tasks.Task<bool> SendLineAsync(string text) => Terminal.SendLineAsync(text);
|
||||||
|
|
||||||
[RelayCommand]
|
[RelayCommand]
|
||||||
private void Close() => CloseRequested?.Invoke(this);
|
private void Close() => CloseRequested?.Invoke(this);
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ public sealed partial class MissionControlViewModel : ViewModelBase, IDisposable
|
|||||||
private readonly Action<string> _onTaskUpdated;
|
private readonly Action<string> _onTaskUpdated;
|
||||||
private readonly Action _onConnectionRestored;
|
private readonly Action _onConnectionRestored;
|
||||||
private readonly Action<string, IReadOnlyList<string>, string> _onHandoffRequested;
|
private readonly Action<string, IReadOnlyList<string>, string> _onHandoffRequested;
|
||||||
|
private readonly Action<string, string> _onHandlerUsageWarning;
|
||||||
|
|
||||||
// Embedded ConPTY sessions (task-based only) — a manual cockpit detached from the
|
// Embedded ConPTY sessions (task-based only) — a manual cockpit detached from the
|
||||||
// review/merge/status machinery.
|
// review/merge/status machinery.
|
||||||
@@ -98,6 +99,9 @@ public sealed partial class MissionControlViewModel : ViewModelBase, IDisposable
|
|||||||
_onHandoffRequested = (taskId, survivingTaskIds, nextPhase) => { _ = OpenMergeHelperHandoffConPtySessionAsync(taskId, survivingTaskIds, nextPhase); };
|
_onHandoffRequested = (taskId, survivingTaskIds, nextPhase) => { _ = OpenMergeHelperHandoffConPtySessionAsync(taskId, survivingTaskIds, nextPhase); };
|
||||||
_worker.HandoffRequestedEvent += _onHandoffRequested;
|
_worker.HandoffRequestedEvent += _onHandoffRequested;
|
||||||
|
|
||||||
|
_onHandlerUsageWarning = (taskId, message) => { _ = WarnHandlerSessionAsync(taskId, message); };
|
||||||
|
_worker.HandlerUsageWarningEvent += _onHandlerUsageWarning;
|
||||||
|
|
||||||
_ = RefreshQueueAsync();
|
_ = RefreshQueueAsync();
|
||||||
_ = RefreshSessionTargetsAsync();
|
_ = RefreshSessionTargetsAsync();
|
||||||
}
|
}
|
||||||
@@ -406,6 +410,19 @@ public sealed partial class MissionControlViewModel : ViewModelBase, IDisposable
|
|||||||
() => DescribeAsync(() => _worker.GetMergeHelperHandoffLaunchSpecAsync(taskId, survivingTaskIds, nextPhase))));
|
() => DescribeAsync(() => _worker.GetMergeHelperHandoffLaunchSpecAsync(taskId, survivingTaskIds, nextPhase))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The 5h usage window is about to run out under this handler session: type the worker's warning
|
||||||
|
// into its pane so the session can hand off cleanly instead of dying mid-merge. The UI process
|
||||||
|
// owns the terminal, so this is the only place that can reach it at all. No pane (the tile was
|
||||||
|
// closed, or this is a second app instance) = nothing to warn, and that is not an error.
|
||||||
|
internal async System.Threading.Tasks.Task WarnHandlerSessionAsync(string taskId, string message)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(taskId) || string.IsNullOrEmpty(message)) return;
|
||||||
|
if (ConPtySessions.FirstOrDefault(s => s.TaskId == taskId) is not { } pane) return;
|
||||||
|
|
||||||
|
if (!await pane.SendLineAsync(message))
|
||||||
|
ErrorReported?.Invoke(Loc.T("missionControl.usageWarningNotDelivered"));
|
||||||
|
}
|
||||||
|
|
||||||
// Wires a freshly built pane and shows it immediately — the pane resolves its own launch spec,
|
// Wires a freshly built pane and shows it immediately — the pane resolves its own launch spec,
|
||||||
// so the tile is on screen (spinner running) while the worker is still preparing the worktree.
|
// so the tile is on screen (spinner running) while the worker is still preparing the worktree.
|
||||||
private void AddConPtyPane(ConPtyPaneViewModel pane)
|
private void AddConPtyPane(ConPtyPaneViewModel pane)
|
||||||
@@ -519,6 +536,7 @@ public sealed partial class MissionControlViewModel : ViewModelBase, IDisposable
|
|||||||
_worker.TaskUpdatedEvent -= _onTaskUpdated;
|
_worker.TaskUpdatedEvent -= _onTaskUpdated;
|
||||||
_worker.ConnectionRestoredEvent -= _onConnectionRestored;
|
_worker.ConnectionRestoredEvent -= _onConnectionRestored;
|
||||||
_worker.HandoffRequestedEvent -= _onHandoffRequested;
|
_worker.HandoffRequestedEvent -= _onHandoffRequested;
|
||||||
|
_worker.HandlerUsageWarningEvent -= _onHandlerUsageWarning;
|
||||||
ConPtySessions.CollectionChanged -= OnConPtySessionsChanged;
|
ConPtySessions.CollectionChanged -= OnConPtySessionsChanged;
|
||||||
Panes.CollectionChanged -= OnPanesChanged;
|
Panes.CollectionChanged -= OnPanesChanged;
|
||||||
foreach (var c in ConPtySessions.ToList())
|
foreach (var c in ConPtySessions.ToList())
|
||||||
|
|||||||
+21
-5
@@ -11,7 +11,11 @@ namespace ClaudeDo.Worker.External;
|
|||||||
// Coerced=true means nextPhase asked for a non-final phase but the server-side round cap
|
// Coerced=true means nextPhase asked for a non-final phase but the server-side round cap
|
||||||
// (HandoffRoundTracker) forced it to its "_final" counterpart instead -- NextPhase already
|
// (HandoffRoundTracker) forced it to its "_final" counterpart instead -- NextPhase already
|
||||||
// reflects the effective (post-coercion) value in that case.
|
// reflects the effective (post-coercion) value in that case.
|
||||||
public sealed record HandoffListHandlerResult(bool Requested, string TaskId, int Number, int SurvivingCount, string NextPhase, bool Coerced = false);
|
// DeferredUntil set means the next session was NOT opened yet: the 5h usage window is exhausted and
|
||||||
|
// the handoff is parked until it resets (HandlerUsageGuard).
|
||||||
|
public sealed record HandoffListHandlerResult(
|
||||||
|
bool Requested, string TaskId, int Number, int SurvivingCount, string NextPhase, bool Coerced = false,
|
||||||
|
DateTimeOffset? DeferredUntil = null);
|
||||||
|
|
||||||
public sealed record HandoffStatusResult(string TaskId, int Number, int Round, int Remaining, bool Final);
|
public sealed record HandoffStatusResult(string TaskId, int Number, int Round, int Remaining, bool Final);
|
||||||
|
|
||||||
@@ -22,17 +26,20 @@ public sealed class HandoffMcpTools
|
|||||||
private readonly HubBroadcaster _broadcaster;
|
private readonly HubBroadcaster _broadcaster;
|
||||||
private readonly HandoffRoundTracker _rounds;
|
private readonly HandoffRoundTracker _rounds;
|
||||||
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||||
|
private readonly HandlerUsageGuard _usageGuard;
|
||||||
|
|
||||||
public HandoffMcpTools(
|
public HandoffMcpTools(
|
||||||
TaskRepository tasks,
|
TaskRepository tasks,
|
||||||
HubBroadcaster broadcaster,
|
HubBroadcaster broadcaster,
|
||||||
HandoffRoundTracker rounds,
|
HandoffRoundTracker rounds,
|
||||||
IDbContextFactory<ClaudeDoDbContext> dbFactory)
|
IDbContextFactory<ClaudeDoDbContext> dbFactory,
|
||||||
|
HandlerUsageGuard usageGuard)
|
||||||
{
|
{
|
||||||
_tasks = tasks;
|
_tasks = tasks;
|
||||||
_broadcaster = broadcaster;
|
_broadcaster = broadcaster;
|
||||||
_rounds = rounds;
|
_rounds = rounds;
|
||||||
_dbFactory = dbFactory;
|
_dbFactory = dbFactory;
|
||||||
|
_usageGuard = usageGuard;
|
||||||
}
|
}
|
||||||
|
|
||||||
[McpServerTool, Description(
|
[McpServerTool, Description(
|
||||||
@@ -47,8 +54,10 @@ public sealed class HandoffMcpTools
|
|||||||
"4 by default, i.e. two " +
|
"4 by default, i.e. two " +
|
||||||
"full wait/merge cycles) for a given handler task: once reached, a \"wait\"/\"merge\" request is " +
|
"full wait/merge cycles) for a given handler task: once reached, a \"wait\"/\"merge\" request is " +
|
||||||
"silently coerced to its \"_final\" counterpart instead (check the result's coerced field), and " +
|
"silently coerced to its \"_final\" counterpart instead (check the result's coerced field), and " +
|
||||||
"any handoff at all after a merge_final round for this task throws. The current tile stays " +
|
"any handoff at all after a merge_final round for this task throws. If the 5h usage window is " +
|
||||||
"open; you must end your own turn immediately after calling this.")]
|
"already exhausted the next session is NOT opened right away -- the handoff is parked until " +
|
||||||
|
"the window resets and the result carries deferredUntil; say so in your summary. The current " +
|
||||||
|
"tile stays open; you must end your own turn immediately after calling this either way.")]
|
||||||
public async Task<HandoffListHandlerResult> HandoffListHandler(
|
public async Task<HandoffListHandlerResult> HandoffListHandler(
|
||||||
[Description("This session's own handler task id.")] string taskId,
|
[Description("This session's own handler task id.")] string taskId,
|
||||||
[Description("The tasks that made it past dedupe, in the order to run them.")] IReadOnlyList<string> survivingTaskIds,
|
[Description("The tasks that made it past dedupe, in the order to run them.")] IReadOnlyList<string> survivingTaskIds,
|
||||||
@@ -77,8 +86,15 @@ public sealed class HandoffMcpTools
|
|||||||
if (round.Rejected)
|
if (round.Rejected)
|
||||||
throw new InvalidOperationException(round.Error);
|
throw new InvalidOperationException(round.Error);
|
||||||
|
|
||||||
|
// The round is registered either way (the budget is spent by asking, not by the tile
|
||||||
|
// opening); only the broadcast waits when the 5h window is exhausted.
|
||||||
|
var deferredUntil = await _usageGuard.TryDeferHandoffAsync(
|
||||||
|
taskId, survivingTaskIds, round.EffectivePhase, cancellationToken);
|
||||||
|
if (deferredUntil is null)
|
||||||
await _broadcaster.HandoffRequested(taskId, survivingTaskIds, round.EffectivePhase);
|
await _broadcaster.HandoffRequested(taskId, survivingTaskIds, round.EffectivePhase);
|
||||||
return new HandoffListHandlerResult(true, taskId, task.Number, survivingTaskIds.Count, round.EffectivePhase, round.Coerced);
|
|
||||||
|
return new HandoffListHandlerResult(
|
||||||
|
true, taskId, task.Number, survivingTaskIds.Count, round.EffectivePhase, round.Coerced, deferredUntil);
|
||||||
}
|
}
|
||||||
|
|
||||||
[McpServerTool, Description(
|
[McpServerTool, Description(
|
||||||
|
|||||||
@@ -37,6 +37,12 @@ public sealed class HubBroadcaster : IPrimeBroadcaster, IRefineBroadcaster
|
|||||||
public Task HandoffRequested(string taskId, IReadOnlyList<string> survivingTaskIds, string nextPhase) =>
|
public Task HandoffRequested(string taskId, IReadOnlyList<string> survivingTaskIds, string nextPhase) =>
|
||||||
_hub.Clients.All.SendAsync("HandoffRequested", taskId, survivingTaskIds, nextPhase);
|
_hub.Clients.All.SendAsync("HandoffRequested", taskId, survivingTaskIds, nextPhase);
|
||||||
|
|
||||||
|
// The 5h usage window is about to run out under a running list-handler session (HandlerUsageGuard).
|
||||||
|
// Mission Control types `message` straight into that task's ConPTY pane, so the session can hand
|
||||||
|
// off cleanly instead of dying mid-merge. No pane for the task = nothing happens.
|
||||||
|
public Task HandlerUsageWarning(string handlerTaskId, string message) =>
|
||||||
|
_hub.Clients.All.SendAsync("HandlerUsageWarning", handlerTaskId, message);
|
||||||
|
|
||||||
/// <summary>An agent wrote a note through MCP — the open notes editor reloads.</summary>
|
/// <summary>An agent wrote a note through MCP — the open notes editor reloads.</summary>
|
||||||
public Task NotesUpdated() =>
|
public Task NotesUpdated() =>
|
||||||
_hub.Clients.All.SendAsync("NotesUpdated");
|
_hub.Clients.All.SendAsync("NotesUpdated");
|
||||||
|
|||||||
@@ -97,6 +97,8 @@ builder.Services.AddSingleton<ISessionSkillSeeder, SessionSkillSeeder>();
|
|||||||
builder.Services.AddSingleton<HandoffRoundTracker>();
|
builder.Services.AddSingleton<HandoffRoundTracker>();
|
||||||
builder.Services.AddSingleton<HandlerRoundState>();
|
builder.Services.AddSingleton<HandlerRoundState>();
|
||||||
builder.Services.AddSingleton<HandlerRestartCoordinator>();
|
builder.Services.AddSingleton<HandlerRestartCoordinator>();
|
||||||
|
builder.Services.AddSingleton<HandlerUsageGuard>();
|
||||||
|
builder.Services.AddHostedService(sp => sp.GetRequiredService<HandlerUsageGuard>());
|
||||||
builder.Services.AddSingleton<InteractiveLaunchSpecService>();
|
builder.Services.AddSingleton<InteractiveLaunchSpecService>();
|
||||||
builder.Services.AddSingleton<TaskRunner>();
|
builder.Services.AddSingleton<TaskRunner>();
|
||||||
builder.Services.AddSingleton<WorktreeMaintenanceService>();
|
builder.Services.AddSingleton<WorktreeMaintenanceService>();
|
||||||
@@ -339,6 +341,7 @@ if (cfg.ExternalMcpPort > 0)
|
|||||||
Share<AttachmentStore>();
|
Share<AttachmentStore>();
|
||||||
Share<FindingsStore>();
|
Share<FindingsStore>();
|
||||||
Share<HandoffRoundTracker>();
|
Share<HandoffRoundTracker>();
|
||||||
|
Share<HandlerUsageGuard>();
|
||||||
|
|
||||||
externalBuilder.Services.AddScoped<ClaudeDoDbContext>(sp =>
|
externalBuilder.Services.AddScoped<ClaudeDoDbContext>(sp =>
|
||||||
sp.GetRequiredService<IDbContextFactory<ClaudeDoDbContext>>().CreateDbContext());
|
sp.GetRequiredService<IDbContextFactory<ClaudeDoDbContext>>().CreateDbContext());
|
||||||
|
|||||||
@@ -20,17 +20,14 @@ namespace ClaudeDo.Worker.Runner;
|
|||||||
///
|
///
|
||||||
/// Fails CLOSED, unlike the usage gate: every missing signal means "don't relaunch". An interactive
|
/// Fails CLOSED, unlike the usage gate: every missing signal means "don't relaunch". An interactive
|
||||||
/// session that pops back up at 3am uninvited is worse than one that stays dead.
|
/// session that pops back up at 3am uninvited is worse than one that stays dead.
|
||||||
|
///
|
||||||
|
/// It does not wait here. The restart is <b>parked in <see cref="HandlerRoundState"/></b> and fired
|
||||||
|
/// by <see cref="HandlerUsageGuard"/> — the same slot a session that handed off cleanly before the
|
||||||
|
/// limit uses, so exactly one of the two reopens a round. Parking it also survives a worker restart
|
||||||
|
/// during the hours-long wait, which an in-process <c>Task.Delay</c> did not.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class HandlerRestartCoordinator
|
public sealed class HandlerRestartCoordinator
|
||||||
{
|
{
|
||||||
// The corroborating signal that this exit really was the usage limit. A ConPTY session gives us
|
|
||||||
// nothing but an exit code -- there is no stdout to match against, so TaskRunner's
|
|
||||||
// IsUsageLimitMessage regex (which reads the CLI's own "session limit reached" text) has nothing
|
|
||||||
// to read here. The 5h utilization at the moment of death is the one independent witness we do
|
|
||||||
// have, and it is pinned at 100% when the limit is what stopped the session. A crash, a manual
|
|
||||||
// Ctrl-C, or a bad launch at 40% utilization is therefore NOT restarted.
|
|
||||||
internal const int UsageLimitProofPct = 95;
|
|
||||||
|
|
||||||
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||||
private readonly UsageState _usageState;
|
private readonly UsageState _usageState;
|
||||||
private readonly HandlerRoundState _rounds;
|
private readonly HandlerRoundState _rounds;
|
||||||
@@ -51,14 +48,10 @@ public sealed class HandlerRestartCoordinator
|
|||||||
_logger = logger;
|
_logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Test seams — a real 5h wait is not a test.</summary>
|
/// <summary>Handles a ConPTY session's process exit. Returns true when a restart was parked for
|
||||||
internal Func<DateTimeOffset> NowUtc { get; set; } = () => DateTimeOffset.UtcNow;
|
/// the reset (HandlerUsageGuard fires it). Any non-handler task, clean exit, non-usage-limit
|
||||||
internal Func<TimeSpan, CancellationToken, Task> Delay { get; set; } = (d, ct) => Task.Delay(d, ct);
|
/// death, PowerUse-off list, already-used round, or round that already has a handoff parked is a
|
||||||
|
/// silent no-op.</summary>
|
||||||
/// <summary>Handles a ConPTY session's process exit. Returns true when a restart was claimed and
|
|
||||||
/// carried out (the returned task completes only after the wait, so a caller that wants
|
|
||||||
/// fire-and-forget must not await it). Any non-handler task, clean exit, non-usage-limit death,
|
|
||||||
/// PowerUse-off list, or already-used round is a silent no-op.</summary>
|
|
||||||
public async Task<bool> OnSessionExitedAsync(string taskId, int exitCode, CancellationToken ct = default)
|
public async Task<bool> OnSessionExitedAsync(string taskId, int exitCode, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
if (exitCode == 0 || string.IsNullOrEmpty(taskId)) return false;
|
if (exitCode == 0 || string.IsNullOrEmpty(taskId)) return false;
|
||||||
@@ -71,7 +64,6 @@ public sealed class HandlerRestartCoordinator
|
|||||||
|
|
||||||
var snapshot = _usageState.Snapshot;
|
var snapshot = _usageState.Snapshot;
|
||||||
if (snapshot?.FiveHour is not { } fiveHour) return false;
|
if (snapshot?.FiveHour is not { } fiveHour) return false;
|
||||||
if (fiveHour.Utilization < UsageLimitProofPct) return false;
|
|
||||||
if (fiveHour.ResetsAt is not { } resetsAt) return false;
|
if (fiveHour.ResetsAt is not { } resetsAt) return false;
|
||||||
|
|
||||||
int number;
|
int number;
|
||||||
@@ -85,8 +77,18 @@ public sealed class HandlerRestartCoordinator
|
|||||||
// Both PowerUse keys, exactly as ComposeHandlerPromptAsync gates the PowerUse prompt
|
// Both PowerUse keys, exactly as ComposeHandlerPromptAsync gates the PowerUse prompt
|
||||||
// addendum: the global switch alone must not make an interactive run on some unrelated
|
// addendum: the global switch alone must not make an interactive run on some unrelated
|
||||||
// list restart itself overnight.
|
// list restart itself overnight.
|
||||||
if (!(await new AppSettingsRepository(ctx).GetAsync(ct)).PowerUseEnabled) return false;
|
var settings = await new AppSettingsRepository(ctx).GetAsync(ct);
|
||||||
|
if (!settings.PowerUseEnabled) return false;
|
||||||
if ((await new ListRepository(ctx).GetConfigAsync(task.ListId, ct))?.PowerUseAllowed != true) return false;
|
if ((await new ListRepository(ctx).GetConfigAsync(task.ListId, ct))?.PowerUseAllowed != true) return false;
|
||||||
|
|
||||||
|
// The corroborating signal that this exit really was the usage limit. A ConPTY session
|
||||||
|
// gives us nothing but an exit code -- there is no stdout to match against, so
|
||||||
|
// TaskRunner's IsUsageLimitMessage regex (which reads the CLI's own "session limit
|
||||||
|
// reached" text) has nothing to read here. The 5h utilization at the moment of death is
|
||||||
|
// the one independent witness we do have. Same threshold the warning and the withheld
|
||||||
|
// handoff use (PowerUseHandoffPct) so all three agree on where "at the limit" starts;
|
||||||
|
// a crash or a manual Ctrl-C well below it is NOT restarted.
|
||||||
|
if (fiveHour.Utilization < Math.Clamp(settings.PowerUseHandoffPct, 50, 99)) return false;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
@@ -94,36 +96,16 @@ public sealed class HandlerRestartCoordinator
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// One restart per round, claimed before the wait -- a restart that dies again lands back
|
// One restart per round, and never on top of a handoff the session itself already parked.
|
||||||
// here and must stop, not relaunch.
|
// The budget is NOT spent here -- HandlerUsageGuard spends it when the broadcast goes out,
|
||||||
if (!_rounds.TryClaimAutoRestart(taskId)) return false;
|
// so a worker that dies during the wait doesn't leave the round marked "restart used".
|
||||||
|
var dueAt = resetsAt + HandlerUsageGuard.ResetGrace;
|
||||||
|
if (!_rounds.TryScheduleAutoRestart(taskId, dueAt, round.Phase, round.TaskIds)) return false;
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await _broadcaster.WorkerLog(
|
await _broadcaster.WorkerLog(
|
||||||
$"List handler #{number} died at the usage limit (5h {fiveHour.Utilization:0}%) — " +
|
$"List handler #{number} died at the usage limit (5h {fiveHour.Utilization:0}%) — " +
|
||||||
$"restarting phase {round.Phase} after the reset at {resetsAt.ToLocalTime():HH:mm}",
|
$"restarting phase {round.Phase} after the reset, at {dueAt.ToLocalTime():HH:mm}",
|
||||||
WorkerLogLevel.Warn, DateTime.UtcNow);
|
WorkerLogLevel.Warn, DateTime.UtcNow);
|
||||||
|
|
||||||
var wait = resetsAt - NowUtc();
|
|
||||||
if (wait > TimeSpan.Zero) await Delay(wait, ct);
|
|
||||||
|
|
||||||
await _broadcaster.HandoffRequested(taskId, round.TaskIds, round.Phase);
|
|
||||||
await _broadcaster.WorkerLog(
|
|
||||||
$"Auto-restarting list handler #{number} at phase {round.Phase} after usage-limit reset",
|
|
||||||
WorkerLogLevel.Info, DateTime.UtcNow);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
// Callers fire-and-forget this (WorkerHub.ReportInteractiveSessionExited), so an
|
|
||||||
// unobserved throw here would be invisible.
|
|
||||||
_logger.LogError(ex, "HandlerRestartCoordinator: auto-restart of handler task {TaskId} failed", taskId);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,17 +9,26 @@ namespace ClaudeDo.Worker.Runner;
|
|||||||
/// <c>HandoffRequested</c> broadcast that is the only other place these ids exist.</summary>
|
/// <c>HandoffRequested</c> broadcast that is the only other place these ids exist.</summary>
|
||||||
/// <param name="Phase">A <see cref="MergeHelperPhase"/> value, or "triage" for the initial session
|
/// <param name="Phase">A <see cref="MergeHelperPhase"/> value, or "triage" for the initial session
|
||||||
/// (which has no handoff spec and therefore cannot be reopened — see HandlerRestartCoordinator).</param>
|
/// (which has no handoff spec and therefore cannot be reopened — see HandlerRestartCoordinator).</param>
|
||||||
/// <param name="AutoRestartUsed">Claimed once per round by HandlerRestartCoordinator, so a handler
|
/// <param name="AutoRestartUsed">Spent once per round, and only once the deferred
|
||||||
/// that dies again after its auto-restart stays dead instead of relaunching all night.</param>
|
/// <c>HandoffRequested</c> broadcast has actually gone out — a restart that is merely *scheduled*
|
||||||
/// <param name="DeferredHandoffAt">Reserved for the "hand off cleanly before the usage limit"
|
/// must not count, or a worker that dies during the hours-long wait leaves the round permanently
|
||||||
/// follow-up, which parks a handoff until the 5h reset and needs to survive a worker restart.</param>
|
/// marked "restart used" without one ever having happened.</param>
|
||||||
|
/// <param name="DeferredHandoffAt">Due time of a handoff parked until the 5h usage window resets.
|
||||||
|
/// One slot per round, shared by both producers: a session that handed off cleanly just before the
|
||||||
|
/// limit (HandlerUsageGuard) and a session that died at it (HandlerRestartCoordinator) — so the two
|
||||||
|
/// can never reopen the same round twice. Fired by HandlerUsageGuard, which re-picks it up after a
|
||||||
|
/// worker restart because it lives in this file rather than in a timer.</param>
|
||||||
public sealed record HandlerRound(
|
public sealed record HandlerRound(
|
||||||
string HandlerTaskId,
|
string HandlerTaskId,
|
||||||
string Phase,
|
string Phase,
|
||||||
IReadOnlyList<string> TaskIds,
|
IReadOnlyList<string> TaskIds,
|
||||||
DateTime StartedAtUtc,
|
DateTime StartedAtUtc,
|
||||||
bool AutoRestartUsed = false,
|
bool AutoRestartUsed = false,
|
||||||
DateTimeOffset? DeferredHandoffAt = null);
|
DateTimeOffset? DeferredHandoffAt = null,
|
||||||
|
string? DeferredPhase = null,
|
||||||
|
IReadOnlyList<string>? DeferredTaskIds = null,
|
||||||
|
bool DeferredIsAutoRestart = false,
|
||||||
|
bool UsageWarningSent = false);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// File-backed store for <see cref="HandlerRound"/>, one entry per handler task. Deliberately a
|
/// File-backed store for <see cref="HandlerRound"/>, one entry per handler task. Deliberately a
|
||||||
@@ -74,22 +83,95 @@ public sealed class HandlerRoundState
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Takes this round's single auto-restart budget. False when there is no round for the
|
public IReadOnlyList<HandlerRound> All()
|
||||||
/// task or its budget is already spent — the same "never twice" discipline
|
{
|
||||||
/// <c>TaskRepository.TryClaimUsageLimitAutoContinueAsync</c> enforces for headless runs.</summary>
|
lock (_lock)
|
||||||
public bool TryClaimAutoRestart(string handlerTaskId)
|
return Rounds().Values.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Claims this round's single "warn the session before the usage limit" shot. True only
|
||||||
|
/// the first time, so a monitor that keeps ticking above the threshold does not retype the same
|
||||||
|
/// warning into the session every 30 seconds.</summary>
|
||||||
|
public bool TryMarkUsageWarned(string handlerTaskId)
|
||||||
{
|
{
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
var rounds = Rounds();
|
var rounds = Rounds();
|
||||||
if (!rounds.TryGetValue(handlerTaskId, out var round) || round.AutoRestartUsed) return false;
|
if (!rounds.TryGetValue(handlerTaskId, out var round) || round.UsageWarningSent) return false;
|
||||||
|
|
||||||
rounds[handlerTaskId] = round with { AutoRestartUsed = true };
|
rounds[handlerTaskId] = round with { UsageWarningSent = true };
|
||||||
Persist(rounds);
|
Persist(rounds);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>Parks a handoff the running session asked for until <paramref name="dueAt"/>.
|
||||||
|
/// Overwrites any existing slot: the live session's own request is the most recent word on what
|
||||||
|
/// the next round should be.</summary>
|
||||||
|
public void SetDeferredHandoff(
|
||||||
|
string handlerTaskId, DateTimeOffset dueAt, string phase, IReadOnlyList<string> taskIds)
|
||||||
|
=> WriteDeferred(handlerTaskId, dueAt, phase, taskIds, isAutoRestart: false);
|
||||||
|
|
||||||
|
/// <summary>Schedules this round's single auto-restart (the session died at the limit). False
|
||||||
|
/// when there is no round, the restart budget is already spent, or a handoff is *already* parked
|
||||||
|
/// for this round — the deferred handoff and the auto-restart are one trigger, not two.
|
||||||
|
/// <see cref="HandlerRound.AutoRestartUsed"/> is deliberately NOT set here; only
|
||||||
|
/// <see cref="ClearDeferredHandoff"/> spends it, once the broadcast really went out.</summary>
|
||||||
|
public bool TryScheduleAutoRestart(
|
||||||
|
string handlerTaskId, DateTimeOffset dueAt, string phase, IReadOnlyList<string> taskIds)
|
||||||
|
{
|
||||||
|
lock (_lock) // Monitor is reentrant — WriteDeferred takes the same lock.
|
||||||
|
{
|
||||||
|
var rounds = Rounds();
|
||||||
|
if (!rounds.TryGetValue(handlerTaskId, out var round)) return false;
|
||||||
|
if (round.AutoRestartUsed || round.DeferredHandoffAt is not null) return false;
|
||||||
|
|
||||||
|
WriteDeferred(handlerTaskId, dueAt, phase, taskIds, isAutoRestart: true);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Clears the parked handoff. <paramref name="spendAutoRestart"/> marks the round's
|
||||||
|
/// restart budget as used — pass the slot's own <see cref="HandlerRound.DeferredIsAutoRestart"/>,
|
||||||
|
/// so a clean pre-limit handoff never costs the next session its restart.</summary>
|
||||||
|
public void ClearDeferredHandoff(string handlerTaskId, bool spendAutoRestart)
|
||||||
|
{
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
var rounds = Rounds();
|
||||||
|
if (!rounds.TryGetValue(handlerTaskId, out var round)) return;
|
||||||
|
|
||||||
|
rounds[handlerTaskId] = round with
|
||||||
|
{
|
||||||
|
DeferredHandoffAt = null,
|
||||||
|
DeferredPhase = null,
|
||||||
|
DeferredTaskIds = null,
|
||||||
|
DeferredIsAutoRestart = false,
|
||||||
|
AutoRestartUsed = round.AutoRestartUsed || spendAutoRestart,
|
||||||
|
};
|
||||||
|
Persist(rounds);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void WriteDeferred(
|
||||||
|
string handlerTaskId, DateTimeOffset dueAt, string phase, IReadOnlyList<string> taskIds, bool isAutoRestart)
|
||||||
|
{
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
var rounds = Rounds();
|
||||||
|
if (!rounds.TryGetValue(handlerTaskId, out var round)) return;
|
||||||
|
|
||||||
|
rounds[handlerTaskId] = round with
|
||||||
|
{
|
||||||
|
DeferredHandoffAt = dueAt,
|
||||||
|
DeferredPhase = phase,
|
||||||
|
DeferredTaskIds = taskIds.ToList(),
|
||||||
|
DeferredIsAutoRestart = isAutoRestart,
|
||||||
|
};
|
||||||
|
Persist(rounds);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void Remove(string handlerTaskId)
|
public void Remove(string handlerTaskId)
|
||||||
{
|
{
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
|
|||||||
@@ -0,0 +1,207 @@
|
|||||||
|
using ClaudeDo.Data;
|
||||||
|
using ClaudeDo.Data.Models;
|
||||||
|
using ClaudeDo.Data.Repositories;
|
||||||
|
using ClaudeDo.Worker.External;
|
||||||
|
using ClaudeDo.Worker.Hub;
|
||||||
|
using ClaudeDo.Worker.Usage;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
|
namespace ClaudeDo.Worker.Runner;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Hands a PowerUse list-handler round over cleanly across the 5h usage limit, instead of letting it
|
||||||
|
/// die mid-merge:
|
||||||
|
///
|
||||||
|
/// 1. <b>Warn</b> — once per round, as soon as the 5h window crosses
|
||||||
|
/// <c>AppSettings.PowerUseHandoffPct</c>, a message is typed straight into the live ConPTY session
|
||||||
|
/// (<c>HandlerUsageWarning</c> → Mission Control) telling it to finish what it is doing and call
|
||||||
|
/// <c>handoff_list_handler</c> itself. A session that hands off on its own leaves no half-merged
|
||||||
|
/// tree behind; an auto-restart after the fact cannot clean one up.
|
||||||
|
/// 2. <b>Withhold</b> — the resulting handoff is not broadcast immediately (the next session would
|
||||||
|
/// open straight into the same exhausted window) but parked in <see cref="HandlerRoundState"/>
|
||||||
|
/// until the window resets. Same slot the auto-restart of a session that *did* die uses
|
||||||
|
/// (<see cref="HandlerRestartCoordinator"/>), so the two can never reopen one round twice.
|
||||||
|
/// 3. <b>Fire</b> — this service's tick is also what releases a parked handoff. Deliberately a poll
|
||||||
|
/// over the persisted state rather than a timer: a worker restart mid-wait re-picks the parked
|
||||||
|
/// handoff up on the next tick, and an overdue one goes out immediately, for free.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class HandlerUsageGuard : BackgroundService
|
||||||
|
{
|
||||||
|
internal static readonly TimeSpan TickInterval = TimeSpan.FromSeconds(30);
|
||||||
|
|
||||||
|
// Small cushion past ResetsAt: the reported reset time is the server's, and reopening a session
|
||||||
|
// one second early buys nothing but another dead tile.
|
||||||
|
internal static readonly TimeSpan ResetGrace = TimeSpan.FromSeconds(60);
|
||||||
|
|
||||||
|
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||||
|
private readonly UsageState _usageState;
|
||||||
|
private readonly HandlerRoundState _rounds;
|
||||||
|
private readonly HandoffRoundTracker _handoffRounds;
|
||||||
|
private readonly HubBroadcaster _broadcaster;
|
||||||
|
private readonly ILogger<HandlerUsageGuard> _logger;
|
||||||
|
|
||||||
|
public HandlerUsageGuard(
|
||||||
|
IDbContextFactory<ClaudeDoDbContext> dbFactory,
|
||||||
|
UsageState usageState,
|
||||||
|
HandlerRoundState rounds,
|
||||||
|
HandoffRoundTracker handoffRounds,
|
||||||
|
HubBroadcaster broadcaster,
|
||||||
|
ILogger<HandlerUsageGuard> logger)
|
||||||
|
{
|
||||||
|
_dbFactory = dbFactory;
|
||||||
|
_usageState = usageState;
|
||||||
|
_rounds = rounds;
|
||||||
|
_handoffRounds = handoffRounds;
|
||||||
|
_broadcaster = broadcaster;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Test seam — a real 5h wait is not a test.</summary>
|
||||||
|
internal Func<DateTimeOffset> NowUtc { get; set; } = () => DateTimeOffset.UtcNow;
|
||||||
|
|
||||||
|
/// <summary>PowerUse verdict for one handler task: both PowerUse switches on, a 5h bucket
|
||||||
|
/// available, plus the configured threshold and the task's display number.</summary>
|
||||||
|
internal sealed record Policy(int HandoffPct, UsageBucket FiveHour, int Number);
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await TickAsync(stoppingToken);
|
||||||
|
await Task.Delay(TickInterval, stoppingToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "HandlerUsageGuard: tick failed");
|
||||||
|
await Task.Delay(TickInterval, stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal async Task TickAsync(CancellationToken ct)
|
||||||
|
{
|
||||||
|
foreach (var round in _rounds.All())
|
||||||
|
{
|
||||||
|
await FireIfDueAsync(round);
|
||||||
|
await WarnIfNearLimitAsync(round, ct);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Called by <c>handoff_list_handler</c>. Returns the time the handoff was parked until
|
||||||
|
/// when it must be withheld, or null when it should go out now (PowerUse off for this list, well
|
||||||
|
/// below the threshold, or no reset time to wait for).</summary>
|
||||||
|
public async Task<DateTimeOffset?> TryDeferHandoffAsync(
|
||||||
|
string handlerTaskId, IReadOnlyList<string> survivingTaskIds, string phase, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var policy = await EvaluateAsync(handlerTaskId, ct);
|
||||||
|
if (policy is null) return null;
|
||||||
|
|
||||||
|
// The warning flag matters on its own: utilization can dip back under the threshold between
|
||||||
|
// the warning and the session actually finishing its merge, and a handoff produced *because*
|
||||||
|
// of the warning must still wait for the reset.
|
||||||
|
var warned = _rounds.Get(handlerTaskId)?.UsageWarningSent == true;
|
||||||
|
if (policy.FiveHour.Utilization < policy.HandoffPct && !warned) return null;
|
||||||
|
|
||||||
|
if (policy.FiveHour.ResetsAt is not { } resetsAt)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(
|
||||||
|
"HandlerUsageGuard: 5h window at {Pct}% has no reset time; handing off {TaskId} immediately",
|
||||||
|
policy.FiveHour.Utilization, handlerTaskId);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var dueAt = resetsAt + ResetGrace;
|
||||||
|
_rounds.SetDeferredHandoff(handlerTaskId, dueAt, phase, survivingTaskIds);
|
||||||
|
await _broadcaster.WorkerLog(
|
||||||
|
$"List handler #{policy.Number}: handoff to phase {phase} withheld until " +
|
||||||
|
$"{dueAt.ToLocalTime():HH:mm} (5h window at {policy.FiveHour.Utilization:0}%)",
|
||||||
|
WorkerLogLevel.Warn, DateTime.UtcNow);
|
||||||
|
return dueAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task FireIfDueAsync(HandlerRound round)
|
||||||
|
{
|
||||||
|
if (round.DeferredHandoffAt is not { } dueAt || dueAt > NowUtc()) return;
|
||||||
|
|
||||||
|
if (round.DeferredPhase is not { Length: > 0 } phase || round.DeferredTaskIds is not { Count: > 0 } taskIds)
|
||||||
|
{
|
||||||
|
_rounds.ClearDeferredHandoff(round.HandlerTaskId, spendAutoRestart: false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cleared BEFORE the broadcast so a throw on the way out can't leave a slot that re-fires on
|
||||||
|
// every tick; the restart budget is spent here too, which is the point of the whole
|
||||||
|
// rework -- it is now only spent once a handoff really went out.
|
||||||
|
_rounds.ClearDeferredHandoff(round.HandlerTaskId, spendAutoRestart: round.DeferredIsAutoRestart);
|
||||||
|
|
||||||
|
await _broadcaster.HandoffRequested(round.HandlerTaskId, taskIds, phase);
|
||||||
|
await _broadcaster.WorkerLog(
|
||||||
|
$"Reopening list handler at phase {phase} after the usage-limit reset",
|
||||||
|
WorkerLogLevel.Info, DateTime.UtcNow);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task WarnIfNearLimitAsync(HandlerRound round, CancellationToken ct)
|
||||||
|
{
|
||||||
|
if (round.UsageWarningSent) return;
|
||||||
|
// A sealed chain has no further handoff to ask for -- telling it to call
|
||||||
|
// handoff_list_handler would only earn the session an exception.
|
||||||
|
if (_handoffRounds.GetState(round.HandlerTaskId).Sealed) return;
|
||||||
|
|
||||||
|
var policy = await EvaluateAsync(round.HandlerTaskId, ct);
|
||||||
|
if (policy is null || policy.FiveHour.Utilization < policy.HandoffPct) return;
|
||||||
|
if (!_rounds.TryMarkUsageWarned(round.HandlerTaskId)) return;
|
||||||
|
|
||||||
|
await _broadcaster.HandlerUsageWarning(
|
||||||
|
round.HandlerTaskId, BuildWarning(round.HandlerTaskId, round.Phase, policy.FiveHour.Utilization));
|
||||||
|
await _broadcaster.WorkerLog(
|
||||||
|
$"List handler #{policy.Number}: warned the session at 5h {policy.FiveHour.Utilization:0}% " +
|
||||||
|
$"(handoff threshold {policy.HandoffPct}%)",
|
||||||
|
WorkerLogLevel.Warn, DateTime.UtcNow);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plain ASCII on purpose: this is typed character-by-character into a PTY, not rendered by us.
|
||||||
|
internal static string BuildWarning(string handlerTaskId, string phase, double utilization) =>
|
||||||
|
$"USAGE LIMIT IMMINENT ({utilization:0}% of the 5h window). Finish or abort the merge you are " +
|
||||||
|
"in - never leave conflict markers in the tree - then call " +
|
||||||
|
$"handoff_list_handler(taskId: \"{handlerTaskId}\", survivingTaskIds: <every task not yet Done>, " +
|
||||||
|
$"nextPhase: \"{NextPhaseFor(phase)}\") right now. Do not start anything new. " +
|
||||||
|
"The next session opens automatically after the limit resets.";
|
||||||
|
|
||||||
|
// The session should resume where it is, so the warning names its own phase back. The opening
|
||||||
|
// triage round is the exception: "triage" is not a handoff phase, and what follows triage is
|
||||||
|
// always the wait phase.
|
||||||
|
private static string NextPhaseFor(string phase) =>
|
||||||
|
MergeHelperPhase.All.Contains(phase, StringComparer.Ordinal) ? phase : MergeHelperPhase.Wait;
|
||||||
|
|
||||||
|
/// <summary>Both PowerUse switches, exactly as <c>ComposeHandlerPromptAsync</c> and
|
||||||
|
/// <see cref="HandlerRestartCoordinator"/> gate them — the global toggle alone must not make an
|
||||||
|
/// unrelated list's handler session get typed into. Null = leave this round alone.</summary>
|
||||||
|
private async Task<Policy?> EvaluateAsync(string handlerTaskId, CancellationToken ct)
|
||||||
|
{
|
||||||
|
if (_usageState.Snapshot?.FiveHour is not { } fiveHour) return null;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var ctx = await _dbFactory.CreateDbContextAsync(ct);
|
||||||
|
var task = await new TaskRepository(ctx).GetByIdAsync(handlerTaskId, ct);
|
||||||
|
if (task is null) return null;
|
||||||
|
|
||||||
|
var settings = await new AppSettingsRepository(ctx).GetAsync(ct);
|
||||||
|
if (!settings.PowerUseEnabled) return null;
|
||||||
|
if ((await new ListRepository(ctx).GetConfigAsync(task.ListId, ct))?.PowerUseAllowed != true) return null;
|
||||||
|
|
||||||
|
return new Policy(Math.Clamp(settings.PowerUseHandoffPct, 50, 99), fiveHour, task.Number);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "HandlerUsageGuard: could not evaluate handler task {TaskId}", handlerTaskId);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,6 +26,7 @@ public abstract class StubWorkerClient : IWorkerClient
|
|||||||
public event Action<string, string, string>? TaskQuestionAskedEvent;
|
public event Action<string, string, string>? TaskQuestionAskedEvent;
|
||||||
public event Action<string, string>? TaskQuestionResolvedEvent;
|
public event Action<string, string>? TaskQuestionResolvedEvent;
|
||||||
public event Action<string, IReadOnlyList<string>, string>? HandoffRequestedEvent;
|
public event Action<string, IReadOnlyList<string>, string>? HandoffRequestedEvent;
|
||||||
|
public event Action<string, string>? HandlerUsageWarningEvent;
|
||||||
public event Action? NotesUpdatedEvent;
|
public event Action? NotesUpdatedEvent;
|
||||||
public event Action? PrepStartedEvent;
|
public event Action? PrepStartedEvent;
|
||||||
public event Action<string>? PrepLineEvent;
|
public event Action<string>? PrepLineEvent;
|
||||||
@@ -57,6 +58,7 @@ public abstract class StubWorkerClient : IWorkerClient
|
|||||||
public void RaiseTaskQuestionAsked(string taskId, string questionId, string question) => TaskQuestionAskedEvent?.Invoke(taskId, questionId, question);
|
public void RaiseTaskQuestionAsked(string taskId, string questionId, string question) => TaskQuestionAskedEvent?.Invoke(taskId, questionId, question);
|
||||||
public void RaiseTaskQuestionResolved(string taskId, string questionId) => TaskQuestionResolvedEvent?.Invoke(taskId, questionId);
|
public void RaiseTaskQuestionResolved(string taskId, string questionId) => TaskQuestionResolvedEvent?.Invoke(taskId, questionId);
|
||||||
public void RaiseHandoffRequested(string taskId, IReadOnlyList<string> survivingTaskIds, string nextPhase = "wait") => HandoffRequestedEvent?.Invoke(taskId, survivingTaskIds, nextPhase);
|
public void RaiseHandoffRequested(string taskId, IReadOnlyList<string> survivingTaskIds, string nextPhase = "wait") => HandoffRequestedEvent?.Invoke(taskId, survivingTaskIds, nextPhase);
|
||||||
|
public void RaiseHandlerUsageWarning(string taskId, string message) => HandlerUsageWarningEvent?.Invoke(taskId, message);
|
||||||
public void RaisePlanningMergeConflict(string planningTaskId, string subtaskId, IReadOnlyList<string> files, bool externallyDriven)
|
public void RaisePlanningMergeConflict(string planningTaskId, string subtaskId, IReadOnlyList<string> files, bool externallyDriven)
|
||||||
=> PlanningMergeConflictEvent?.Invoke(planningTaskId, subtaskId, files, externallyDriven);
|
=> PlanningMergeConflictEvent?.Invoke(planningTaskId, subtaskId, files, externallyDriven);
|
||||||
public void RaiseOperationProgress(string opKey, string phase, int current, int total) => OperationProgressEvent?.Invoke(opKey, phase, current, total);
|
public void RaiseOperationProgress(string opKey, string phase, int current, int total) => OperationProgressEvent?.Invoke(opKey, phase, current, total);
|
||||||
|
|||||||
@@ -922,6 +922,38 @@ public class MissionControlViewModelTests : IDisposable
|
|||||||
pane.Terminal.OnSessionProcessExited(null, exitCode);
|
pane.Terminal.OnSessionProcessExited(null, exitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The worker's pre-usage-limit warning is typed into the pane for that handler task. There is no
|
||||||
|
// ConPTY process in a headless test, so SendLineAsync reports "not delivered" -- what this pins
|
||||||
|
// is the routing: the right pane is picked, and a warning for a task with no pane open is a
|
||||||
|
// silent no-op rather than an error in the footer.
|
||||||
|
[Fact]
|
||||||
|
public async Task HandlerUsageWarning_WithNoPaneForTheTask_IsSilentlyIgnored()
|
||||||
|
{
|
||||||
|
var worker = new FakeWorker();
|
||||||
|
using var vm = BuildVm(worker);
|
||||||
|
string? error = null;
|
||||||
|
vm.ErrorReported += msg => error = msg;
|
||||||
|
|
||||||
|
await vm.WarnHandlerSessionAsync("handler-nobody-opened", "USAGE LIMIT IMMINENT");
|
||||||
|
|
||||||
|
Assert.Null(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task HandlerUsageWarning_UndeliverableToItsPane_SurfacesAnError()
|
||||||
|
{
|
||||||
|
var worker = new FakeWorker();
|
||||||
|
using var vm = BuildVm(worker);
|
||||||
|
await vm.OpenMergeHelperHandoffConPtySessionAsync("handler-1", new[] { "t1" }, "merge");
|
||||||
|
Assert.Single(vm.ConPtySessions);
|
||||||
|
string? error = null;
|
||||||
|
vm.ErrorReported += msg => error = msg;
|
||||||
|
|
||||||
|
await vm.WarnHandlerSessionAsync("handler-1", "USAGE LIMIT IMMINENT");
|
||||||
|
|
||||||
|
Assert.NotNull(error);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void ToggleLayoutCommand_FlipsIsFocusMode()
|
public void ToggleLayoutCommand_FlipsIsFocusMode()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -3,8 +3,11 @@ using ClaudeDo.Data.Models;
|
|||||||
using ClaudeDo.Data.Repositories;
|
using ClaudeDo.Data.Repositories;
|
||||||
using ClaudeDo.Worker.External;
|
using ClaudeDo.Worker.External;
|
||||||
using ClaudeDo.Worker.Hub;
|
using ClaudeDo.Worker.Hub;
|
||||||
|
using ClaudeDo.Worker.Runner;
|
||||||
using ClaudeDo.Worker.Tests.Infrastructure;
|
using ClaudeDo.Worker.Tests.Infrastructure;
|
||||||
|
using ClaudeDo.Worker.Usage;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
|
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
|
||||||
|
|
||||||
namespace ClaudeDo.Worker.Tests.External;
|
namespace ClaudeDo.Worker.Tests.External;
|
||||||
@@ -16,18 +19,35 @@ public sealed class HandoffMcpToolsTests : IDisposable
|
|||||||
private readonly TaskRepository _tasks;
|
private readonly TaskRepository _tasks;
|
||||||
private readonly ListRepository _lists;
|
private readonly ListRepository _lists;
|
||||||
private readonly CapturingHubContext _hubContext = new();
|
private readonly CapturingHubContext _hubContext = new();
|
||||||
|
private readonly string _tempDir;
|
||||||
|
|
||||||
public HandoffMcpToolsTests()
|
public HandoffMcpToolsTests()
|
||||||
{
|
{
|
||||||
_ctx = _db.CreateContext();
|
_ctx = _db.CreateContext();
|
||||||
_tasks = new TaskRepository(_ctx);
|
_tasks = new TaskRepository(_ctx);
|
||||||
_lists = new ListRepository(_ctx);
|
_lists = new ListRepository(_ctx);
|
||||||
|
_tempDir = Path.Combine(Path.GetTempPath(), $"cd_handoffmcp_{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(_tempDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Dispose() { _ctx.Dispose(); _db.Dispose(); }
|
public void Dispose()
|
||||||
|
{
|
||||||
|
_ctx.Dispose();
|
||||||
|
_db.Dispose();
|
||||||
|
try { Directory.Delete(_tempDir, true); } catch { }
|
||||||
|
}
|
||||||
|
|
||||||
private HandoffMcpTools BuildSut(HandoffRoundTracker? tracker = null) =>
|
// No usage snapshot -> HandlerUsageGuard never withholds, which is the behaviour every test
|
||||||
new(_tasks, new HubBroadcaster(_hubContext), tracker ?? new HandoffRoundTracker(), _db.CreateFactory());
|
// below assumes. The withholding path has its own tests in HandlerUsageGuardTests.
|
||||||
|
private HandoffMcpTools BuildSut(HandoffRoundTracker? tracker = null)
|
||||||
|
{
|
||||||
|
var rounds = tracker ?? new HandoffRoundTracker();
|
||||||
|
var guard = new HandlerUsageGuard(
|
||||||
|
_db.CreateFactory(), new UsageState(),
|
||||||
|
new HandlerRoundState(Path.Combine(_tempDir, "handler-rounds.json")), rounds,
|
||||||
|
new HubBroadcaster(_hubContext), NullLogger<HandlerUsageGuard>.Instance);
|
||||||
|
return new HandoffMcpTools(_tasks, new HubBroadcaster(_hubContext), rounds, _db.CreateFactory(), guard);
|
||||||
|
}
|
||||||
|
|
||||||
/// The cap the SUT reads comes from app_settings, so a test that wants a different cap has to
|
/// The cap the SUT reads comes from app_settings, so a test that wants a different cap has to
|
||||||
/// write it there. GetAsync is AsNoTracking, hence the separate tracked query.
|
/// write it there. GetAsync is AsNoTracking, hence the separate tracked query.
|
||||||
|
|||||||
@@ -31,17 +31,18 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable
|
|||||||
|
|
||||||
public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } }
|
public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } }
|
||||||
|
|
||||||
private readonly List<TimeSpan> _waited = new();
|
private static readonly DateTimeOffset DueAt = ResetsAt + HandlerUsageGuard.ResetGrace;
|
||||||
|
|
||||||
private HandlerRestartCoordinator BuildCoordinator(UsageState usage)
|
private HandlerRestartCoordinator BuildCoordinator(UsageState usage) =>
|
||||||
{
|
new(_db.CreateFactory(), usage, _rounds, new HubBroadcaster(_hub),
|
||||||
var coordinator = new HandlerRestartCoordinator(
|
|
||||||
_db.CreateFactory(), usage, _rounds, new HubBroadcaster(_hub),
|
|
||||||
NullLogger<HandlerRestartCoordinator>.Instance);
|
NullLogger<HandlerRestartCoordinator>.Instance);
|
||||||
coordinator.NowUtc = () => ResetsAt.AddHours(-3);
|
|
||||||
coordinator.Delay = (d, _) => { _waited.Add(d); return Task.CompletedTask; };
|
/// The coordinator only parks the restart; this is what actually fires it. `now` decides whether
|
||||||
return coordinator;
|
/// the parked handoff is already due, so a 5h wait costs a test nothing.
|
||||||
}
|
private HandlerUsageGuard BuildGuard(UsageState usage, DateTimeOffset now) =>
|
||||||
|
new(_db.CreateFactory(), usage, _rounds, new HandoffRoundTracker(), new HubBroadcaster(_hub),
|
||||||
|
NullLogger<HandlerUsageGuard>.Instance)
|
||||||
|
{ NowUtc = () => now };
|
||||||
|
|
||||||
/// The one signal a ConPTY death gives the worker: the 5h window pinned at the limit.
|
/// The one signal a ConPTY death gives the worker: the 5h window pinned at the limit.
|
||||||
private static UsageState AtUsageLimit(double percent = 100) =>
|
private static UsageState AtUsageLimit(double percent = 100) =>
|
||||||
@@ -91,16 +92,45 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable
|
|||||||
{
|
{
|
||||||
var taskId = await SeedHandlerTaskAsync();
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a", "b" }, DateTime.UtcNow));
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a", "b" }, DateTime.UtcNow));
|
||||||
var coordinator = BuildCoordinator(AtUsageLimit());
|
|
||||||
|
|
||||||
Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1));
|
Assert.True(await BuildCoordinator(AtUsageLimit()).OnSessionExitedAsync(taskId, 1));
|
||||||
|
|
||||||
|
// Parked, not relaunched straight into the same exhausted window.
|
||||||
|
Assert.Empty(HandoffCalls);
|
||||||
|
Assert.Equal(DueAt, _rounds.Get(taskId)!.DeferredHandoffAt);
|
||||||
|
|
||||||
|
await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default);
|
||||||
|
|
||||||
var call = Assert.Single(HandoffCalls);
|
var call = Assert.Single(HandoffCalls);
|
||||||
Assert.Equal(taskId, call.Args[0]);
|
Assert.Equal(taskId, call.Args[0]);
|
||||||
Assert.Equal(new[] { "a", "b" }, (IReadOnlyList<string>)call.Args[1]!);
|
Assert.Equal(new[] { "a", "b" }, (IReadOnlyList<string>)call.Args[1]!);
|
||||||
Assert.Equal(MergeHelperPhase.Merge, call.Args[2]);
|
Assert.Equal(MergeHelperPhase.Merge, call.Args[2]);
|
||||||
// Waited out the window instead of relaunching straight into the same limit.
|
}
|
||||||
Assert.Equal(new[] { TimeSpan.FromHours(3) }, _waited);
|
|
||||||
|
// The #272 review finding: the restart budget used to be spent when the wait STARTED, so a
|
||||||
|
// worker that died during the hours-long wait left the round marked "restart used" without one
|
||||||
|
// ever having happened -- and the session stayed dead forever.
|
||||||
|
[Fact]
|
||||||
|
public async Task ParkedRestart_SurvivesAWorkerRestart_AndOnlySpendsTheBudgetOnceItFires()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
Assert.True(await BuildCoordinator(AtUsageLimit()).OnSessionExitedAsync(taskId, 1));
|
||||||
|
|
||||||
|
// Worker restart: a fresh state object re-reads the same file, nothing else carries over.
|
||||||
|
var afterRestart = new HandlerRoundState(Path.Combine(_tempDir, "handler-rounds.json"));
|
||||||
|
Assert.False(afterRestart.Get(taskId)!.AutoRestartUsed);
|
||||||
|
Assert.Equal(DueAt, afterRestart.Get(taskId)!.DeferredHandoffAt);
|
||||||
|
|
||||||
|
// Still parked while the window is open...
|
||||||
|
await BuildGuard(AtUsageLimit(), DueAt.AddMinutes(-5)).TickAsync(default);
|
||||||
|
Assert.Empty(HandoffCalls);
|
||||||
|
|
||||||
|
// ...and sent as soon as it is due, which is also when the budget is spent.
|
||||||
|
await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default);
|
||||||
|
Assert.Single(HandoffCalls);
|
||||||
|
Assert.True(_rounds.Get(taskId)!.AutoRestartUsed);
|
||||||
|
Assert.Null(_rounds.Get(taskId)!.DeferredHandoffAt);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── acceptance 2: at most one auto-restart per round ───────────────────────────────────────
|
// ── acceptance 2: at most one auto-restart per round ───────────────────────────────────────
|
||||||
@@ -112,9 +142,25 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable
|
|||||||
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
var coordinator = BuildCoordinator(AtUsageLimit());
|
var coordinator = BuildCoordinator(AtUsageLimit());
|
||||||
|
|
||||||
|
Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1));
|
||||||
|
await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default);
|
||||||
|
|
||||||
|
Assert.False(await coordinator.OnSessionExitedAsync(taskId, 1));
|
||||||
|
Assert.Single(HandoffCalls);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second death while the first restart is still parked must not overwrite or duplicate it.
|
||||||
|
[Fact]
|
||||||
|
public async Task SecondDeathWhileTheRestartIsStillParked_ChangesNothing()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
var coordinator = BuildCoordinator(AtUsageLimit());
|
||||||
|
|
||||||
Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1));
|
Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1));
|
||||||
Assert.False(await coordinator.OnSessionExitedAsync(taskId, 1));
|
Assert.False(await coordinator.OnSessionExitedAsync(taskId, 1));
|
||||||
|
|
||||||
|
await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default);
|
||||||
Assert.Single(HandoffCalls);
|
Assert.Single(HandoffCalls);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,6 +174,7 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable
|
|||||||
var coordinator = BuildCoordinator(AtUsageLimit());
|
var coordinator = BuildCoordinator(AtUsageLimit());
|
||||||
|
|
||||||
Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1));
|
Assert.True(await coordinator.OnSessionExitedAsync(taskId, 1));
|
||||||
|
await BuildGuard(AtUsageLimit(), DueAt).TickAsync(default);
|
||||||
// The relaunched session's launch spec writes the round again, same phase.
|
// The relaunched session's launch spec writes the round again, same phase.
|
||||||
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
|
||||||
@@ -176,11 +223,14 @@ public sealed class HandlerRestartCoordinatorTests : IDisposable
|
|||||||
tracker.Register(taskId, MergeHelperPhase.Merge, 4); // the run's real handoff into this phase
|
tracker.Register(taskId, MergeHelperPhase.Merge, 4); // the run's real handoff into this phase
|
||||||
|
|
||||||
using var ctx = _db.CreateContext();
|
using var ctx = _db.CreateContext();
|
||||||
|
var usage = AtUsageLimit();
|
||||||
var tools = new HandoffMcpTools(
|
var tools = new HandoffMcpTools(
|
||||||
new TaskRepository(ctx), new HubBroadcaster(_hub), tracker, _db.CreateFactory());
|
new TaskRepository(ctx), new HubBroadcaster(_hub), tracker, _db.CreateFactory(),
|
||||||
|
BuildGuard(usage, ResetsAt));
|
||||||
var before = await tools.GetHandoffStatus(taskId);
|
var before = await tools.GetHandoffStatus(taskId);
|
||||||
|
|
||||||
Assert.True(await BuildCoordinator(AtUsageLimit()).OnSessionExitedAsync(taskId, 1));
|
Assert.True(await BuildCoordinator(usage).OnSessionExitedAsync(taskId, 1));
|
||||||
|
await BuildGuard(usage, DueAt).TickAsync(default);
|
||||||
|
|
||||||
var after = await tools.GetHandoffStatus(taskId);
|
var after = await tools.GetHandoffStatus(taskId);
|
||||||
Assert.Equal(before.Round, after.Round);
|
Assert.Equal(before.Round, after.Round);
|
||||||
|
|||||||
@@ -0,0 +1,277 @@
|
|||||||
|
using ClaudeDo.Data.Models;
|
||||||
|
using ClaudeDo.Data.Repositories;
|
||||||
|
using ClaudeDo.Worker.External;
|
||||||
|
using ClaudeDo.Worker.Hub;
|
||||||
|
using ClaudeDo.Worker.Runner;
|
||||||
|
using ClaudeDo.Worker.Tests.Infrastructure;
|
||||||
|
using ClaudeDo.Worker.Usage;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
|
namespace ClaudeDo.Worker.Tests.Runner;
|
||||||
|
|
||||||
|
/// Handing a PowerUse list-handler round over cleanly across the 5h usage limit: warn the live
|
||||||
|
/// session once, then withhold the handoff it produces until the window resets. The UI half (the
|
||||||
|
/// warning actually reaching the ConPTY session) lives in Ui.Tests' MissionControlViewModel tests.
|
||||||
|
public sealed class HandlerUsageGuardTests : IDisposable
|
||||||
|
{
|
||||||
|
private readonly DbFixture _db = new();
|
||||||
|
private readonly string _tempDir;
|
||||||
|
private readonly CapturingHubContext _hub = new();
|
||||||
|
private readonly HandlerRoundState _rounds;
|
||||||
|
private readonly HandoffRoundTracker _handoffRounds = new();
|
||||||
|
|
||||||
|
private static readonly DateTimeOffset ResetsAt = new(2026, 9, 3, 2, 0, 0, TimeSpan.Zero);
|
||||||
|
private static readonly DateTimeOffset DueAt = ResetsAt + HandlerUsageGuard.ResetGrace;
|
||||||
|
private static readonly DateTimeOffset Now = ResetsAt.AddHours(-3);
|
||||||
|
|
||||||
|
public HandlerUsageGuardTests()
|
||||||
|
{
|
||||||
|
_tempDir = Path.Combine(Path.GetTempPath(), $"cd_usageguard_{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(_tempDir);
|
||||||
|
_rounds = new HandlerRoundState(RoundsPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string RoundsPath => Path.Combine(_tempDir, "handler-rounds.json");
|
||||||
|
|
||||||
|
public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } }
|
||||||
|
|
||||||
|
private static UsageState Usage(double percent, DateTimeOffset? resetsAt) =>
|
||||||
|
Snapshot(new UsageBucket(percent, resetsAt));
|
||||||
|
|
||||||
|
private static UsageState Snapshot(UsageBucket? fiveHour)
|
||||||
|
{
|
||||||
|
var state = new UsageState();
|
||||||
|
state.ReportSuccess(new UsageSnapshot(fiveHour, null, Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
private HandlerUsageGuard BuildGuard(UsageState usage, DateTimeOffset? now = null, HandlerRoundState? rounds = null) =>
|
||||||
|
new(_db.CreateFactory(), usage, rounds ?? _rounds, _handoffRounds, new HubBroadcaster(_hub),
|
||||||
|
NullLogger<HandlerUsageGuard>.Instance)
|
||||||
|
{ NowUtc = () => now ?? Now };
|
||||||
|
|
||||||
|
private int _nextNumber = 1;
|
||||||
|
|
||||||
|
private async Task<string> SeedHandlerTaskAsync(bool powerUseEnabled = true, bool powerUseAllowed = true)
|
||||||
|
{
|
||||||
|
var listId = Guid.NewGuid().ToString();
|
||||||
|
var taskId = Guid.NewGuid().ToString();
|
||||||
|
using (var ctx = _db.CreateContext())
|
||||||
|
{
|
||||||
|
ctx.Lists.Add(new ListEntity { Id = listId, Name = "L", WorkingDir = _tempDir, CreatedAt = DateTime.UtcNow });
|
||||||
|
ctx.ListConfigs.Add(new ListConfigEntity { ListId = listId, PowerUseAllowed = powerUseAllowed });
|
||||||
|
ctx.Tasks.Add(new TaskEntity
|
||||||
|
{
|
||||||
|
Id = taskId, ListId = listId, Title = "Handler", Number = _nextNumber++,
|
||||||
|
IsManual = true, CreatedAt = DateTime.UtcNow,
|
||||||
|
});
|
||||||
|
await ctx.SaveChangesAsync();
|
||||||
|
|
||||||
|
// Written straight to the row: AppSettingsRepository.UpdateAsync deliberately doesn't
|
||||||
|
// copy PowerUseEnabled (the usage monitor owns that toggle, not the settings modal).
|
||||||
|
await new AppSettingsRepository(ctx).GetAsync();
|
||||||
|
var settings = await ctx.AppSettings.FirstAsync(s => s.Id == AppSettingsEntity.SingletonId);
|
||||||
|
settings.PowerUseEnabled = powerUseEnabled;
|
||||||
|
await ctx.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
return taskId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private IReadOnlyList<CapturedHubCall> Calls(string method) =>
|
||||||
|
_hub.Proxy.Calls.Where(c => c.Method == method).ToList();
|
||||||
|
|
||||||
|
private IReadOnlyList<CapturedHubCall> Warnings => Calls("HandlerUsageWarning");
|
||||||
|
private IReadOnlyList<CapturedHubCall> HandoffCalls => Calls("HandoffRequested");
|
||||||
|
|
||||||
|
// ── acceptance 1: exactly one warning per round ────────────────────────────────────────────
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AtTheThreshold_WarnsTheSessionExactlyOncePerRound()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
var guard = BuildGuard(Usage(95, ResetsAt));
|
||||||
|
|
||||||
|
await guard.TickAsync(default);
|
||||||
|
await guard.TickAsync(default);
|
||||||
|
await BuildGuard(Usage(99, ResetsAt)).TickAsync(default);
|
||||||
|
|
||||||
|
var warning = Assert.Single(Warnings);
|
||||||
|
Assert.Equal(taskId, warning.Args[0]);
|
||||||
|
var message = (string)warning.Args[1]!;
|
||||||
|
Assert.Contains("USAGE LIMIT IMMINENT", message);
|
||||||
|
// Names the session's own phase back, so it resumes where it is.
|
||||||
|
Assert.Contains($"nextPhase: \"{MergeHelperPhase.Merge}\"", message);
|
||||||
|
Assert.Contains(taskId, message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BelowTheThreshold_NoWarning()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
|
||||||
|
await BuildGuard(Usage(94, ResetsAt)).TickAsync(default);
|
||||||
|
|
||||||
|
Assert.Empty(Warnings);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The triage round has no "triage" handoff phase -- what follows it is always the wait phase.
|
||||||
|
[Fact]
|
||||||
|
public async Task TriageRound_IsToldToHandOffToTheWaitPhase()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(
|
||||||
|
taskId, InteractiveLaunchSpecService.TriagePhase, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
|
||||||
|
await BuildGuard(Usage(96, ResetsAt)).TickAsync(default);
|
||||||
|
|
||||||
|
Assert.Contains($"nextPhase: \"{MergeHelperPhase.Wait}\"", (string)Assert.Single(Warnings).Args[1]!);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A sealed chain can't hand off at all -- telling it to would only earn the session an exception.
|
||||||
|
[Fact]
|
||||||
|
public async Task SealedChain_IsNotWarned()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.MergeFinal, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
_handoffRounds.Register(taskId, MergeHelperPhase.MergeFinal);
|
||||||
|
|
||||||
|
await BuildGuard(Usage(99, ResetsAt)).TickAsync(default);
|
||||||
|
|
||||||
|
Assert.Empty(Warnings);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── acceptance 5: PowerUse gates both halves ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData(false, true)]
|
||||||
|
[InlineData(true, false)]
|
||||||
|
public async Task PowerUseOff_NeitherWarnsNorWithholds(bool powerUseEnabled, bool powerUseAllowed)
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync(powerUseEnabled, powerUseAllowed);
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
var guard = BuildGuard(Usage(99, ResetsAt));
|
||||||
|
|
||||||
|
await guard.TickAsync(default);
|
||||||
|
|
||||||
|
Assert.Empty(Warnings);
|
||||||
|
Assert.Null(await guard.TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default));
|
||||||
|
Assert.Null(_rounds.Get(taskId)!.DeferredHandoffAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── acceptance 2 + 3: withholding the handoff ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task BelowTheThreshold_HandoffIsNotWithheld()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
|
||||||
|
var deferred = await BuildGuard(Usage(94, ResetsAt))
|
||||||
|
.TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default);
|
||||||
|
|
||||||
|
Assert.Null(deferred);
|
||||||
|
Assert.Null(_rounds.Get(taskId)!.DeferredHandoffAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AtTheThreshold_HandoffIsWithheldUntilTheResetPlusGrace()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Wait, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
var guard = BuildGuard(Usage(95, ResetsAt));
|
||||||
|
|
||||||
|
var deferred = await guard.TryDeferHandoffAsync(taskId, new[] { "a", "b" }, MergeHelperPhase.Merge, default);
|
||||||
|
|
||||||
|
Assert.Equal(DueAt, deferred);
|
||||||
|
Assert.Empty(HandoffCalls);
|
||||||
|
|
||||||
|
// Still parked before it is due, sent afterwards -- with the handed-off phase and task set,
|
||||||
|
// not the round's own.
|
||||||
|
await BuildGuard(Usage(95, ResetsAt), DueAt.AddMinutes(-1)).TickAsync(default);
|
||||||
|
Assert.Empty(HandoffCalls);
|
||||||
|
|
||||||
|
await BuildGuard(Usage(95, ResetsAt), DueAt).TickAsync(default);
|
||||||
|
var call = Assert.Single(HandoffCalls);
|
||||||
|
Assert.Equal(taskId, call.Args[0]);
|
||||||
|
Assert.Equal(new[] { "a", "b" }, (IReadOnlyList<string>)call.Args[1]!);
|
||||||
|
Assert.Equal(MergeHelperPhase.Merge, call.Args[2]);
|
||||||
|
// A clean handoff must not cost the next session its auto-restart budget.
|
||||||
|
Assert.False(_rounds.Get(taskId)!.AutoRestartUsed);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Utilization can dip back below the threshold while the session finishes its merge; the handoff
|
||||||
|
// that the warning ASKED for still has to wait for the reset.
|
||||||
|
[Fact]
|
||||||
|
public async Task AfterAWarning_HandoffIsWithheldEvenIfUtilizationDropped()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
await BuildGuard(Usage(97, ResetsAt)).TickAsync(default);
|
||||||
|
Assert.Single(Warnings);
|
||||||
|
|
||||||
|
var deferred = await BuildGuard(Usage(80, ResetsAt))
|
||||||
|
.TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default);
|
||||||
|
|
||||||
|
Assert.Equal(DueAt, deferred);
|
||||||
|
}
|
||||||
|
|
||||||
|
// No reset time = nothing to wait for. Sending now beats parking a handoff that never fires.
|
||||||
|
[Fact]
|
||||||
|
public async Task NoResetTime_HandsOffImmediately()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
|
||||||
|
var deferred = await BuildGuard(Usage(99, null))
|
||||||
|
.TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default);
|
||||||
|
|
||||||
|
Assert.Null(deferred);
|
||||||
|
Assert.Null(_rounds.Get(taskId)!.DeferredHandoffAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── acceptance 4: a parked handoff survives a worker restart ───────────────────────────────
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task WorkerRestart_ReschedulesAParkedHandoff_AndSendsAnOverdueOneAtOnce()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
await BuildGuard(Usage(99, ResetsAt))
|
||||||
|
.TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default);
|
||||||
|
|
||||||
|
// Worker restart: nothing but the file carries over.
|
||||||
|
var afterRestart = new HandlerRoundState(RoundsPath);
|
||||||
|
Assert.Equal(DueAt, afterRestart.Get(taskId)!.DeferredHandoffAt);
|
||||||
|
|
||||||
|
// Not due yet -> still parked.
|
||||||
|
await BuildGuard(Usage(99, ResetsAt), Now, afterRestart).TickAsync(default);
|
||||||
|
Assert.Empty(HandoffCalls);
|
||||||
|
|
||||||
|
// Long overdue (the worker was down past the reset) -> goes out on the first tick.
|
||||||
|
await BuildGuard(Usage(99, ResetsAt), DueAt.AddHours(2), afterRestart).TickAsync(default);
|
||||||
|
Assert.Single(HandoffCalls);
|
||||||
|
Assert.Null(afterRestart.Get(taskId)!.DeferredHandoffAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
// One trigger, one state: a parked handoff blocks the auto-restart of the same round, so the
|
||||||
|
// round can never be reopened twice.
|
||||||
|
[Fact]
|
||||||
|
public async Task AParkedHandoff_BlocksTheAutoRestartOfTheSameRound()
|
||||||
|
{
|
||||||
|
var taskId = await SeedHandlerTaskAsync();
|
||||||
|
_rounds.Save(new HandlerRound(taskId, MergeHelperPhase.Merge, new[] { "a" }, DateTime.UtcNow));
|
||||||
|
var usage = Usage(99, ResetsAt);
|
||||||
|
await BuildGuard(usage).TryDeferHandoffAsync(taskId, new[] { "a" }, MergeHelperPhase.Merge, default);
|
||||||
|
|
||||||
|
var coordinator = new HandlerRestartCoordinator(
|
||||||
|
_db.CreateFactory(), usage, _rounds, new HubBroadcaster(_hub),
|
||||||
|
NullLogger<HandlerRestartCoordinator>.Instance);
|
||||||
|
Assert.False(await coordinator.OnSessionExitedAsync(taskId, 1));
|
||||||
|
|
||||||
|
await BuildGuard(usage, DueAt).TickAsync(default);
|
||||||
|
Assert.Single(HandoffCalls);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user