fix(worker): Stuck-Running-Fenster in Continue- und Stop-Pfad schließen

TaskRunner.ContinueAsync: Claim, SeedAsync und SetupMcpConfigAsync liefen
vor dem try/catch (anders als RunAsync) - warf einer davon nach dem
Running-Claim, propagierte die Exception ungefangen bis zu
OverrideSlotService.RunContinueInSlotAsync, das nur loggt. Task blieb
Running. Fix: derselbe Aufbau wie RunAsync (Claim+Seed+MCP-Setup im
try, MarkFailed im catch, mcpToken/mcpConfigPath vor dem try auf null).
SetupMcpConfigAsync bekommt zusätzlich einen onTokenRegistered-Callback,
damit die äußere mcpToken-Variable den Token sofort nach dem Register
sieht - sonst hätte ein Fehler zwischen Register und Rückgabe (z.B.
File.WriteAllTextAsync) den Token in der TaskRunTokenRegistry geleakt
(betrifft denselben Aufruf in RunAsync mit, daher dort ebenfalls
verdrahtet - RunAsync-Struktur selbst unverändert).

QueueService.RunInSlotAsync: der Stop-Button (CancelTask) cancelt die
Slot-CTS direkt ohne TaskStateService-Schreibzugriff. Traf das die
Pre-Dispatch-DB-Reads, loggte der OCE-Catch nur und die vom Picker
bereits auf Running geclaimte Task blieb dort für immer hängen. Fix:
Status im Catch neu lesen und nur wenn er noch Running ist über
TaskStateService.CancelAsync auf Cancelled setzen - ein Ursprung, der
bereits selbst einen Terminalstatus geschrieben hat (z.B. CancelReview),
wird nicht überschrieben. Kommentar korrigiert.

TDD: neue Tests in ContinueAsyncExceptionTests (Seed-/MCP-Setup-Fehler
nach Claim -> Failed, kein Token-Leak) und QueueServiceSlotFailureTests
(Stop während Pre-Dispatch -> Cancelled statt Running; ein bereits
terminal geschriebener Status wird nicht gestompt) vorher rot, jetzt
grün. Worker.Tests: 1213/1213 grün, Worker baut in Release.
This commit is contained in:
mika kuns
2026-08-20 15:00:14 +02:00
parent 4cf08f8159
commit 389c9126c8
6 changed files with 266 additions and 49 deletions
+25 -2
View File
@@ -348,9 +348,32 @@ public sealed class QueueService : BackgroundService
}
catch (OperationCanceledException)
{
// Cancellation is driven by the cancel path, which already wrote the terminal status.
// Marking the task Failed here would be a regression (it would stomp Cancelled).
// Most cancellation origins (TaskStateService.CancelAsync, the runner's own
// OCE handling) already write a terminal status before the CTS is actually
// cancelled, so by the time we get here the task has already left Running.
// A plain Stop (QueueService.CancelTask cancels the slot's CTS directly, with no
// TaskStateService write) landing during the pre-dispatch reads above is the one
// gap nothing else covers — the picker's claim already committed status='running'
// and nothing since has moved it off that. Only close that specific gap: write
// Cancelled when (and only when) the task is still Running, so an origin that
// already wrote its own terminal status is never stomped.
_logger.LogInformation("Slot runner cancelled for task {TaskId}", taskId);
try
{
TaskStatus current;
using (var context = _dbFactory.CreateDbContext())
current = await context.Tasks.AsNoTracking()
.Where(t => t.Id == taskId)
.Select(t => t.Status)
.FirstAsync(CancellationToken.None);
if (current == TaskStatus.Running)
await _state.CancelAsync(taskId, DateTime.UtcNow, CancellationToken.None);
}
catch (Exception cancelEx)
{
_logger.LogError(cancelEx, "Could not finalize cancelled task {TaskId} after slot cancellation", taskId);
}
}
catch (Exception ex)
{
@@ -17,6 +17,11 @@ public sealed class TaskRunTokenRegistry
return false;
}
public void Unregister(string token) => _tokenToTaskId.TryRemove(token, out _);
// Test-only leak check: a run must never leave its token registered after it ends,
// no matter which step of setup failed.
public bool HasAnyTokenFor(string taskId) => _tokenToTaskId.Values.Contains(taskId);
public static string GenerateToken()
{
var bytes = RandomNumberGenerator.GetBytes(32);
+39 -28
View File
@@ -124,7 +124,7 @@ public sealed class TaskRunner
var runDir = prep.RunDir!;
var resolvedConfig = await ResolveConfigAsync(task, list, listConfig, null, ct);
(mcpToken, mcpConfigPath, resolvedConfig) = await SetupMcpConfigAsync(task, resolvedConfig, ct);
(mcpToken, mcpConfigPath, resolvedConfig) = await SetupMcpConfigAsync(task, resolvedConfig, ct, token => mcpToken = token);
await _skillSeeder.SeedAsync(runDir, resolvedConfig.SkillNames, wtCtx is not null, ct);
@@ -237,32 +237,35 @@ public sealed class TaskRunner
runDir = Path.Combine(_cfg.SandboxRoot, taskId);
}
var now = DateTime.UtcNow;
// See RunAsync: queue dispatches arrive pre-claimed by the picker.
if (!alreadyClaimed)
{
var startResult = await _state.StartRunningAsync(taskId, now, ct);
if (!startResult.Ok)
{
_logger.LogWarning("Task {TaskId} skipped: StartRunningAsync rejected ({Reason})", taskId, startResult.Reason);
return;
}
}
else
{
// Queue-claimed dispatches skip StartRunningAsync, so broadcast TaskUpdated here
// (see RunAsync for the full rationale).
await _broadcaster.TaskUpdated(taskId);
}
await _broadcaster.TaskStarted(slot, taskId, now);
await _skillSeeder.SeedAsync(runDir, resolvedConfig.SkillNames, wtCtx is not null, ct);
var (mcpToken, mcpConfigPath, mcpConfig) = await SetupMcpConfigAsync(task, resolvedConfig, ct);
resolvedConfig = mcpConfig;
string? mcpToken = null;
string? mcpConfigPath = null;
try
{
var now = DateTime.UtcNow;
// See RunAsync: queue dispatches arrive pre-claimed by the picker. Claim, skill
// seeding, and MCP setup all live inside this try (mirroring RunAsync) so a failure
// in any of them still lands the task in Failed instead of stuck Running.
if (!alreadyClaimed)
{
var startResult = await _state.StartRunningAsync(taskId, now, ct);
if (!startResult.Ok)
{
_logger.LogWarning("Task {TaskId} skipped: StartRunningAsync rejected ({Reason})", taskId, startResult.Reason);
return;
}
}
else
{
// Queue-claimed dispatches skip StartRunningAsync, so broadcast TaskUpdated here
// (see RunAsync for the full rationale).
await _broadcaster.TaskUpdated(taskId);
}
await _broadcaster.TaskStarted(slot, taskId, now);
await _skillSeeder.SeedAsync(runDir, resolvedConfig.SkillNames, wtCtx is not null, ct);
(mcpToken, mcpConfigPath, resolvedConfig) = await SetupMcpConfigAsync(task, resolvedConfig, ct, token => mcpToken = token);
var nextRunNumber = lastRun.RunNumber + 1;
var result = await RunOnceAsync(taskId, task.Number, task.Title, slot, runDir, resolvedConfig, nextRunNumber, false, followUpPrompt, ct);
@@ -290,8 +293,12 @@ public sealed class TaskRunner
}
finally
{
_tokens.Unregister(mcpToken);
try { File.Delete(mcpConfigPath); } catch { /* best effort */ }
if (mcpToken is not null)
{
_tokens.Unregister(mcpToken);
if (mcpConfigPath is not null)
try { File.Delete(mcpConfigPath); } catch { /* best effort */ }
}
}
}
@@ -300,11 +307,15 @@ public sealed class TaskRunner
// SuggestImprovement for filing out-of-scope follow-ups. Used by both a fresh run and a
// --resume continuation, which needs the same wiring or the resumed session loses every
// mcp__claudedo_run__* tool.
// onTokenRegistered fires the instant the token is registered, before any I/O that could
// still fail — callers assign it straight into their outer (pre-try) mcpToken variable so a
// later throw in this method still leaves the finally block able to unregister it.
private async Task<(string McpToken, string McpConfigPath, ClaudeRunConfig Config)> SetupMcpConfigAsync(
TaskEntity task, ClaudeRunConfig resolvedConfig, CancellationToken ct)
TaskEntity task, ClaudeRunConfig resolvedConfig, CancellationToken ct, Action<string>? onTokenRegistered = null)
{
var mcpToken = TaskRunTokenRegistry.GenerateToken();
_tokens.Register(mcpToken, task.Id);
onTokenRegistered?.Invoke(mcpToken);
Directory.CreateDirectory(_cfg.LogRoot);
var mcpConfigPath = Path.Combine(_cfg.LogRoot, $"{task.Id}_mcp.json");
await File.WriteAllTextAsync(mcpConfigPath, BuildRunMcpConfigJson(mcpToken), ct);