fix(worker): claim Running before creating run resources
TaskRunner.RunAsync created the worktree (PrepareRunDirectoryAsync) before claiming Running via StartRunningAsync. RunNow dispatches by task id with no atomic claim of their own, so a Queued task racing the queue picker's atomic SQL claim could hit WorktreeManager's branch-collision self-heal, which force-removes and recreates the winner's live worktree mid-run. Move the claim before any resource creation and bail out immediately when it's rejected. OverrideSlotService.RunNow also fast-rejects a task already Running in the DB (defense in depth). RunCancellationRegistry now refuses (and logs) a double registration instead of silently overwriting the first CTS, so a losing dispatch's cleanup can no longer unregister the winner's cancellation token.
This commit is contained in:
@@ -33,9 +33,15 @@ public sealed class OverrideSlotService
|
||||
{
|
||||
using (var context = _dbFactory.CreateDbContext())
|
||||
{
|
||||
var exists = await new TaskRepository(context).GetByIdAsync(taskId);
|
||||
if (exists is null)
|
||||
var task = await new TaskRepository(context).GetByIdAsync(taskId);
|
||||
if (task is null)
|
||||
throw new KeyNotFoundException($"Task '{taskId}' not found.");
|
||||
|
||||
// Fast-fail precheck only — not the race guard. A task sitting Queued still
|
||||
// passes here and can race the queue picker's atomic claim; TaskRunner.RunAsync
|
||||
// resolves that by claiming Running before creating any resources.
|
||||
if (task.Status == Data.Models.TaskStatus.Running)
|
||||
throw new InvalidOperationException("task is already running");
|
||||
}
|
||||
|
||||
StartInSlot(taskId, ct => RunInSlotAsync(taskId, ct), "RunInSlotAsync failed for task {TaskId}");
|
||||
|
||||
@@ -10,8 +10,23 @@ namespace ClaudeDo.Worker.Queue;
|
||||
public sealed class RunCancellationRegistry
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, CancellationTokenSource> _running = new(StringComparer.Ordinal);
|
||||
private readonly ILogger<RunCancellationRegistry> _logger;
|
||||
|
||||
public void Register(string taskId, CancellationTokenSource cts) => _running[taskId] = cts;
|
||||
public RunCancellationRegistry(ILogger<RunCancellationRegistry> logger) => _logger = logger;
|
||||
|
||||
/// Registers the CTS driving <paramref name="taskId"/>'s run. Refuses (and logs) instead of
|
||||
/// silently overwriting when a CTS is already registered for this task id — an overwrite
|
||||
/// would mean a double-dispatch is in flight, and the caller that lost the race would then
|
||||
/// unregister the winner's CTS out from under it, leaving CancelAsync unable to reach the
|
||||
/// still-running process.
|
||||
public bool Register(string taskId, CancellationTokenSource cts)
|
||||
{
|
||||
if (_running.TryAdd(taskId, cts)) return true;
|
||||
|
||||
_logger.LogWarning(
|
||||
"Task {TaskId} already has a registered run; refusing to overwrite (double-dispatch?)", taskId);
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Removes the registration only if <paramref name="cts"/> is still the registered
|
||||
/// one — a re-run may already have registered a newer CTS under the same task id.
|
||||
|
||||
Reference in New Issue
Block a user