feat(worker): add WorktreeMaintenanceService for idle-worktree cleanup
This commit is contained in:
@@ -29,6 +29,7 @@ builder.Services.AddSingleton<GitService>();
|
||||
builder.Services.AddSingleton<WorktreeManager>();
|
||||
builder.Services.AddSingleton<ClaudeArgsBuilder>();
|
||||
builder.Services.AddSingleton<TaskRunner>();
|
||||
builder.Services.AddSingleton<WorktreeMaintenanceService>();
|
||||
|
||||
// Agent file management.
|
||||
var agentsDir = Path.Combine(ClaudeDo.Data.Paths.AppDataRoot(), "agents");
|
||||
|
||||
@@ -39,8 +39,19 @@ public sealed class WorktreeManager
|
||||
var branchName = $"claudedo/{idForBranch}";
|
||||
var slug = CommitMessageBuilder.ToSlug(list.Name);
|
||||
|
||||
var worktreePath = _cfg.WorktreeRootStrategy.Equals("central", StringComparison.OrdinalIgnoreCase)
|
||||
? Path.Combine(_cfg.CentralWorktreeRoot, slug, task.Id)
|
||||
string strategy;
|
||||
string? centralRoot;
|
||||
using (var settingsCtx = _dbFactory.CreateDbContext())
|
||||
{
|
||||
var settings = await new AppSettingsRepository(settingsCtx).GetAsync(ct);
|
||||
strategy = settings.WorktreeStrategy;
|
||||
centralRoot = !string.IsNullOrWhiteSpace(settings.CentralWorktreeRoot)
|
||||
? settings.CentralWorktreeRoot
|
||||
: _cfg.CentralWorktreeRoot;
|
||||
}
|
||||
|
||||
var worktreePath = strategy.Equals("central", StringComparison.OrdinalIgnoreCase)
|
||||
? Path.Combine(centralRoot ?? _cfg.CentralWorktreeRoot, slug, task.Id)
|
||||
: Path.Combine(Path.GetDirectoryName(workingDir)!, ".claudedo-worktrees", slug, task.Id);
|
||||
|
||||
worktreePath = Path.GetFullPath(worktreePath);
|
||||
@@ -48,12 +59,43 @@ public sealed class WorktreeManager
|
||||
// Ensure parent directory exists.
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(worktreePath)!);
|
||||
|
||||
// Create the worktree (this also creates the directory).
|
||||
await _git.WorktreeAddAsync(workingDir, branchName, worktreePath, baseCommit, ct);
|
||||
// Create the worktree. If a stale branch from a previous run remains
|
||||
// (e.g. after force-remove), delete it and retry once.
|
||||
try
|
||||
{
|
||||
await _git.WorktreeAddAsync(workingDir, branchName, worktreePath, baseCommit, ct);
|
||||
}
|
||||
catch (InvalidOperationException ex) when (ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogWarning("Branch {Branch} already exists; cleaning phantom worktrees and retrying", branchName);
|
||||
|
||||
// Find and forcefully remove any existing worktree registered against this branch.
|
||||
List<string> stalePaths;
|
||||
try { stalePaths = await _git.ListWorktreePathsForBranchAsync(workingDir, branchName, ct); }
|
||||
catch (Exception listEx)
|
||||
{
|
||||
_logger.LogWarning(listEx, "git worktree list failed during self-heal");
|
||||
stalePaths = new();
|
||||
}
|
||||
foreach (var stalePath in stalePaths)
|
||||
{
|
||||
try { await _git.WorktreeRemoveAsync(workingDir, stalePath, force: true, ct); }
|
||||
catch (Exception wrEx) { _logger.LogWarning(wrEx, "Failed to remove stale worktree at {Path}", stalePath); }
|
||||
}
|
||||
|
||||
try { await _git.WorktreePruneAsync(workingDir, ct); }
|
||||
catch (Exception pruneEx) { _logger.LogWarning(pruneEx, "git worktree prune failed during self-heal"); }
|
||||
try { await _git.BranchDeleteAsync(workingDir, branchName, force: true, ct); }
|
||||
catch (Exception delEx) { _logger.LogWarning(delEx, "git branch -D failed during self-heal"); }
|
||||
|
||||
await _git.WorktreeAddAsync(workingDir, branchName, worktreePath, baseCommit, ct);
|
||||
}
|
||||
|
||||
// Insert worktrees row AFTER git succeeds — if git throws, no row is created.
|
||||
using var context = _dbFactory.CreateDbContext();
|
||||
var wtRepo = new WorktreeRepository(context);
|
||||
// Drop any stale row from a prior run (force-remove may have left the DB side behind).
|
||||
await wtRepo.DeleteAsync(task.Id, ct);
|
||||
await wtRepo.AddAsync(new WorktreeEntity
|
||||
{
|
||||
TaskId = task.Id,
|
||||
|
||||
128
src/ClaudeDo.Worker/Services/WorktreeMaintenanceService.cs
Normal file
128
src/ClaudeDo.Worker/Services/WorktreeMaintenanceService.cs
Normal file
@@ -0,0 +1,128 @@
|
||||
using ClaudeDo.Data;
|
||||
using ClaudeDo.Data.Git;
|
||||
using ClaudeDo.Data.Models;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ClaudeDo.Worker.Services;
|
||||
|
||||
public sealed class WorktreeMaintenanceService
|
||||
{
|
||||
public sealed record CleanupResult(int Removed);
|
||||
public sealed record ResetResult(int Removed, int TasksAffected, bool Blocked, int RunningTasks);
|
||||
|
||||
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||
private readonly GitService _git;
|
||||
private readonly ILogger<WorktreeMaintenanceService> _logger;
|
||||
|
||||
public WorktreeMaintenanceService(
|
||||
IDbContextFactory<ClaudeDoDbContext> dbFactory,
|
||||
GitService git,
|
||||
ILogger<WorktreeMaintenanceService> logger)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_git = git;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<CleanupResult> CleanupFinishedAsync(CancellationToken ct = default)
|
||||
{
|
||||
using var context = _dbFactory.CreateDbContext();
|
||||
var rows = await (from w in context.Worktrees
|
||||
join t in context.Tasks on w.TaskId equals t.Id
|
||||
join l in context.Lists on t.ListId equals l.Id
|
||||
where w.State == WorktreeState.Merged || w.State == WorktreeState.Discarded
|
||||
select new WorktreeRow(w.TaskId, w.Path, w.BranchName, l.WorkingDir))
|
||||
.AsNoTracking()
|
||||
.ToListAsync(ct);
|
||||
|
||||
int removed = 0;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (await TryRemoveAsync(row, force: false, ct))
|
||||
removed++;
|
||||
}
|
||||
return new CleanupResult(removed);
|
||||
}
|
||||
|
||||
public async Task<ResetResult> ResetAllAsync(CancellationToken ct = default)
|
||||
{
|
||||
using var context = _dbFactory.CreateDbContext();
|
||||
var running = await context.Tasks.AsNoTracking()
|
||||
.CountAsync(t => t.Status == ClaudeDo.Data.Models.TaskStatus.Running, ct);
|
||||
if (running > 0)
|
||||
return new ResetResult(0, 0, Blocked: true, RunningTasks: running);
|
||||
|
||||
var rows = await (from w in context.Worktrees
|
||||
join t in context.Tasks on w.TaskId equals t.Id
|
||||
join l in context.Lists on t.ListId equals l.Id
|
||||
select new WorktreeRow(w.TaskId, w.Path, w.BranchName, l.WorkingDir))
|
||||
.AsNoTracking()
|
||||
.ToListAsync(ct);
|
||||
|
||||
int removed = 0;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (await TryRemoveAsync(row, force: true, ct))
|
||||
removed++;
|
||||
}
|
||||
return new ResetResult(removed, rows.Count, Blocked: false, RunningTasks: 0);
|
||||
}
|
||||
|
||||
private async Task<bool> TryRemoveAsync(WorktreeRow row, bool force, CancellationToken ct)
|
||||
{
|
||||
var repoDirExists = !string.IsNullOrWhiteSpace(row.WorkingDir) && Directory.Exists(row.WorkingDir);
|
||||
|
||||
if (repoDirExists)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _git.WorktreeRemoveAsync(row.WorkingDir!, row.Path, force, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"git worktree remove failed for {Path}; falling back to directory delete", row.Path);
|
||||
try { if (Directory.Exists(row.Path)) Directory.Delete(row.Path, recursive: true); }
|
||||
catch (Exception delEx)
|
||||
{
|
||||
_logger.LogError(delEx, "Directory.Delete fallback also failed for {Path}", row.Path);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
try { if (Directory.Exists(row.Path)) Directory.Delete(row.Path, recursive: true); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Directory.Delete failed for {Path}", row.Path);
|
||||
}
|
||||
}
|
||||
|
||||
// Branch cleanup: otherwise rerunning the task hits "branch already exists".
|
||||
// Prune first so git no longer thinks the branch is checked out by a phantom worktree.
|
||||
if (repoDirExists)
|
||||
{
|
||||
try { await _git.WorktreePruneAsync(row.WorkingDir!, ct); }
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "git worktree prune failed for {Repo}", row.WorkingDir); }
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(row.BranchName))
|
||||
{
|
||||
try
|
||||
{
|
||||
await _git.BranchDeleteAsync(row.WorkingDir!, row.BranchName, force: true, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete branch {Branch} for worktree {Path}",
|
||||
row.BranchName, row.Path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
using var context = _dbFactory.CreateDbContext();
|
||||
await context.Worktrees.Where(w => w.TaskId == row.TaskId).ExecuteDeleteAsync(ct);
|
||||
return true;
|
||||
}
|
||||
|
||||
private sealed record WorktreeRow(string TaskId, string Path, string BranchName, string? WorkingDir);
|
||||
}
|
||||
Reference in New Issue
Block a user