feat(worker-mcp): raise wait_for_task_change timeout, expose queue slot state
MaxTimeoutSeconds was 170s against runs that take tens of minutes, forcing a dozen full-context wait rounds per long-running batch. Raise it to 900s and raise MCP_TOOL_TIMEOUT in lockstep (ClaudeProcess + every InteractiveLaunchSpecService launch spec) to 930000ms so the client connection actually stays open that long instead of aborting first. Add get_queue_state (QueueStateMcpTools): configured vs. effective parallel-slot count (via QueueService.GetSlotCountsAsync, extracted from the former GetEffectiveMaxParallelAsync), active slots with taskId + startedAt including the run_task_now override slot, and queued tasks in pick order -- so a caller can observe queue occupancy instead of inferring it from maxParallelExecutions.
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
using System.ComponentModel;
|
||||
using ClaudeDo.Data;
|
||||
using ClaudeDo.Worker.Queue;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ModelContextProtocol.Server;
|
||||
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
|
||||
|
||||
namespace ClaudeDo.Worker.External;
|
||||
|
||||
public sealed record QueueSlotDto(string Slot, string TaskId, DateTime StartedAt);
|
||||
|
||||
public sealed record GetQueueStateResult(
|
||||
int ConfiguredSlots,
|
||||
int EffectiveSlots,
|
||||
IReadOnlyList<QueueSlotDto> ActiveSlots,
|
||||
IReadOnlyList<string> WaitingTaskIds);
|
||||
|
||||
[McpServerToolType]
|
||||
public sealed class QueueStateMcpTools
|
||||
{
|
||||
private readonly QueueService _queue;
|
||||
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||
|
||||
public QueueStateMcpTools(QueueService queue, IDbContextFactory<ClaudeDoDbContext> dbFactory)
|
||||
{
|
||||
_queue = queue;
|
||||
_dbFactory = dbFactory;
|
||||
}
|
||||
|
||||
[McpServerTool, Description(
|
||||
"Read-only snapshot of the execution queue -- observe slot occupancy instead of inferring " +
|
||||
"it from maxParallelExecutions. Result: { configuredSlots, effectiveSlots, activeSlots: " +
|
||||
"[{ slot, taskId, startedAt }], waitingTaskIds }. configuredSlots is Settings -> " +
|
||||
"MaxParallelExecutions; effectiveSlots is that value stepped down by the usage throttle " +
|
||||
"(lower when the 5h/7d usage window is filling up) -- compare the two to see whether " +
|
||||
"throttling is currently active. activeSlots lists every task presently holding an " +
|
||||
"execution slot, with slot \"queue\" for a normal queue slot or \"override\" for the single " +
|
||||
"run_task_now/continue_task slot. waitingTaskIds lists queued, unblocked, non-manual, due " +
|
||||
"tasks in the order the queue would pick them next.")]
|
||||
public async Task<GetQueueStateResult> GetQueueState(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (configured, effective) = await _queue.GetSlotCountsAsync(cancellationToken);
|
||||
|
||||
var activeSlots = _queue.GetActive()
|
||||
.Select(a => new QueueSlotDto(a.slot, a.taskId, a.startedAt))
|
||||
.ToList();
|
||||
|
||||
await using var ctx = await _dbFactory.CreateDbContextAsync(cancellationToken);
|
||||
var now = DateTime.UtcNow;
|
||||
var waitingTaskIds = await ctx.Tasks
|
||||
.Where(t => t.Status == TaskStatus.Queued
|
||||
&& t.BlockedByTaskId == null
|
||||
&& !t.IsManual
|
||||
&& (t.ScheduledFor == null || t.ScheduledFor <= now))
|
||||
.OrderBy(t => t.SortOrder).ThenBy(t => t.CreatedAt)
|
||||
.Select(t => t.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return new GetQueueStateResult(configured, effective, activeSlots, waitingTaskIds);
|
||||
}
|
||||
}
|
||||
+11
-6
@@ -12,10 +12,13 @@ public sealed record WaitForTaskChangeResult(IReadOnlyList<TaskStatusChangeDto>
|
||||
[McpServerToolType]
|
||||
public sealed class TaskWaitMcpTools
|
||||
{
|
||||
// InteractiveLaunchSpecService sets MCP_TOOL_TIMEOUT=200000ms for the list handler
|
||||
// session; this cap leaves a ~30s margin so the tool itself reports TimedOut instead
|
||||
// of racing the client's own abort.
|
||||
internal const int MaxTimeoutSeconds = 170;
|
||||
// Every ClaudeDo-owned launcher (ClaudeProcess for headless runs, InteractiveLaunchSpecService
|
||||
// for ConPTY sessions) sets MCP_TOOL_TIMEOUT=930000ms on the claude CLI process; this cap
|
||||
// leaves a ~30s margin under that so the tool itself reports TimedOut instead of racing the
|
||||
// client's own abort. A caller running claude with a different MCP_TOOL_TIMEOUT (or none --
|
||||
// the CLI default is 60s) will see its own client-side timeout fire first; this tool has no
|
||||
// way to detect or compensate for that from the server side.
|
||||
internal const int MaxTimeoutSeconds = 900;
|
||||
private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(500);
|
||||
|
||||
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||
@@ -27,9 +30,11 @@ public sealed class TaskWaitMcpTools
|
||||
|
||||
[McpServerTool, Description(
|
||||
"Blocks until at least one of the given tasks leaves Queued/Running, or until timeoutSeconds elapses " +
|
||||
"(clamped server-side to 170s). Returns immediately if any task is already outside Queued/Running " +
|
||||
"(clamped server-side to 900s). Returns immediately if any task is already outside Queued/Running " +
|
||||
"when called (an unknown id is reported as status \"NotFound\" and counts as changed). Use this instead " +
|
||||
"of polling get_task in a loop. Result: { changed: [{ taskId, status }], timedOut }.")]
|
||||
"of polling get_task in a loop. Requires the calling claude process to run with " +
|
||||
"MCP_TOOL_TIMEOUT >= 930000 (ms) for a long wait to actually be held open -- ClaudeDo's own " +
|
||||
"launchers already set this. Result: { changed: [{ taskId, status }], timedOut }.")]
|
||||
public async Task<WaitForTaskChangeResult> WaitForTaskChange(
|
||||
string[] taskIds, int timeoutSeconds = 60, CancellationToken cancellationToken = default)
|
||||
{
|
||||
|
||||
@@ -302,6 +302,7 @@ if (cfg.ExternalMcpPort > 0)
|
||||
externalBuilder.Services.AddScoped<LifecycleMcpTools>();
|
||||
externalBuilder.Services.AddScoped<AppSettingsMcpTools>();
|
||||
externalBuilder.Services.AddScoped<TaskWaitMcpTools>();
|
||||
externalBuilder.Services.AddScoped<QueueStateMcpTools>();
|
||||
externalBuilder.Services.AddSingleton(app.Services.GetRequiredService<AttachmentStore>());
|
||||
externalBuilder.Services.AddScoped<TaskAttachmentRepository>();
|
||||
externalBuilder.Services.AddScoped<AttachmentMcpTools>();
|
||||
@@ -317,6 +318,7 @@ if (cfg.ExternalMcpPort > 0)
|
||||
.WithTools<LifecycleMcpTools>()
|
||||
.WithTools<AppSettingsMcpTools>()
|
||||
.WithTools<TaskWaitMcpTools>()
|
||||
.WithTools<QueueStateMcpTools>()
|
||||
.WithTools<AttachmentMcpTools>();
|
||||
externalBuilder.WebHost.UseUrls($"http://127.0.0.1:{cfg.ExternalMcpPort}");
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ public sealed class QueueService : BackgroundService
|
||||
|
||||
await Task.WhenAny(wakeTask, timerTask);
|
||||
|
||||
var maxParallel = await GetEffectiveMaxParallelAsync(stoppingToken);
|
||||
var (_, maxParallel) = await GetSlotCountsAsync(stoppingToken);
|
||||
|
||||
var gateDecision = await _usageGate.EvaluateAsync(stoppingToken);
|
||||
await ReportUsageGateTransitionAsync(gateDecision);
|
||||
@@ -200,11 +200,13 @@ public sealed class QueueService : BackgroundService
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configured parallelism, stepped down by <see cref="UsageThrottle"/> ahead of the hard usage
|
||||
/// gate. A missing snapshot (poll hasn't landed / endpoint unreachable) fails open to the
|
||||
/// configured value — a broken usage poll must never stall the queue.
|
||||
/// Configured parallelism, and that same value stepped down by <see cref="UsageThrottle"/>
|
||||
/// ahead of the hard usage gate. A missing snapshot (poll hasn't landed / endpoint
|
||||
/// unreachable) fails open to the configured value — a broken usage poll must never stall
|
||||
/// the queue. Also called by <c>get_queue_state</c> (External MCP) to surface the throttle
|
||||
/// from outside the process.
|
||||
/// </summary>
|
||||
private async Task<int> GetEffectiveMaxParallelAsync(CancellationToken ct)
|
||||
public async Task<(int Configured, int Effective)> GetSlotCountsAsync(CancellationToken ct)
|
||||
{
|
||||
int configured;
|
||||
int softPct, hardPct, gateFivePct, gateSevenPct;
|
||||
@@ -221,14 +223,14 @@ public sealed class QueueService : BackgroundService
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read max parallel executions; defaulting to 1");
|
||||
return 1;
|
||||
return (1, 1);
|
||||
}
|
||||
|
||||
var snapshot = _usageState.Snapshot;
|
||||
if (snapshot is null || _usageState.LastError is not null)
|
||||
{
|
||||
_lastEffectiveSlots = configured;
|
||||
return configured;
|
||||
return (configured, configured);
|
||||
}
|
||||
|
||||
var effective = UsageThrottle.EffectiveSlots(
|
||||
@@ -236,7 +238,7 @@ public sealed class QueueService : BackgroundService
|
||||
softPct, hardPct, gateFivePct, gateSevenPct);
|
||||
|
||||
ReportThrottleTransition(configured, effective, snapshot);
|
||||
return effective;
|
||||
return (configured, effective);
|
||||
}
|
||||
|
||||
private void ReportThrottleTransition(int configured, int effective, UsageSnapshot snapshot)
|
||||
|
||||
@@ -40,9 +40,11 @@ public sealed class ClaudeProcess : IClaudeProcess
|
||||
psi.ArgumentList.Add(arg);
|
||||
|
||||
// Claude Code caps HTTP MCP tool calls at 60 s unless MCP_TOOL_TIMEOUT is raised.
|
||||
// The in-task AskUser tool blocks up to 3 min waiting for the user, so lift the cap
|
||||
// (with margin) or that wait would be killed early. Harmless for every other tool.
|
||||
psi.Environment["MCP_TOOL_TIMEOUT"] = "200000";
|
||||
// wait_for_task_change blocks up to TaskWaitMcpTools.MaxTimeoutSeconds (900 s) and the
|
||||
// in-task AskUser tool blocks up to 3 min waiting for the user, so lift the cap well
|
||||
// past the longer of the two (with margin) or that wait would be killed early. Harmless
|
||||
// for every other tool. Keep in sync with InteractiveLaunchSpecService's MCP_TOOL_TIMEOUT.
|
||||
psi.Environment["MCP_TOOL_TIMEOUT"] = "930000";
|
||||
|
||||
using var process = new Process { StartInfo = psi };
|
||||
process.Start();
|
||||
|
||||
@@ -14,7 +14,7 @@ namespace ClaudeDo.Worker.Runner;
|
||||
// session in an existing task's worktree -- the SAME worktree prep as an autonomous run:
|
||||
// session-skills seeded onto disk (reuses ISessionSkillSeeder + TaskRunner.UnionSkillNames,
|
||||
// exactly like TaskRunner.RunAsync/ContinueAsync) and the same run environment variables
|
||||
// (reuses ClaudeProcess's MCP_TOOL_TIMEOUT). Exe/Args reuse WindowsTerminalLauncher's
|
||||
// (matches ClaudeProcess's MCP_TOOL_TIMEOUT). Exe/Args reuse WindowsTerminalLauncher's
|
||||
// --resume argument construction. Guards: no running/queued task, and (once a worktree
|
||||
// exists) it must be live on disk -- but a never-run task (no persisted SessionId) is not
|
||||
// an error here, it's a fresh-start spec.
|
||||
@@ -107,11 +107,11 @@ public sealed class InteractiveLaunchSpecService : IInteractiveLaunchSpecService
|
||||
: await BuildFreshTaskArgsAsync(task, effort, ct);
|
||||
|
||||
// Same run environment variable ClaudeProcess sets for every headless run: the
|
||||
// AskUser MCP tool call caps at 60s unless raised, and lifting it is harmless for
|
||||
// every other tool.
|
||||
// AskUser MCP tool call and wait_for_task_change cap at 60s unless raised, and lifting
|
||||
// it is harmless for every other tool. Keep in sync with ClaudeProcess's MCP_TOOL_TIMEOUT.
|
||||
var env = new Dictionary<string, string>
|
||||
{
|
||||
["MCP_TOOL_TIMEOUT"] = "200000",
|
||||
["MCP_TOOL_TIMEOUT"] = "930000",
|
||||
};
|
||||
|
||||
return new LaunchSpec(worktree.Path, resolvedClaude, args, env);
|
||||
@@ -129,7 +129,7 @@ public sealed class InteractiveLaunchSpecService : IInteractiveLaunchSpecService
|
||||
{
|
||||
["MAX_THINKING_TOKENS"] = "20000",
|
||||
["CLAUDEDO_PLANNING_TOKEN"] = ctx.Token,
|
||||
["MCP_TOOL_TIMEOUT"] = "200000",
|
||||
["MCP_TOOL_TIMEOUT"] = "930000",
|
||||
};
|
||||
|
||||
return new LaunchSpec(
|
||||
@@ -147,7 +147,7 @@ public sealed class InteractiveLaunchSpecService : IInteractiveLaunchSpecService
|
||||
var env = new Dictionary<string, string>
|
||||
{
|
||||
["CLAUDEDO_PLANNING_TOKEN"] = ctx.Token,
|
||||
["MCP_TOOL_TIMEOUT"] = "200000",
|
||||
["MCP_TOOL_TIMEOUT"] = "930000",
|
||||
};
|
||||
|
||||
return new LaunchSpec(
|
||||
@@ -170,7 +170,7 @@ public sealed class InteractiveLaunchSpecService : IInteractiveLaunchSpecService
|
||||
|
||||
var env = new Dictionary<string, string>
|
||||
{
|
||||
["MCP_TOOL_TIMEOUT"] = "200000",
|
||||
["MCP_TOOL_TIMEOUT"] = "930000",
|
||||
};
|
||||
|
||||
// No task and no list here — the global default model's preset decides the effort.
|
||||
@@ -247,7 +247,7 @@ public sealed class InteractiveLaunchSpecService : IInteractiveLaunchSpecService
|
||||
|
||||
var env = new Dictionary<string, string>
|
||||
{
|
||||
["MCP_TOOL_TIMEOUT"] = "200000",
|
||||
["MCP_TOOL_TIMEOUT"] = "930000",
|
||||
};
|
||||
|
||||
return new LaunchSpec(repoDir, resolvedClaude, args, env);
|
||||
|
||||
Reference in New Issue
Block a user