fix(worker): surface the real reason a Claude run failed instead of a generic exit-code message

StreamAnalyzer now reads subtype/terminal_reason/errors from the CLI's result
event, and TaskRunner builds a specific ErrorMarkdown from them: max_turns names
the turn budget and points at set_task_config + requeue, api_error passes
through the provider's own message (which carries the reset time), and any
other terminal_reason is appended to the previous generic text instead of
staying invisible. Falls through to the old "exited with code N and no result"
text when there's no terminal_reason at all (a real crash).
This commit is contained in:
mika kuns
2026-08-05 15:55:09 +02:00
parent 83ea429b8a
commit 2ebdadff08
6 changed files with 298 additions and 1 deletions
@@ -105,6 +105,9 @@ public sealed class ClaudeProcess : IClaudeProcess
TokensIn = streamResult.TokensIn,
TokensOut = streamResult.TokensOut,
Blocks = streamResult.Blocks,
ResultSubtype = streamResult.ResultSubtype,
TerminalReason = streamResult.TerminalReason,
Errors = streamResult.Errors,
};
}
@@ -115,12 +118,19 @@ public sealed class ClaudeProcess : IClaudeProcess
return new RunResult
{
ExitCode = exitCode,
// Kept even on failure: a terminal reason like api_error often carries the
// provider's own message (e.g. session-limit + reset time) in this field,
// with nothing useful on stderr.
ResultMarkdown = streamResult.ResultMarkdown,
ErrorMarkdown = error,
SessionId = streamResult.SessionId,
TurnCount = streamResult.TurnCount,
TokensIn = streamResult.TokensIn,
TokensOut = streamResult.TokensOut,
Blocks = streamResult.Blocks,
ResultSubtype = streamResult.ResultSubtype,
TerminalReason = streamResult.TerminalReason,
Errors = streamResult.Errors,
};
}
}
+4 -1
View File
@@ -1,6 +1,6 @@
namespace ClaudeDo.Worker.Runner;
public sealed class RunResult
public sealed record RunResult
{
public required int ExitCode { get; init; }
public string? ResultMarkdown { get; init; }
@@ -11,6 +11,9 @@ public sealed class RunResult
public int TokensIn { get; init; }
public int TokensOut { get; init; }
public IReadOnlyList<string> Blocks { get; init; } = Array.Empty<string>();
public string? ResultSubtype { get; init; }
public string? TerminalReason { get; init; }
public IReadOnlyList<string> Errors { get; init; } = Array.Empty<string>();
public bool IsSuccess => ExitCode == 0 && ResultMarkdown is not null;
}
@@ -13,6 +13,9 @@ public sealed class StreamResult
public int TokensOut { get; set; }
public int ApiRetryCount { get; set; }
public IReadOnlyList<string> Blocks { get; set; } = Array.Empty<string>();
public string? ResultSubtype { get; set; }
public string? TerminalReason { get; set; }
public IReadOnlyList<string> Errors { get; set; } = Array.Empty<string>();
}
public sealed class StreamAnalyzer
@@ -25,6 +28,9 @@ public sealed class StreamAnalyzer
private int _tokensOut;
private int _apiRetryCount;
private readonly List<string> _blocks = new();
private string? _resultSubtype;
private string? _terminalReason;
private readonly List<string> _errors = new();
private const string BlockedPrefix = "CLAUDEDO_BLOCKED:";
public void ProcessLine(string ndjsonLine)
@@ -48,6 +54,17 @@ public sealed class StreamAnalyzer
_structuredOutputJson = structuredProp.ToString();
if (root.TryGetProperty("session_id", out var sessionProp))
_sessionId = sessionProp.GetString();
if (root.TryGetProperty("subtype", out var resultSubtypeProp))
_resultSubtype = resultSubtypeProp.GetString();
if (root.TryGetProperty("terminal_reason", out var terminalReasonProp))
_terminalReason = terminalReasonProp.GetString();
if (root.TryGetProperty("errors", out var errorsProp) && errorsProp.ValueKind == JsonValueKind.Array)
{
foreach (var errorItem in errorsProp.EnumerateArray())
if (errorItem.ValueKind == JsonValueKind.String && errorItem.GetString() is { } errorText
&& !string.IsNullOrEmpty(errorText))
_errors.Add(errorText);
}
// Authoritative token totals live on the result event.
if (root.TryGetProperty("usage", out var resultUsage))
{
@@ -87,6 +104,9 @@ public sealed class StreamAnalyzer
TokensOut = _tokensOut,
ApiRetryCount = _apiRetryCount,
Blocks = _blocks.Distinct().ToList(),
ResultSubtype = _resultSubtype,
TerminalReason = _terminalReason,
Errors = _errors,
};
private string? FallbackResult()
+25
View File
@@ -352,6 +352,9 @@ public sealed class TaskRunner
},
ct);
if (!result.IsSuccess)
result = result with { ErrorMarkdown = BuildFailureMarkdown(result, config.MaxTurns) };
// Update the run record with results. Use CancellationToken.None:
// this is a terminal write that must always complete, even if the
// caller's token is already cancelled.
@@ -597,6 +600,28 @@ public sealed class TaskRunner
return string.Join("\n\n", trimmed);
}
public static string? BuildFailureMarkdown(RunResult result, int? configuredMaxTurns)
{
switch (result.TerminalReason)
{
case "max_turns":
var ofConfigured = configuredMaxTurns is int m ? $" von {m}" : "";
return $"Turn-Budget erschöpft ({result.TurnCount}{ofConfigured} Turns). "
+ "Erhöhe `maxTurns` am Task (`set_task_config`) und stelle den Task erneut in die Queue "
+ " der Worktree bleibt erhalten.";
case "api_error":
if (!string.IsNullOrWhiteSpace(result.ResultMarkdown)) return result.ResultMarkdown;
if (result.Errors.Count > 0) return string.Join(" ", result.Errors);
return result.ErrorMarkdown;
default:
return string.IsNullOrWhiteSpace(result.TerminalReason)
? result.ErrorMarkdown
: $"{result.ErrorMarkdown} (terminal_reason: {result.TerminalReason})";
}
}
public static string BuildRetryPrompt(string? capturedError)
{
var basePrompt = PromptFiles.ReadOrDefault(PromptKind.Retry);
@@ -0,0 +1,206 @@
using ClaudeDo.Data;
using ClaudeDo.Data.Git;
using ClaudeDo.Data.Models;
using ClaudeDo.Data.Repositories;
using ClaudeDo.Worker.Config;
using ClaudeDo.Worker.Hub;
using ClaudeDo.Worker.Runner;
using ClaudeDo.Worker.Tests.Infrastructure;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
using Xunit;
namespace ClaudeDo.Worker.Tests.Runner;
public class FailureDiagnosisTests
{
[Fact]
public void Max_Turns_Message_Names_The_Limit_And_Requeue()
{
var result = new RunResult { ExitCode = 1, TerminalReason = "max_turns", TurnCount = 55 };
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
Assert.Contains("55", message);
Assert.Contains("60", message);
Assert.Contains("maxTurns", message);
Assert.Contains("Queue", message);
Assert.DoesNotContain(ClaudeProcess.NoResultPrefix, message);
}
[Fact]
public void Max_Turns_Message_Without_Configured_Limit_Still_Shows_Actual_Turns()
{
var result = new RunResult { ExitCode = 1, TerminalReason = "max_turns", TurnCount = 30 };
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: null);
Assert.Contains("30", message);
}
[Fact]
public void Api_Error_Passes_Through_Provider_Text_With_Reset_Time()
{
var result = new RunResult
{
ExitCode = 1,
TerminalReason = "api_error",
ResultMarkdown = "You've hit your session limit resets 1pm (Europe/Berlin)",
ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.",
};
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
Assert.Equal("You've hit your session limit resets 1pm (Europe/Berlin)", message);
}
[Fact]
public void Api_Error_Falls_Back_To_Errors_List_When_No_Result_Text()
{
var result = new RunResult
{
ExitCode = 1,
TerminalReason = "api_error",
Errors = new[] { "rate limited" },
ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.",
};
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
Assert.Equal("rate limited", message);
}
[Fact]
public void Unknown_Reason_Keeps_Existing_Text_And_Appends_Terminal_Reason()
{
var result = new RunResult
{
ExitCode = 1,
TerminalReason = "some_new_reason",
ErrorMarkdown = "stderr output here",
};
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
Assert.Contains("stderr output here", message);
Assert.Contains("some_new_reason", message);
}
[Fact]
public void No_Terminal_Reason_Behaves_As_Before()
{
var result = new RunResult
{
ExitCode = 1,
ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.",
};
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
Assert.Equal($"{ClaudeProcess.NoResultPrefix} 1 and no result.", message);
}
}
public sealed class FailureDiagnosisEndToEndTests : IDisposable
{
private readonly DbFixture _db = new();
private readonly WorkerConfig _cfg;
private readonly string _tempDir;
public FailureDiagnosisEndToEndTests()
{
_tempDir = Path.Combine(Path.GetTempPath(), $"cd_faildiag_{Guid.NewGuid():N}");
Directory.CreateDirectory(_tempDir);
_cfg = new WorkerConfig { SandboxRoot = _tempDir, LogRoot = _tempDir };
}
public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } }
private TaskRunner MakeRunner(IDbContextFactory<ClaudeDoDbContext> dbFactory, FakeClaudeProcess fake)
{
var state = TaskStateServiceBuilder.Build(dbFactory).State;
var wt = new WorktreeManager(new GitService(), dbFactory, _cfg, NullLogger<WorktreeManager>.Instance);
return new TaskRunner(fake, dbFactory, new HubBroadcaster(new CapturingHubContext()), wt,
new ClaudeArgsBuilder(), _cfg, NullLogger<TaskRunner>.Instance, state, new TaskRunTokenRegistry(),
new AttachmentStore(), new FakeSessionSkillSeeder());
}
[Fact]
public async Task Max_Turns_Failure_Surfaces_Specific_Message_On_Task_And_Run()
{
var dbFactory = _db.CreateFactory();
using (var ctx = _db.CreateContext())
{
ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow });
ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T", MaxTurns = 60,
Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow });
await ctx.SaveChangesAsync();
}
var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult
{
ExitCode = 1,
TerminalReason = "max_turns",
Errors = new[] { "Reached maximum number of turns (60)" },
TurnCount = 60,
}));
var runner = MakeRunner(dbFactory, fake);
using (var ctx = _db.CreateContext())
await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true);
using var verify = _db.CreateContext();
var task = await new TaskRepository(verify).GetByIdAsync("t1");
Assert.Equal(TaskStatus.Failed, task!.Status);
Assert.Contains("Turn-Budget", task.Result);
Assert.Contains("60", task.Result);
Assert.DoesNotContain(ClaudeProcess.NoResultPrefix, task.Result);
var run = await new TaskRunRepository(verify).GetLatestByTaskIdAsync("t1");
Assert.Contains("Turn-Budget", run!.ErrorMarkdown);
}
[Fact]
public async Task Api_Error_Failure_Surfaces_Provider_Text_With_Reset_Time()
{
var dbFactory = _db.CreateFactory();
using (var ctx = _db.CreateContext())
{
ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow });
ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T",
Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow });
await ctx.SaveChangesAsync();
}
var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult
{
ExitCode = 1,
TerminalReason = "api_error",
ResultMarkdown = "You've hit your session limit resets 1pm (Europe/Berlin)",
}));
var runner = MakeRunner(dbFactory, fake);
using (var ctx = _db.CreateContext())
await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true);
using var verify = _db.CreateContext();
var task = await new TaskRepository(verify).GetByIdAsync("t1");
Assert.Equal(TaskStatus.Failed, task!.Status);
Assert.Contains("session limit", task.Result);
Assert.Contains("Europe/Berlin", task.Result);
}
[Fact]
public async Task Crash_Without_Result_Line_Keeps_Generic_Message()
{
var dbFactory = _db.CreateFactory();
using (var ctx = _db.CreateContext())
{
ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow });
ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T",
Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow });
await ctx.SaveChangesAsync();
}
var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult
{
ExitCode = 1,
ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.",
}));
var runner = MakeRunner(dbFactory, fake);
using (var ctx = _db.CreateContext())
await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true);
using var verify = _db.CreateContext();
var task = await new TaskRepository(verify).GetByIdAsync("t1");
Assert.Equal(TaskStatus.Failed, task!.Status);
Assert.Equal($"{ClaudeProcess.NoResultPrefix} 1 and no result.", task.Result);
}
}
@@ -153,6 +153,39 @@ public sealed class StreamAnalyzerTests
Assert.Empty(analyzer.GetResult().Blocks);
}
[Fact]
public void Extracts_Terminal_Reason_And_Errors_For_Max_Turns()
{
var analyzer = new StreamAnalyzer();
analyzer.ProcessLine("""{"type":"result","subtype":"error_max_turns","terminal_reason":"max_turns","errors":["Reached maximum number of turns (30)"],"session_id":"s1"}""");
var result = analyzer.GetResult();
Assert.Equal("max_turns", result.TerminalReason);
Assert.Equal("error_max_turns", result.ResultSubtype);
Assert.Single(result.Errors);
Assert.Equal("Reached maximum number of turns (30)", result.Errors[0]);
}
[Fact]
public void Extracts_Terminal_Reason_And_Result_Text_For_Api_Error()
{
var analyzer = new StreamAnalyzer();
analyzer.ProcessLine("""{"type":"result","terminal_reason":"api_error","result":"You've hit your session limit resets 1pm (Europe/Berlin)","session_id":"s1"}""");
var result = analyzer.GetResult();
Assert.Equal("api_error", result.TerminalReason);
Assert.Contains("session limit", result.ResultMarkdown);
}
[Fact]
public void No_Terminal_Reason_Fields_Means_Null_And_Empty()
{
var analyzer = new StreamAnalyzer();
analyzer.ProcessLine("""{"type":"result","result":"done","session_id":"s1"}""");
var result = analyzer.GetResult();
Assert.Null(result.TerminalReason);
Assert.Null(result.ResultSubtype);
Assert.Empty(result.Errors);
}
[Fact]
public void Duplicate_Marker_In_Assistant_And_Result_Is_Collected_Once()
{