fix(worker): surface the real reason a Claude run failed instead of a generic exit-code message
StreamAnalyzer now reads subtype/terminal_reason/errors from the CLI's result event, and TaskRunner builds a specific ErrorMarkdown from them: max_turns names the turn budget and points at set_task_config + requeue, api_error passes through the provider's own message (which carries the reset time), and any other terminal_reason is appended to the previous generic text instead of staying invisible. Falls through to the old "exited with code N and no result" text when there's no terminal_reason at all (a real crash).
This commit is contained in:
mika kuns
@@ -0,0 +1,206 @@
|
||||
using ClaudeDo.Data;
|
||||
using ClaudeDo.Data.Git;
|
||||
using ClaudeDo.Data.Models;
|
||||
using ClaudeDo.Data.Repositories;
|
||||
using ClaudeDo.Worker.Config;
|
||||
using ClaudeDo.Worker.Hub;
|
||||
using ClaudeDo.Worker.Runner;
|
||||
using ClaudeDo.Worker.Tests.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
|
||||
using Xunit;
|
||||
|
||||
namespace ClaudeDo.Worker.Tests.Runner;
|
||||
|
||||
public class FailureDiagnosisTests
|
||||
{
|
||||
[Fact]
|
||||
public void Max_Turns_Message_Names_The_Limit_And_Requeue()
|
||||
{
|
||||
var result = new RunResult { ExitCode = 1, TerminalReason = "max_turns", TurnCount = 55 };
|
||||
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
|
||||
Assert.Contains("55", message);
|
||||
Assert.Contains("60", message);
|
||||
Assert.Contains("maxTurns", message);
|
||||
Assert.Contains("Queue", message);
|
||||
Assert.DoesNotContain(ClaudeProcess.NoResultPrefix, message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Max_Turns_Message_Without_Configured_Limit_Still_Shows_Actual_Turns()
|
||||
{
|
||||
var result = new RunResult { ExitCode = 1, TerminalReason = "max_turns", TurnCount = 30 };
|
||||
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: null);
|
||||
Assert.Contains("30", message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Api_Error_Passes_Through_Provider_Text_With_Reset_Time()
|
||||
{
|
||||
var result = new RunResult
|
||||
{
|
||||
ExitCode = 1,
|
||||
TerminalReason = "api_error",
|
||||
ResultMarkdown = "You've hit your session limit – resets 1pm (Europe/Berlin)",
|
||||
ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.",
|
||||
};
|
||||
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
|
||||
Assert.Equal("You've hit your session limit – resets 1pm (Europe/Berlin)", message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Api_Error_Falls_Back_To_Errors_List_When_No_Result_Text()
|
||||
{
|
||||
var result = new RunResult
|
||||
{
|
||||
ExitCode = 1,
|
||||
TerminalReason = "api_error",
|
||||
Errors = new[] { "rate limited" },
|
||||
ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.",
|
||||
};
|
||||
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
|
||||
Assert.Equal("rate limited", message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_Reason_Keeps_Existing_Text_And_Appends_Terminal_Reason()
|
||||
{
|
||||
var result = new RunResult
|
||||
{
|
||||
ExitCode = 1,
|
||||
TerminalReason = "some_new_reason",
|
||||
ErrorMarkdown = "stderr output here",
|
||||
};
|
||||
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
|
||||
Assert.Contains("stderr output here", message);
|
||||
Assert.Contains("some_new_reason", message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_Terminal_Reason_Behaves_As_Before()
|
||||
{
|
||||
var result = new RunResult
|
||||
{
|
||||
ExitCode = 1,
|
||||
ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.",
|
||||
};
|
||||
var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60);
|
||||
Assert.Equal($"{ClaudeProcess.NoResultPrefix} 1 and no result.", message);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class FailureDiagnosisEndToEndTests : IDisposable
|
||||
{
|
||||
private readonly DbFixture _db = new();
|
||||
private readonly WorkerConfig _cfg;
|
||||
private readonly string _tempDir;
|
||||
|
||||
public FailureDiagnosisEndToEndTests()
|
||||
{
|
||||
_tempDir = Path.Combine(Path.GetTempPath(), $"cd_faildiag_{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_tempDir);
|
||||
_cfg = new WorkerConfig { SandboxRoot = _tempDir, LogRoot = _tempDir };
|
||||
}
|
||||
|
||||
public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } }
|
||||
|
||||
private TaskRunner MakeRunner(IDbContextFactory<ClaudeDoDbContext> dbFactory, FakeClaudeProcess fake)
|
||||
{
|
||||
var state = TaskStateServiceBuilder.Build(dbFactory).State;
|
||||
var wt = new WorktreeManager(new GitService(), dbFactory, _cfg, NullLogger<WorktreeManager>.Instance);
|
||||
return new TaskRunner(fake, dbFactory, new HubBroadcaster(new CapturingHubContext()), wt,
|
||||
new ClaudeArgsBuilder(), _cfg, NullLogger<TaskRunner>.Instance, state, new TaskRunTokenRegistry(),
|
||||
new AttachmentStore(), new FakeSessionSkillSeeder());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Max_Turns_Failure_Surfaces_Specific_Message_On_Task_And_Run()
|
||||
{
|
||||
var dbFactory = _db.CreateFactory();
|
||||
using (var ctx = _db.CreateContext())
|
||||
{
|
||||
ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow });
|
||||
ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T", MaxTurns = 60,
|
||||
Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow });
|
||||
await ctx.SaveChangesAsync();
|
||||
}
|
||||
var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult
|
||||
{
|
||||
ExitCode = 1,
|
||||
TerminalReason = "max_turns",
|
||||
Errors = new[] { "Reached maximum number of turns (60)" },
|
||||
TurnCount = 60,
|
||||
}));
|
||||
var runner = MakeRunner(dbFactory, fake);
|
||||
|
||||
using (var ctx = _db.CreateContext())
|
||||
await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true);
|
||||
|
||||
using var verify = _db.CreateContext();
|
||||
var task = await new TaskRepository(verify).GetByIdAsync("t1");
|
||||
Assert.Equal(TaskStatus.Failed, task!.Status);
|
||||
Assert.Contains("Turn-Budget", task.Result);
|
||||
Assert.Contains("60", task.Result);
|
||||
Assert.DoesNotContain(ClaudeProcess.NoResultPrefix, task.Result);
|
||||
|
||||
var run = await new TaskRunRepository(verify).GetLatestByTaskIdAsync("t1");
|
||||
Assert.Contains("Turn-Budget", run!.ErrorMarkdown);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Api_Error_Failure_Surfaces_Provider_Text_With_Reset_Time()
|
||||
{
|
||||
var dbFactory = _db.CreateFactory();
|
||||
using (var ctx = _db.CreateContext())
|
||||
{
|
||||
ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow });
|
||||
ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T",
|
||||
Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow });
|
||||
await ctx.SaveChangesAsync();
|
||||
}
|
||||
var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult
|
||||
{
|
||||
ExitCode = 1,
|
||||
TerminalReason = "api_error",
|
||||
ResultMarkdown = "You've hit your session limit – resets 1pm (Europe/Berlin)",
|
||||
}));
|
||||
var runner = MakeRunner(dbFactory, fake);
|
||||
|
||||
using (var ctx = _db.CreateContext())
|
||||
await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true);
|
||||
|
||||
using var verify = _db.CreateContext();
|
||||
var task = await new TaskRepository(verify).GetByIdAsync("t1");
|
||||
Assert.Equal(TaskStatus.Failed, task!.Status);
|
||||
Assert.Contains("session limit", task.Result);
|
||||
Assert.Contains("Europe/Berlin", task.Result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Crash_Without_Result_Line_Keeps_Generic_Message()
|
||||
{
|
||||
var dbFactory = _db.CreateFactory();
|
||||
using (var ctx = _db.CreateContext())
|
||||
{
|
||||
ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow });
|
||||
ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T",
|
||||
Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow });
|
||||
await ctx.SaveChangesAsync();
|
||||
}
|
||||
var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult
|
||||
{
|
||||
ExitCode = 1,
|
||||
ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.",
|
||||
}));
|
||||
var runner = MakeRunner(dbFactory, fake);
|
||||
|
||||
using (var ctx = _db.CreateContext())
|
||||
await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true);
|
||||
|
||||
using var verify = _db.CreateContext();
|
||||
var task = await new TaskRepository(verify).GetByIdAsync("t1");
|
||||
Assert.Equal(TaskStatus.Failed, task!.Status);
|
||||
Assert.Equal($"{ClaudeProcess.NoResultPrefix} 1 and no result.", task.Result);
|
||||
}
|
||||
}
|
||||
@@ -153,6 +153,39 @@ public sealed class StreamAnalyzerTests
|
||||
Assert.Empty(analyzer.GetResult().Blocks);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Extracts_Terminal_Reason_And_Errors_For_Max_Turns()
|
||||
{
|
||||
var analyzer = new StreamAnalyzer();
|
||||
analyzer.ProcessLine("""{"type":"result","subtype":"error_max_turns","terminal_reason":"max_turns","errors":["Reached maximum number of turns (30)"],"session_id":"s1"}""");
|
||||
var result = analyzer.GetResult();
|
||||
Assert.Equal("max_turns", result.TerminalReason);
|
||||
Assert.Equal("error_max_turns", result.ResultSubtype);
|
||||
Assert.Single(result.Errors);
|
||||
Assert.Equal("Reached maximum number of turns (30)", result.Errors[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Extracts_Terminal_Reason_And_Result_Text_For_Api_Error()
|
||||
{
|
||||
var analyzer = new StreamAnalyzer();
|
||||
analyzer.ProcessLine("""{"type":"result","terminal_reason":"api_error","result":"You've hit your session limit – resets 1pm (Europe/Berlin)","session_id":"s1"}""");
|
||||
var result = analyzer.GetResult();
|
||||
Assert.Equal("api_error", result.TerminalReason);
|
||||
Assert.Contains("session limit", result.ResultMarkdown);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_Terminal_Reason_Fields_Means_Null_And_Empty()
|
||||
{
|
||||
var analyzer = new StreamAnalyzer();
|
||||
analyzer.ProcessLine("""{"type":"result","result":"done","session_id":"s1"}""");
|
||||
var result = analyzer.GetResult();
|
||||
Assert.Null(result.TerminalReason);
|
||||
Assert.Null(result.ResultSubtype);
|
||||
Assert.Empty(result.Errors);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Duplicate_Marker_In_Assistant_And_Result_Is_Collected_Once()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user