diff --git a/src/ClaudeDo.Worker/Runner/ClaudeProcess.cs b/src/ClaudeDo.Worker/Runner/ClaudeProcess.cs index 76bf9798..d94b10ac 100644 --- a/src/ClaudeDo.Worker/Runner/ClaudeProcess.cs +++ b/src/ClaudeDo.Worker/Runner/ClaudeProcess.cs @@ -105,6 +105,9 @@ public sealed class ClaudeProcess : IClaudeProcess TokensIn = streamResult.TokensIn, TokensOut = streamResult.TokensOut, Blocks = streamResult.Blocks, + ResultSubtype = streamResult.ResultSubtype, + TerminalReason = streamResult.TerminalReason, + Errors = streamResult.Errors, }; } @@ -115,12 +118,19 @@ public sealed class ClaudeProcess : IClaudeProcess return new RunResult { ExitCode = exitCode, + // Kept even on failure: a terminal reason like api_error often carries the + // provider's own message (e.g. session-limit + reset time) in this field, + // with nothing useful on stderr. + ResultMarkdown = streamResult.ResultMarkdown, ErrorMarkdown = error, SessionId = streamResult.SessionId, TurnCount = streamResult.TurnCount, TokensIn = streamResult.TokensIn, TokensOut = streamResult.TokensOut, Blocks = streamResult.Blocks, + ResultSubtype = streamResult.ResultSubtype, + TerminalReason = streamResult.TerminalReason, + Errors = streamResult.Errors, }; } } diff --git a/src/ClaudeDo.Worker/Runner/RunResult.cs b/src/ClaudeDo.Worker/Runner/RunResult.cs index cc61fa72..3b26e527 100644 --- a/src/ClaudeDo.Worker/Runner/RunResult.cs +++ b/src/ClaudeDo.Worker/Runner/RunResult.cs @@ -1,6 +1,6 @@ namespace ClaudeDo.Worker.Runner; -public sealed class RunResult +public sealed record RunResult { public required int ExitCode { get; init; } public string? ResultMarkdown { get; init; } @@ -11,6 +11,9 @@ public sealed class RunResult public int TokensIn { get; init; } public int TokensOut { get; init; } public IReadOnlyList Blocks { get; init; } = Array.Empty(); + public string? ResultSubtype { get; init; } + public string? TerminalReason { get; init; } + public IReadOnlyList Errors { get; init; } = Array.Empty(); public bool IsSuccess => ExitCode == 0 && ResultMarkdown is not null; } diff --git a/src/ClaudeDo.Worker/Runner/StreamAnalyzer.cs b/src/ClaudeDo.Worker/Runner/StreamAnalyzer.cs index c96f4dcd..4ee65552 100644 --- a/src/ClaudeDo.Worker/Runner/StreamAnalyzer.cs +++ b/src/ClaudeDo.Worker/Runner/StreamAnalyzer.cs @@ -13,6 +13,9 @@ public sealed class StreamResult public int TokensOut { get; set; } public int ApiRetryCount { get; set; } public IReadOnlyList Blocks { get; set; } = Array.Empty(); + public string? ResultSubtype { get; set; } + public string? TerminalReason { get; set; } + public IReadOnlyList Errors { get; set; } = Array.Empty(); } public sealed class StreamAnalyzer @@ -25,6 +28,9 @@ public sealed class StreamAnalyzer private int _tokensOut; private int _apiRetryCount; private readonly List _blocks = new(); + private string? _resultSubtype; + private string? _terminalReason; + private readonly List _errors = new(); private const string BlockedPrefix = "CLAUDEDO_BLOCKED:"; public void ProcessLine(string ndjsonLine) @@ -48,6 +54,17 @@ public sealed class StreamAnalyzer _structuredOutputJson = structuredProp.ToString(); if (root.TryGetProperty("session_id", out var sessionProp)) _sessionId = sessionProp.GetString(); + if (root.TryGetProperty("subtype", out var resultSubtypeProp)) + _resultSubtype = resultSubtypeProp.GetString(); + if (root.TryGetProperty("terminal_reason", out var terminalReasonProp)) + _terminalReason = terminalReasonProp.GetString(); + if (root.TryGetProperty("errors", out var errorsProp) && errorsProp.ValueKind == JsonValueKind.Array) + { + foreach (var errorItem in errorsProp.EnumerateArray()) + if (errorItem.ValueKind == JsonValueKind.String && errorItem.GetString() is { } errorText + && !string.IsNullOrEmpty(errorText)) + _errors.Add(errorText); + } // Authoritative token totals live on the result event. if (root.TryGetProperty("usage", out var resultUsage)) { @@ -87,6 +104,9 @@ public sealed class StreamAnalyzer TokensOut = _tokensOut, ApiRetryCount = _apiRetryCount, Blocks = _blocks.Distinct().ToList(), + ResultSubtype = _resultSubtype, + TerminalReason = _terminalReason, + Errors = _errors, }; private string? FallbackResult() diff --git a/src/ClaudeDo.Worker/Runner/TaskRunner.cs b/src/ClaudeDo.Worker/Runner/TaskRunner.cs index 820b2457..61eaac72 100644 --- a/src/ClaudeDo.Worker/Runner/TaskRunner.cs +++ b/src/ClaudeDo.Worker/Runner/TaskRunner.cs @@ -352,6 +352,9 @@ public sealed class TaskRunner }, ct); + if (!result.IsSuccess) + result = result with { ErrorMarkdown = BuildFailureMarkdown(result, config.MaxTurns) }; + // Update the run record with results. Use CancellationToken.None: // this is a terminal write that must always complete, even if the // caller's token is already cancelled. @@ -597,6 +600,28 @@ public sealed class TaskRunner return string.Join("\n\n", trimmed); } + public static string? BuildFailureMarkdown(RunResult result, int? configuredMaxTurns) + { + switch (result.TerminalReason) + { + case "max_turns": + var ofConfigured = configuredMaxTurns is int m ? $" von {m}" : ""; + return $"Turn-Budget erschöpft ({result.TurnCount}{ofConfigured} Turns). " + + "Erhöhe `maxTurns` am Task (`set_task_config`) und stelle den Task erneut in die Queue " + + "– der Worktree bleibt erhalten."; + + case "api_error": + if (!string.IsNullOrWhiteSpace(result.ResultMarkdown)) return result.ResultMarkdown; + if (result.Errors.Count > 0) return string.Join(" ", result.Errors); + return result.ErrorMarkdown; + + default: + return string.IsNullOrWhiteSpace(result.TerminalReason) + ? result.ErrorMarkdown + : $"{result.ErrorMarkdown} (terminal_reason: {result.TerminalReason})"; + } + } + public static string BuildRetryPrompt(string? capturedError) { var basePrompt = PromptFiles.ReadOrDefault(PromptKind.Retry); diff --git a/tests/ClaudeDo.Worker.Tests/Runner/FailureDiagnosisTests.cs b/tests/ClaudeDo.Worker.Tests/Runner/FailureDiagnosisTests.cs new file mode 100644 index 00000000..a647284a --- /dev/null +++ b/tests/ClaudeDo.Worker.Tests/Runner/FailureDiagnosisTests.cs @@ -0,0 +1,206 @@ +using ClaudeDo.Data; +using ClaudeDo.Data.Git; +using ClaudeDo.Data.Models; +using ClaudeDo.Data.Repositories; +using ClaudeDo.Worker.Config; +using ClaudeDo.Worker.Hub; +using ClaudeDo.Worker.Runner; +using ClaudeDo.Worker.Tests.Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using TaskStatus = ClaudeDo.Data.Models.TaskStatus; +using Xunit; + +namespace ClaudeDo.Worker.Tests.Runner; + +public class FailureDiagnosisTests +{ + [Fact] + public void Max_Turns_Message_Names_The_Limit_And_Requeue() + { + var result = new RunResult { ExitCode = 1, TerminalReason = "max_turns", TurnCount = 55 }; + var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60); + Assert.Contains("55", message); + Assert.Contains("60", message); + Assert.Contains("maxTurns", message); + Assert.Contains("Queue", message); + Assert.DoesNotContain(ClaudeProcess.NoResultPrefix, message); + } + + [Fact] + public void Max_Turns_Message_Without_Configured_Limit_Still_Shows_Actual_Turns() + { + var result = new RunResult { ExitCode = 1, TerminalReason = "max_turns", TurnCount = 30 }; + var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: null); + Assert.Contains("30", message); + } + + [Fact] + public void Api_Error_Passes_Through_Provider_Text_With_Reset_Time() + { + var result = new RunResult + { + ExitCode = 1, + TerminalReason = "api_error", + ResultMarkdown = "You've hit your session limit – resets 1pm (Europe/Berlin)", + ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.", + }; + var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60); + Assert.Equal("You've hit your session limit – resets 1pm (Europe/Berlin)", message); + } + + [Fact] + public void Api_Error_Falls_Back_To_Errors_List_When_No_Result_Text() + { + var result = new RunResult + { + ExitCode = 1, + TerminalReason = "api_error", + Errors = new[] { "rate limited" }, + ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.", + }; + var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60); + Assert.Equal("rate limited", message); + } + + [Fact] + public void Unknown_Reason_Keeps_Existing_Text_And_Appends_Terminal_Reason() + { + var result = new RunResult + { + ExitCode = 1, + TerminalReason = "some_new_reason", + ErrorMarkdown = "stderr output here", + }; + var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60); + Assert.Contains("stderr output here", message); + Assert.Contains("some_new_reason", message); + } + + [Fact] + public void No_Terminal_Reason_Behaves_As_Before() + { + var result = new RunResult + { + ExitCode = 1, + ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.", + }; + var message = TaskRunner.BuildFailureMarkdown(result, configuredMaxTurns: 60); + Assert.Equal($"{ClaudeProcess.NoResultPrefix} 1 and no result.", message); + } +} + +public sealed class FailureDiagnosisEndToEndTests : IDisposable +{ + private readonly DbFixture _db = new(); + private readonly WorkerConfig _cfg; + private readonly string _tempDir; + + public FailureDiagnosisEndToEndTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"cd_faildiag_{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + _cfg = new WorkerConfig { SandboxRoot = _tempDir, LogRoot = _tempDir }; + } + + public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } } + + private TaskRunner MakeRunner(IDbContextFactory dbFactory, FakeClaudeProcess fake) + { + var state = TaskStateServiceBuilder.Build(dbFactory).State; + var wt = new WorktreeManager(new GitService(), dbFactory, _cfg, NullLogger.Instance); + return new TaskRunner(fake, dbFactory, new HubBroadcaster(new CapturingHubContext()), wt, + new ClaudeArgsBuilder(), _cfg, NullLogger.Instance, state, new TaskRunTokenRegistry(), + new AttachmentStore(), new FakeSessionSkillSeeder()); + } + + [Fact] + public async Task Max_Turns_Failure_Surfaces_Specific_Message_On_Task_And_Run() + { + var dbFactory = _db.CreateFactory(); + using (var ctx = _db.CreateContext()) + { + ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow }); + ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T", MaxTurns = 60, + Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow }); + await ctx.SaveChangesAsync(); + } + var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult + { + ExitCode = 1, + TerminalReason = "max_turns", + Errors = new[] { "Reached maximum number of turns (60)" }, + TurnCount = 60, + })); + var runner = MakeRunner(dbFactory, fake); + + using (var ctx = _db.CreateContext()) + await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true); + + using var verify = _db.CreateContext(); + var task = await new TaskRepository(verify).GetByIdAsync("t1"); + Assert.Equal(TaskStatus.Failed, task!.Status); + Assert.Contains("Turn-Budget", task.Result); + Assert.Contains("60", task.Result); + Assert.DoesNotContain(ClaudeProcess.NoResultPrefix, task.Result); + + var run = await new TaskRunRepository(verify).GetLatestByTaskIdAsync("t1"); + Assert.Contains("Turn-Budget", run!.ErrorMarkdown); + } + + [Fact] + public async Task Api_Error_Failure_Surfaces_Provider_Text_With_Reset_Time() + { + var dbFactory = _db.CreateFactory(); + using (var ctx = _db.CreateContext()) + { + ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow }); + ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T", + Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow }); + await ctx.SaveChangesAsync(); + } + var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult + { + ExitCode = 1, + TerminalReason = "api_error", + ResultMarkdown = "You've hit your session limit – resets 1pm (Europe/Berlin)", + })); + var runner = MakeRunner(dbFactory, fake); + + using (var ctx = _db.CreateContext()) + await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true); + + using var verify = _db.CreateContext(); + var task = await new TaskRepository(verify).GetByIdAsync("t1"); + Assert.Equal(TaskStatus.Failed, task!.Status); + Assert.Contains("session limit", task.Result); + Assert.Contains("Europe/Berlin", task.Result); + } + + [Fact] + public async Task Crash_Without_Result_Line_Keeps_Generic_Message() + { + var dbFactory = _db.CreateFactory(); + using (var ctx = _db.CreateContext()) + { + ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow }); + ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T", + Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow }); + await ctx.SaveChangesAsync(); + } + var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult + { + ExitCode = 1, + ErrorMarkdown = $"{ClaudeProcess.NoResultPrefix} 1 and no result.", + })); + var runner = MakeRunner(dbFactory, fake); + + using (var ctx = _db.CreateContext()) + await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true); + + using var verify = _db.CreateContext(); + var task = await new TaskRepository(verify).GetByIdAsync("t1"); + Assert.Equal(TaskStatus.Failed, task!.Status); + Assert.Equal($"{ClaudeProcess.NoResultPrefix} 1 and no result.", task.Result); + } +} diff --git a/tests/ClaudeDo.Worker.Tests/Runner/StreamAnalyzerTests.cs b/tests/ClaudeDo.Worker.Tests/Runner/StreamAnalyzerTests.cs index 27c84062..4e164a6e 100644 --- a/tests/ClaudeDo.Worker.Tests/Runner/StreamAnalyzerTests.cs +++ b/tests/ClaudeDo.Worker.Tests/Runner/StreamAnalyzerTests.cs @@ -153,6 +153,39 @@ public sealed class StreamAnalyzerTests Assert.Empty(analyzer.GetResult().Blocks); } + [Fact] + public void Extracts_Terminal_Reason_And_Errors_For_Max_Turns() + { + var analyzer = new StreamAnalyzer(); + analyzer.ProcessLine("""{"type":"result","subtype":"error_max_turns","terminal_reason":"max_turns","errors":["Reached maximum number of turns (30)"],"session_id":"s1"}"""); + var result = analyzer.GetResult(); + Assert.Equal("max_turns", result.TerminalReason); + Assert.Equal("error_max_turns", result.ResultSubtype); + Assert.Single(result.Errors); + Assert.Equal("Reached maximum number of turns (30)", result.Errors[0]); + } + + [Fact] + public void Extracts_Terminal_Reason_And_Result_Text_For_Api_Error() + { + var analyzer = new StreamAnalyzer(); + analyzer.ProcessLine("""{"type":"result","terminal_reason":"api_error","result":"You've hit your session limit – resets 1pm (Europe/Berlin)","session_id":"s1"}"""); + var result = analyzer.GetResult(); + Assert.Equal("api_error", result.TerminalReason); + Assert.Contains("session limit", result.ResultMarkdown); + } + + [Fact] + public void No_Terminal_Reason_Fields_Means_Null_And_Empty() + { + var analyzer = new StreamAnalyzer(); + analyzer.ProcessLine("""{"type":"result","result":"done","session_id":"s1"}"""); + var result = analyzer.GetResult(); + Assert.Null(result.TerminalReason); + Assert.Null(result.ResultSubtype); + Assert.Empty(result.Errors); + } + [Fact] public void Duplicate_Marker_In_Assistant_And_Result_Is_Collected_Once() {