feat(worker): Toggle "Continue on session limit reset" für Usage-Limit-Abbrüche

Klassifiziert einen echten Usage-Limit-Abbruch als eigene FailureReason
"usage_limit" (TaskRunner.ClassifyFailureReason: nur bei terminal_reason
"api_error" plus einem Limit-Muster im gerenderten Fehlertext, nicht an
Status==Failed allein). Neuer Toggle AutoContinueOnUsageLimit (app_settings,
Default aus) unter Settings → General → "Usage limit stop":

- UsageLimitAutoContinueCoordinator feuert pro Task genau einmal ContinueTask
  über OverrideSlotService, sobald das 5h-Fenster (UsageState.Snapshot.FiveHour
  .ResetsAt) tatsächlich zurückgesetzt ist; ein persistenter Marker
  (TaskEntity.UsageLimitAutoContinuedAt) verhindert einen zweiten Anlauf bei
  einem erneuten Limit-Treffer.
- QueueService schedult zusätzlich einen exakten Wake-Timer auf den
  Reset-Zeitpunkt, statt nur auf den 30s-Backstop zu warten.
- Fail-open durchgängig: kein Snapshot/keine Reset-Zeit → kein Timer, kein
  Continue, kein Throw. Toggle aus ändert das heutige Verhalten nicht.

Migration AddUsageLimitAutoContinue fügt beide Spalten hinzu; die von
`dotnet ef migrations add` mitgescaffoldete leere UpdateData auf app_settings
(columns/values: []) erzeugte ungültiges SQL ("near WHERE") und wurde entfernt
— TaskNumberMigrationTests deckte das über den vollen Migrate()-Pfad auf.
This commit is contained in:
mika kuns
2026-08-21 18:43:02 +02:00
parent 290dd1b614
commit 07dd75700d
30 changed files with 1505 additions and 19 deletions
@@ -67,6 +67,9 @@ public class AppSettingsEntityConfiguration : IEntityTypeConfiguration<AppSettin
builder.Property(s => s.UsageThrottleSevenDayHardPct) builder.Property(s => s.UsageThrottleSevenDayHardPct)
.HasColumnName("usage_throttle_seven_day_hard_pct").IsRequired().HasDefaultValue(65); .HasColumnName("usage_throttle_seven_day_hard_pct").IsRequired().HasDefaultValue(65);
builder.Property(s => s.AutoContinueOnUsageLimit)
.HasColumnName("auto_continue_on_usage_limit").IsRequired().HasDefaultValue(false);
builder.HasData(new AppSettingsEntity { Id = AppSettingsEntity.SingletonId }); builder.HasData(new AppSettingsEntity { Id = AppSettingsEntity.SingletonId });
} }
} }
@@ -83,6 +83,7 @@ public class TaskEntityConfiguration : IEntityTypeConfiguration<TaskEntity>
builder.Property(t => t.FailureReason).HasColumnName("failure_reason"); builder.Property(t => t.FailureReason).HasColumnName("failure_reason");
builder.Property(t => t.FailureTurnsUsed).HasColumnName("failure_turns_used"); builder.Property(t => t.FailureTurnsUsed).HasColumnName("failure_turns_used");
builder.Property(t => t.FailureMaxTurns).HasColumnName("failure_max_turns"); builder.Property(t => t.FailureMaxTurns).HasColumnName("failure_max_turns");
builder.Property(t => t.UsageLimitAutoContinuedAt).HasColumnName("usage_limit_auto_continued_at");
builder.Property(t => t.LogPath).HasColumnName("log_path"); builder.Property(t => t.LogPath).HasColumnName("log_path");
builder.Property(t => t.CreatedAt).HasColumnName("created_at").IsRequired(); builder.Property(t => t.CreatedAt).HasColumnName("created_at").IsRequired();
builder.Property(t => t.StartedAt).HasColumnName("started_at"); builder.Property(t => t.StartedAt).HasColumnName("started_at");
@@ -0,0 +1,961 @@
// <auto-generated />
using System;
using ClaudeDo.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
#nullable disable
namespace ClaudeDo.Data.Migrations
{
[DbContext(typeof(ClaudeDoDbContext))]
[Migration("20260821155619_AddUsageLimitAutoContinue")]
partial class AddUsageLimitAutoContinue
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.11");
modelBuilder.Entity("ClaudeDo.Data.Models.AppSettingsEntity", b =>
{
b.Property<int>("Id")
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<bool>("AutoContinueOnUsageLimit")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("auto_continue_on_usage_limit");
b.Property<string>("CentralWorktreeRoot")
.HasColumnType("TEXT")
.HasColumnName("central_worktree_root");
b.Property<int>("DailyPrepMaxTasks")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(5)
.HasColumnName("daily_prep_max_tasks");
b.Property<string>("DefaultClaudeInstructions")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("")
.HasColumnName("default_claude_instructions");
b.Property<int>("DefaultMaxTurns")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(40)
.HasColumnName("default_max_turns");
b.Property<string>("DefaultModel")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("sonnet")
.HasColumnName("default_model");
b.Property<string>("DefaultPermissionMode")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("bypassPermissions")
.HasColumnName("default_permission_mode");
b.Property<int>("MaxParallelExecutions")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(1)
.HasColumnName("max_parallel_executions");
b.Property<int>("MaxTurnsCeiling")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(80)
.HasColumnName("max_turns_ceiling");
b.Property<string>("ModelPresets")
.HasColumnType("TEXT")
.HasColumnName("model_presets");
b.Property<int>("NextTaskNumber")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(1)
.HasColumnName("next_task_number");
b.Property<string>("RepoImportFolders")
.HasColumnType("TEXT")
.HasColumnName("repo_import_folders");
b.Property<string>("ReportExcludedPaths")
.HasColumnType("TEXT")
.HasColumnName("report_excluded_paths");
b.Property<string>("SessionSkills")
.HasColumnType("TEXT")
.HasColumnName("session_skills");
b.Property<int>("StandupWeekday")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(3)
.HasColumnName("standup_weekday");
b.Property<int>("UsageGateFiveHourPct")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(80)
.HasColumnName("usage_gate_five_hour_pct");
b.Property<int>("UsageGateSevenDayPct")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(90)
.HasColumnName("usage_gate_seven_day_pct");
b.Property<int>("UsageThrottleFiveHourHardPct")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(65)
.HasColumnName("usage_throttle_five_hour_hard_pct");
b.Property<int>("UsageThrottleFiveHourSoftPct")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(50)
.HasColumnName("usage_throttle_five_hour_soft_pct");
b.Property<int>("UsageThrottleSevenDayHardPct")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(65)
.HasColumnName("usage_throttle_seven_day_hard_pct");
b.Property<int>("UsageThrottleSevenDaySoftPct")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(50)
.HasColumnName("usage_throttle_seven_day_soft_pct");
b.Property<int>("WorktreeAutoCleanupDays")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(7)
.HasColumnName("worktree_auto_cleanup_days");
b.Property<bool>("WorktreeAutoCleanupEnabled")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("worktree_auto_cleanup_enabled");
b.Property<string>("WorktreeStrategy")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("sibling")
.HasColumnName("worktree_strategy");
b.HasKey("Id");
b.ToTable("app_settings", (string)null);
b.HasData(
new
{
Id = 1,
AutoContinueOnUsageLimit = false,
DailyPrepMaxTasks = 5,
DefaultClaudeInstructions = "",
DefaultMaxTurns = 40,
DefaultModel = "sonnet",
DefaultPermissionMode = "auto",
MaxParallelExecutions = 1,
MaxTurnsCeiling = 80,
NextTaskNumber = 1,
StandupWeekday = 3,
UsageGateFiveHourPct = 80,
UsageGateSevenDayPct = 90,
UsageThrottleFiveHourHardPct = 65,
UsageThrottleFiveHourSoftPct = 50,
UsageThrottleSevenDayHardPct = 65,
UsageThrottleSevenDaySoftPct = 50,
WorktreeAutoCleanupDays = 7,
WorktreeAutoCleanupEnabled = false,
WorktreeStrategy = "sibling"
});
});
modelBuilder.Entity("ClaudeDo.Data.Models.DailyNoteEntity", b =>
{
b.Property<string>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
b.Property<DateTime>("CreatedAt")
.HasColumnType("TEXT")
.HasColumnName("created_at");
b.Property<DateOnly>("Date")
.HasColumnType("TEXT")
.HasColumnName("note_date");
b.Property<int>("SortOrder")
.HasColumnType("INTEGER")
.HasColumnName("sort_order");
b.Property<string>("Text")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("text");
b.HasKey("Id");
b.HasIndex("Date");
b.ToTable("daily_notes", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.ListConfigEntity", b =>
{
b.Property<string>("ListId")
.HasColumnType("TEXT")
.HasColumnName("list_id");
b.Property<string>("AgentPath")
.HasColumnType("TEXT")
.HasColumnName("agent_path");
b.Property<int?>("MaxTurns")
.HasColumnType("INTEGER")
.HasColumnName("max_turns");
b.Property<string>("Model")
.HasColumnType("TEXT")
.HasColumnName("model");
b.Property<bool>("SerializeOnFileOverlap")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("serialize_on_file_overlap");
b.Property<string>("SessionSkills")
.HasColumnType("TEXT")
.HasColumnName("session_skills");
b.Property<string>("SystemPrompt")
.HasColumnType("TEXT")
.HasColumnName("system_prompt");
b.Property<string>("VerifyCommand")
.HasColumnType("TEXT")
.HasColumnName("verify_command");
b.HasKey("ListId");
b.ToTable("list_config", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.ListEntity", b =>
{
b.Property<string>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
b.Property<DateTime>("CreatedAt")
.HasColumnType("TEXT")
.HasColumnName("created_at");
b.Property<string>("DefaultCommitType")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("chore")
.HasColumnName("default_commit_type");
b.Property<bool>("FindingsTracked")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("findings_tracked");
b.Property<bool>("IsManual")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("is_manual");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("name");
b.Property<int>("SortOrder")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(0)
.HasColumnName("sort_order");
b.Property<string>("WorkingDir")
.HasColumnType("TEXT")
.HasColumnName("working_dir");
b.HasKey("Id");
b.HasIndex("SortOrder")
.HasDatabaseName("idx_lists_sort");
b.ToTable("lists", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.PrimeScheduleEntity", b =>
{
b.Property<Guid>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
b.Property<DateTimeOffset>("CreatedAt")
.HasColumnType("TEXT")
.HasColumnName("created_at");
b.Property<int>("Days")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(31)
.HasColumnName("days_of_week");
b.Property<bool>("Enabled")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(true)
.HasColumnName("enabled");
b.Property<DateTimeOffset?>("LastRunAt")
.HasColumnType("TEXT")
.HasColumnName("last_run_at");
b.Property<string>("PromptOverride")
.HasColumnType("TEXT")
.HasColumnName("prompt_override");
b.Property<TimeSpan>("TimeOfDay")
.HasColumnType("TEXT")
.HasColumnName("time_of_day");
b.HasKey("Id");
b.ToTable("prime_schedules", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.SessionSkillEntity", b =>
{
b.Property<string>("Name")
.HasColumnType("TEXT")
.HasColumnName("name");
b.Property<DateTimeOffset>("AddedAt")
.HasColumnType("TEXT")
.HasColumnName("added_at");
b.Property<string>("Description")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("description");
b.Property<string>("PinnedRef")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("pinned_ref");
b.Property<string>("SourceUrl")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("source_url");
b.Property<string>("Subpath")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("subpath");
b.HasKey("Name");
b.ToTable("session_skills", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.SubtaskEntity", b =>
{
b.Property<string>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
b.Property<bool>("Completed")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("completed");
b.Property<DateTime>("CreatedAt")
.HasColumnType("TEXT")
.HasColumnName("created_at");
b.Property<int>("OrderNum")
.HasColumnType("INTEGER")
.HasColumnName("order_num");
b.Property<string>("TaskId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("task_id");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("title");
b.HasKey("Id");
b.HasIndex("TaskId")
.HasDatabaseName("idx_subtasks_task_id");
b.ToTable("subtasks", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.TaskAttachmentEntity", b =>
{
b.Property<string>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
b.Property<long>("ByteSize")
.HasColumnType("INTEGER")
.HasColumnName("byte_size");
b.Property<DateTime>("CreatedAt")
.HasColumnType("TEXT")
.HasColumnName("created_at");
b.Property<string>("FileName")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("file_name");
b.Property<string>("TaskId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("task_id");
b.HasKey("Id");
b.HasIndex("TaskId")
.HasDatabaseName("idx_task_attachments_task_id");
b.ToTable("task_attachments", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.TaskEntity", b =>
{
b.Property<string>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
b.Property<string>("AgentPath")
.HasColumnType("TEXT")
.HasColumnName("agent_path");
b.Property<string>("BlockedByTaskId")
.HasColumnType("TEXT")
.HasColumnName("blocked_by_task_id");
b.Property<string>("CommitType")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("chore")
.HasColumnName("commit_type");
b.Property<DateTime>("CreatedAt")
.HasColumnType("TEXT")
.HasColumnName("created_at");
b.Property<string>("CreatedBy")
.HasColumnType("TEXT")
.HasColumnName("created_by");
b.Property<string>("DependsOnTaskId")
.HasColumnType("TEXT")
.HasColumnName("depends_on_task_id");
b.Property<string>("Description")
.HasColumnType("TEXT")
.HasColumnName("description");
b.Property<int?>("FailureMaxTurns")
.HasColumnType("INTEGER")
.HasColumnName("failure_max_turns");
b.Property<string>("FailureReason")
.HasColumnType("TEXT")
.HasColumnName("failure_reason");
b.Property<int?>("FailureTurnsUsed")
.HasColumnType("INTEGER")
.HasColumnName("failure_turns_used");
b.Property<DateTime?>("FinishedAt")
.HasColumnType("TEXT")
.HasColumnName("finished_at");
b.Property<string>("HandlerBaseCommit")
.HasColumnType("TEXT")
.HasColumnName("handler_base_commit");
b.Property<string>("HandlerHeadCommit")
.HasColumnType("TEXT")
.HasColumnName("handler_head_commit");
b.Property<string>("InteractiveSessionId")
.HasColumnType("TEXT")
.HasColumnName("interactive_session_id");
b.Property<bool>("IsManual")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("is_manual");
b.Property<bool>("IsMyDay")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("is_my_day");
b.Property<bool>("IsStarred")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("is_starred");
b.Property<string>("ListId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("list_id");
b.Property<string>("LogPath")
.HasColumnType("TEXT")
.HasColumnName("log_path");
b.Property<int?>("MaxTurns")
.HasColumnType("INTEGER")
.HasColumnName("max_turns");
b.Property<string>("Model")
.HasColumnType("TEXT")
.HasColumnName("model");
b.Property<string>("Notes")
.HasColumnType("TEXT")
.HasColumnName("notes");
b.Property<int>("Number")
.HasColumnType("INTEGER")
.HasColumnName("number");
b.Property<string>("ParentTaskId")
.HasColumnType("TEXT")
.HasColumnName("parent_task_id");
b.Property<DateTime?>("PlanningFinalizedAt")
.HasColumnType("TEXT")
.HasColumnName("planning_finalized_at");
b.Property<string>("PlanningPhase")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("none")
.HasColumnName("planning_phase");
b.Property<string>("PlanningSessionId")
.HasColumnType("TEXT")
.HasColumnName("planning_session_id");
b.Property<string>("PlanningSessionToken")
.HasColumnType("TEXT")
.HasColumnName("planning_session_token");
b.Property<string>("Result")
.HasColumnType("TEXT")
.HasColumnName("result");
b.Property<string>("ReviewFeedback")
.HasColumnType("TEXT")
.HasColumnName("review_feedback");
b.Property<int>("RoadblockCount")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(0)
.HasColumnName("roadblock_count");
b.Property<DateTime?>("ScheduledFor")
.HasColumnType("TEXT")
.HasColumnName("scheduled_for");
b.Property<string>("ScopeGlobs")
.HasColumnType("TEXT")
.HasColumnName("scope_globs");
b.Property<string>("SessionSkills")
.HasColumnType("TEXT")
.HasColumnName("session_skills");
b.Property<int>("SortOrder")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(0)
.HasColumnName("sort_order");
b.Property<DateTime?>("StartedAt")
.HasColumnType("TEXT")
.HasColumnName("started_at");
b.Property<string>("Status")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("status");
b.Property<string>("SystemPrompt")
.HasColumnType("TEXT")
.HasColumnName("system_prompt");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("title");
b.Property<DateTime?>("UsageLimitAutoContinuedAt")
.HasColumnType("TEXT")
.HasColumnName("usage_limit_auto_continued_at");
b.HasKey("Id");
b.HasIndex("BlockedByTaskId")
.HasDatabaseName("idx_tasks_blocked_by");
b.HasIndex("DependsOnTaskId")
.HasDatabaseName("idx_tasks_depends_on");
b.HasIndex("ListId")
.HasDatabaseName("idx_tasks_list_id");
b.HasIndex("Number")
.IsUnique()
.HasDatabaseName("idx_tasks_number");
b.HasIndex("ParentTaskId")
.HasDatabaseName("idx_tasks_parent_task_id");
b.HasIndex("Status")
.HasDatabaseName("idx_tasks_status");
b.HasIndex("ListId", "SortOrder")
.HasDatabaseName("idx_tasks_list_sort");
b.ToTable("tasks", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.TaskRunEntity", b =>
{
b.Property<string>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
b.Property<int?>("CacheReadTokens")
.HasColumnType("INTEGER")
.HasColumnName("cache_read_tokens");
b.Property<int?>("CacheWriteTokens")
.HasColumnType("INTEGER")
.HasColumnName("cache_write_tokens");
b.Property<string>("ErrorMarkdown")
.HasColumnType("TEXT")
.HasColumnName("error_markdown");
b.Property<string>("Errors")
.HasColumnType("TEXT")
.HasColumnName("errors");
b.Property<int?>("ExitCode")
.HasColumnType("INTEGER")
.HasColumnName("exit_code");
b.Property<DateTime?>("FinishedAt")
.HasColumnType("TEXT")
.HasColumnName("finished_at");
b.Property<bool>("IsRetry")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("is_retry");
b.Property<string>("LogPath")
.HasColumnType("TEXT")
.HasColumnName("log_path");
b.Property<string>("Model")
.HasColumnType("TEXT")
.HasColumnName("model");
b.Property<string>("Prompt")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("prompt");
b.Property<string>("ResultMarkdown")
.HasColumnType("TEXT")
.HasColumnName("result_markdown");
b.Property<string>("ResultSubtype")
.HasColumnType("TEXT")
.HasColumnName("result_subtype");
b.Property<int>("RunNumber")
.HasColumnType("INTEGER")
.HasColumnName("run_number");
b.Property<string>("SessionId")
.HasColumnType("TEXT")
.HasColumnName("session_id");
b.Property<DateTime?>("StartedAt")
.HasColumnType("TEXT")
.HasColumnName("started_at");
b.Property<string>("StructuredOutputJson")
.HasColumnType("TEXT")
.HasColumnName("structured_output");
b.Property<string>("TaskId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("task_id");
b.Property<string>("TerminalReason")
.HasColumnType("TEXT")
.HasColumnName("terminal_reason");
b.Property<int?>("TokensIn")
.HasColumnType("INTEGER")
.HasColumnName("tokens_in");
b.Property<int?>("TokensOut")
.HasColumnType("INTEGER")
.HasColumnName("tokens_out");
b.Property<int?>("TurnCount")
.HasColumnType("INTEGER")
.HasColumnName("turn_count");
b.HasKey("Id");
b.HasIndex("TaskId")
.HasDatabaseName("idx_task_runs_task_id");
b.ToTable("task_runs", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.WeekReportEntity", b =>
{
b.Property<string>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
b.Property<DateOnly>("EndDate")
.HasColumnType("TEXT")
.HasColumnName("end_date");
b.Property<DateTime>("GeneratedAt")
.HasColumnType("TEXT")
.HasColumnName("generated_at");
b.Property<string>("Markdown")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("markdown");
b.Property<DateOnly>("StartDate")
.HasColumnType("TEXT")
.HasColumnName("start_date");
b.HasKey("Id");
b.HasIndex("StartDate", "EndDate")
.IsUnique();
b.ToTable("week_reports", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.WorktreeEntity", b =>
{
b.Property<string>("TaskId")
.HasColumnType("TEXT")
.HasColumnName("task_id");
b.Property<string>("BaseCommit")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("base_commit");
b.Property<string>("BranchName")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("branch_name");
b.Property<DateTime>("CreatedAt")
.HasColumnType("TEXT")
.HasColumnName("created_at");
b.Property<string>("DiffStat")
.HasColumnType("TEXT")
.HasColumnName("diff_stat");
b.Property<string>("HeadCommit")
.HasColumnType("TEXT")
.HasColumnName("head_commit");
b.Property<string>("MergeCommit")
.HasColumnType("TEXT")
.HasColumnName("merge_commit");
b.Property<string>("Path")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("path");
b.Property<string>("State")
.IsRequired()
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("active")
.HasColumnName("state");
b.HasKey("TaskId");
b.ToTable("worktrees", (string)null);
});
modelBuilder.Entity("ClaudeDo.Data.Models.ListConfigEntity", b =>
{
b.HasOne("ClaudeDo.Data.Models.ListEntity", "List")
.WithOne("Config")
.HasForeignKey("ClaudeDo.Data.Models.ListConfigEntity", "ListId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("List");
});
modelBuilder.Entity("ClaudeDo.Data.Models.SubtaskEntity", b =>
{
b.HasOne("ClaudeDo.Data.Models.TaskEntity", "Task")
.WithMany("Subtasks")
.HasForeignKey("TaskId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Task");
});
modelBuilder.Entity("ClaudeDo.Data.Models.TaskAttachmentEntity", b =>
{
b.HasOne("ClaudeDo.Data.Models.TaskEntity", "Task")
.WithMany()
.HasForeignKey("TaskId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Task");
});
modelBuilder.Entity("ClaudeDo.Data.Models.TaskEntity", b =>
{
b.HasOne("ClaudeDo.Data.Models.TaskEntity", null)
.WithMany()
.HasForeignKey("BlockedByTaskId")
.OnDelete(DeleteBehavior.SetNull);
b.HasOne("ClaudeDo.Data.Models.TaskEntity", null)
.WithMany()
.HasForeignKey("DependsOnTaskId")
.OnDelete(DeleteBehavior.SetNull);
b.HasOne("ClaudeDo.Data.Models.ListEntity", "List")
.WithMany("Tasks")
.HasForeignKey("ListId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("ClaudeDo.Data.Models.TaskEntity", "Parent")
.WithMany("Children")
.HasForeignKey("ParentTaskId")
.OnDelete(DeleteBehavior.Restrict);
b.Navigation("List");
b.Navigation("Parent");
});
modelBuilder.Entity("ClaudeDo.Data.Models.TaskRunEntity", b =>
{
b.HasOne("ClaudeDo.Data.Models.TaskEntity", "Task")
.WithMany("Runs")
.HasForeignKey("TaskId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Task");
});
modelBuilder.Entity("ClaudeDo.Data.Models.WorktreeEntity", b =>
{
b.HasOne("ClaudeDo.Data.Models.TaskEntity", "Task")
.WithOne("Worktree")
.HasForeignKey("ClaudeDo.Data.Models.WorktreeEntity", "TaskId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Task");
});
modelBuilder.Entity("ClaudeDo.Data.Models.ListEntity", b =>
{
b.Navigation("Config");
b.Navigation("Tasks");
});
modelBuilder.Entity("ClaudeDo.Data.Models.TaskEntity", b =>
{
b.Navigation("Children");
b.Navigation("Runs");
b.Navigation("Subtasks");
b.Navigation("Worktree");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,40 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ClaudeDo.Data.Migrations
{
/// <inheritdoc />
public partial class AddUsageLimitAutoContinue : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<DateTime>(
name: "usage_limit_auto_continued_at",
table: "tasks",
type: "TEXT",
nullable: true);
migrationBuilder.AddColumn<bool>(
name: "auto_continue_on_usage_limit",
table: "app_settings",
type: "INTEGER",
nullable: false,
defaultValue: false);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "usage_limit_auto_continued_at",
table: "tasks");
migrationBuilder.DropColumn(
name: "auto_continue_on_usage_limit",
table: "app_settings");
}
}
}
@@ -23,6 +23,12 @@ namespace ClaudeDo.Data.Migrations
.HasColumnType("INTEGER") .HasColumnType("INTEGER")
.HasColumnName("id"); .HasColumnName("id");
b.Property<bool>("AutoContinueOnUsageLimit")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(false)
.HasColumnName("auto_continue_on_usage_limit");
b.Property<string>("CentralWorktreeRoot") b.Property<string>("CentralWorktreeRoot")
.HasColumnType("TEXT") .HasColumnType("TEXT")
.HasColumnName("central_worktree_root"); .HasColumnName("central_worktree_root");
@@ -163,6 +169,7 @@ namespace ClaudeDo.Data.Migrations
new new
{ {
Id = 1, Id = 1,
AutoContinueOnUsageLimit = false,
DailyPrepMaxTasks = 5, DailyPrepMaxTasks = 5,
DefaultClaudeInstructions = "", DefaultClaudeInstructions = "",
DefaultMaxTurns = 40, DefaultMaxTurns = 40,
@@ -632,6 +639,10 @@ namespace ClaudeDo.Data.Migrations
.HasColumnType("TEXT") .HasColumnType("TEXT")
.HasColumnName("title"); .HasColumnName("title");
b.Property<DateTime?>("UsageLimitAutoContinuedAt")
.HasColumnType("TEXT")
.HasColumnName("usage_limit_auto_continued_at");
b.HasKey("Id"); b.HasKey("Id");
b.HasIndex("BlockedByTaskId") b.HasIndex("BlockedByTaskId")
@@ -55,4 +55,9 @@ public sealed class AppSettingsEntity
public int UsageThrottleFiveHourHardPct { get; set; } = 65; public int UsageThrottleFiveHourHardPct { get; set; } = 65;
public int UsageThrottleSevenDaySoftPct { get; set; } = 50; public int UsageThrottleSevenDaySoftPct { get; set; } = 50;
public int UsageThrottleSevenDayHardPct { get; set; } = 65; public int UsageThrottleSevenDayHardPct { get; set; } = 65;
// Off by default. When on: a task that failed on the Anthropic usage limit gets exactly one
// automatic ContinueTask once the 5h window resets, and the gate-blocked queue wakes exactly
// at that reset time instead of only on the 30s backstop.
public bool AutoContinueOnUsageLimit { get; set; }
} }
+4
View File
@@ -49,6 +49,10 @@ public sealed class TaskEntity
public string? FailureReason { get; set; } public string? FailureReason { get; set; }
public int? FailureTurnsUsed { get; set; } public int? FailureTurnsUsed { get; set; }
public int? FailureMaxTurns { get; set; } public int? FailureMaxTurns { get; set; }
// Set once QueueService's UsageLimitAutoContinueCoordinator fires ContinueTask for a
// FailureReason=="usage_limit" task. Guards against firing it a second time for the same
// task — never reset automatically, so auto-continue never loops on a repeat limit hit.
public DateTime? UsageLimitAutoContinuedAt { get; set; }
public string? LogPath { get; set; } public string? LogPath { get; set; }
public required DateTime CreatedAt { get; init; } public required DateTime CreatedAt { get; init; }
public DateTime? StartedAt { get; set; } public DateTime? StartedAt { get; set; }
@@ -92,6 +92,7 @@ public sealed class AppSettingsRepository
row.UsageThrottleFiveHourHardPct = Math.Clamp(updated.UsageThrottleFiveHourHardPct, 0, 100); row.UsageThrottleFiveHourHardPct = Math.Clamp(updated.UsageThrottleFiveHourHardPct, 0, 100);
row.UsageThrottleSevenDaySoftPct = Math.Clamp(updated.UsageThrottleSevenDaySoftPct, 0, 100); row.UsageThrottleSevenDaySoftPct = Math.Clamp(updated.UsageThrottleSevenDaySoftPct, 0, 100);
row.UsageThrottleSevenDayHardPct = Math.Clamp(updated.UsageThrottleSevenDayHardPct, 0, 100); row.UsageThrottleSevenDayHardPct = Math.Clamp(updated.UsageThrottleSevenDayHardPct, 0, 100);
row.AutoContinueOnUsageLimit = updated.AutoContinueOnUsageLimit;
await _context.SaveChangesAsync(ct); await _context.SaveChangesAsync(ct);
} }
@@ -214,6 +214,27 @@ public sealed class TaskRepository
.ExecuteUpdateAsync(s => s.SetProperty(t => t.RoadblockCount, count), ct); .ExecuteUpdateAsync(s => s.SetProperty(t => t.RoadblockCount, count), ct);
} }
/// Tasks eligible for UsageLimitAutoContinueCoordinator: failed on the usage limit and never
/// auto-continued before (that flag is never reset, so a task is only offered this once).
public async Task<List<TaskEntity>> GetUsageLimitAutoContinueCandidatesAsync(CancellationToken ct = default)
{
return await _context.Tasks
.Where(t => t.Status == TaskStatus.Failed
&& t.FailureReason == "usage_limit"
&& t.UsageLimitAutoContinuedAt == null)
.ToListAsync(ct);
}
/// Atomically claims the auto-continue slot for one task. Returns false if it was already
/// claimed (e.g. by an earlier tick) — the caller must not fire ContinueTask in that case.
public async Task<bool> TryClaimUsageLimitAutoContinueAsync(string taskId, DateTime firedAtUtc, CancellationToken ct = default)
{
var affected = await _context.Tasks
.Where(t => t.Id == taskId && t.UsageLimitAutoContinuedAt == null)
.ExecuteUpdateAsync(s => s.SetProperty(t => t.UsageLimitAutoContinuedAt, firedAtUtc), ct);
return affected > 0;
}
internal async Task<int> FlipAllRunningToFailedAsync(string reason, CancellationToken ct = default) internal async Task<int> FlipAllRunningToFailedAsync(string reason, CancellationToken ct = default)
{ {
var resultText = "[stale] " + reason; var resultText = "[stale] " + reason;
@@ -43,6 +43,8 @@
"usageGateHint": "Ab dieser Auslastung holt die Queue keine neuen Tasks mehr; laufende Tasks laufen zu Ende. 0 schaltet den Stopp ab.", "usageGateHint": "Ab dieser Auslastung holt die Queue keine neuen Tasks mehr; laufende Tasks laufen zu Ende. 0 schaltet den Stopp ab.",
"usageGateFiveHourPct": "5-Stunden-Fenster (%)", "usageGateFiveHourPct": "5-Stunden-Fenster (%)",
"usageGateSevenDayPct": "7-Tage-Fenster (%)", "usageGateSevenDayPct": "7-Tage-Fenster (%)",
"autoContinueOnUsageLimit": "Nach Limit-Reset fortsetzen",
"autoContinueOnUsageLimitHint": "Ist ein Run am Nutzungslimit abgebrochen, wird er nach Ablauf des 5-Stunden-Fensters automatisch fortgesetzt. Weckt außerdem die pausierte Queue genau zum Reset-Zeitpunkt statt erst beim nächsten Backstop-Check.",
"reportExcludedPaths": "Bericht: ausgeschlossene Pfade (einer pro Zeile)", "reportExcludedPaths": "Bericht: ausgeschlossene Pfade (einer pro Zeile)",
"standupWeekday": "Standup-Wochentag", "standupWeekday": "Standup-Wochentag",
"weekdaySunday": "Sonntag", "weekdaySunday": "Sonntag",
@@ -43,6 +43,8 @@
"usageGateHint": "Once usage reaches this level, the queue stops picking up new tasks; running tasks finish normally. 0 turns the stop off.", "usageGateHint": "Once usage reaches this level, the queue stops picking up new tasks; running tasks finish normally. 0 turns the stop off.",
"usageGateFiveHourPct": "5-hour window (%)", "usageGateFiveHourPct": "5-hour window (%)",
"usageGateSevenDayPct": "7-day window (%)", "usageGateSevenDayPct": "7-day window (%)",
"autoContinueOnUsageLimit": "Continue on session limit reset",
"autoContinueOnUsageLimitHint": "When a run stops because it hit the usage limit, automatically continue it once the 5-hour window resets. Also wakes the paused queue exactly at reset time instead of waiting for the next backstop check.",
"reportExcludedPaths": "Report: excluded paths (one per line)", "reportExcludedPaths": "Report: excluded paths (one per line)",
"standupWeekday": "Standup weekday", "standupWeekday": "Standup weekday",
"weekdaySunday": "Sunday", "weekdaySunday": "Sunday",
+2 -1
View File
@@ -732,7 +732,8 @@ public sealed record AppSettingsDto(
int UsageThrottleFiveHourSoftPct = 50, int UsageThrottleFiveHourSoftPct = 50,
int UsageThrottleFiveHourHardPct = 65, int UsageThrottleFiveHourHardPct = 65,
int UsageThrottleSevenDaySoftPct = 50, int UsageThrottleSevenDaySoftPct = 50,
int UsageThrottleSevenDayHardPct = 65); int UsageThrottleSevenDayHardPct = 65,
bool AutoContinueOnUsageLimit = false);
// Per-model run defaults (effort + turn budget) edited in Settings → General. // Per-model run defaults (effort + turn budget) edited in Settings → General.
public sealed record ModelPresetDto(string Model, string Effort, int MaxTurns); public sealed record ModelPresetDto(string Model, string Effort, int MaxTurns);
@@ -24,6 +24,9 @@ public sealed partial class GeneralSettingsTabViewModel : ViewModelBase
// Percentage of the 5h/7d Claude usage window at which the autonomous queue pauses. 0 = gate off. // Percentage of the 5h/7d Claude usage window at which the autonomous queue pauses. 0 = gate off.
[ObservableProperty] private int _usageGateFiveHourPct = 80; [ObservableProperty] private int _usageGateFiveHourPct = 80;
[ObservableProperty] private int _usageGateSevenDayPct = 90; [ObservableProperty] private int _usageGateSevenDayPct = 90;
// Off by default: auto-continues a usage-limit-failed task once the 5h window resets, and
// wakes a gate-blocked queue exactly at that reset time instead of only the 30s backstop.
[ObservableProperty] private bool _autoContinueOnUsageLimit;
// Newline-separated path prefixes excluded from the weekly report. // Newline-separated path prefixes excluded from the weekly report.
[ObservableProperty] private string _reportExcludedPaths = @"C:\Private"; [ObservableProperty] private string _reportExcludedPaths = @"C:\Private";
// 0=Sunday..6=Saturday (System.DayOfWeek); default Wednesday. // 0=Sunday..6=Saturday (System.DayOfWeek); default Wednesday.
@@ -129,6 +129,7 @@ public sealed partial class SettingsModalViewModel : ViewModelBase
General.MaxParallelExecutions = dto.MaxParallelExecutions; General.MaxParallelExecutions = dto.MaxParallelExecutions;
General.UsageGateFiveHourPct = dto.UsageGateFiveHourPct; General.UsageGateFiveHourPct = dto.UsageGateFiveHourPct;
General.UsageGateSevenDayPct = dto.UsageGateSevenDayPct; General.UsageGateSevenDayPct = dto.UsageGateSevenDayPct;
General.AutoContinueOnUsageLimit = dto.AutoContinueOnUsageLimit;
_throttleStages = ( _throttleStages = (
dto.UsageThrottleFiveHourSoftPct, dto.UsageThrottleFiveHourHardPct, dto.UsageThrottleFiveHourSoftPct, dto.UsageThrottleFiveHourHardPct,
dto.UsageThrottleSevenDaySoftPct, dto.UsageThrottleSevenDayHardPct); dto.UsageThrottleSevenDaySoftPct, dto.UsageThrottleSevenDayHardPct);
@@ -189,7 +190,8 @@ public sealed partial class SettingsModalViewModel : ViewModelBase
_throttleStages.FiveSoft, _throttleStages.FiveSoft,
_throttleStages.FiveHard, _throttleStages.FiveHard,
_throttleStages.SevenSoft, _throttleStages.SevenSoft,
_throttleStages.SevenHard); _throttleStages.SevenHard,
General.AutoContinueOnUsageLimit);
await _worker.UpdateAppSettingsAsync(dto); await _worker.UpdateAppSettingsAsync(dto);
await Prime.SaveAsync(); await Prime.SaveAsync();
await OnlineInbox.SaveAsync(); await OnlineInbox.SaveAsync();
@@ -237,6 +237,10 @@
HorizontalAlignment="Left" Width="140"/> HorizontalAlignment="Left" Width="140"/>
</StackPanel> </StackPanel>
</StackPanel> </StackPanel>
<CheckBox IsChecked="{Binding General.AutoContinueOnUsageLimit, Mode=TwoWay}"
Content="{loc:Tr settings.general.autoContinueOnUsageLimit}"/>
<TextBlock Text="{loc:Tr settings.general.autoContinueOnUsageLimitHint}"
Opacity="0.6" FontSize="12" TextWrapping="Wrap"/>
</StackPanel> </StackPanel>
</StackPanel> </StackPanel>
</ScrollViewer> </ScrollViewer>
+5 -2
View File
@@ -55,7 +55,8 @@ public record AppSettingsDto(
int UsageThrottleFiveHourSoftPct = 50, int UsageThrottleFiveHourSoftPct = 50,
int UsageThrottleFiveHourHardPct = 65, int UsageThrottleFiveHourHardPct = 65,
int UsageThrottleSevenDaySoftPct = 50, int UsageThrottleSevenDaySoftPct = 50,
int UsageThrottleSevenDayHardPct = 65); int UsageThrottleSevenDayHardPct = 65,
bool AutoContinueOnUsageLimit = false);
// Per-model run defaults (effort + turn budget) edited in Settings -> General. // Per-model run defaults (effort + turn budget) edited in Settings -> General.
public record ModelPresetDto(string Model, string Effort, int MaxTurns); public record ModelPresetDto(string Model, string Effort, int MaxTurns);
@@ -471,7 +472,8 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
row.UsageThrottleFiveHourSoftPct, row.UsageThrottleFiveHourSoftPct,
row.UsageThrottleFiveHourHardPct, row.UsageThrottleFiveHourHardPct,
row.UsageThrottleSevenDaySoftPct, row.UsageThrottleSevenDaySoftPct,
row.UsageThrottleSevenDayHardPct); row.UsageThrottleSevenDayHardPct,
row.AutoContinueOnUsageLimit);
} }
public async Task UpdateAppSettings(AppSettingsDto dto) public async Task UpdateAppSettings(AppSettingsDto dto)
@@ -506,6 +508,7 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
UsageThrottleFiveHourHardPct = dto.UsageThrottleFiveHourHardPct, UsageThrottleFiveHourHardPct = dto.UsageThrottleFiveHourHardPct,
UsageThrottleSevenDaySoftPct = dto.UsageThrottleSevenDaySoftPct, UsageThrottleSevenDaySoftPct = dto.UsageThrottleSevenDaySoftPct,
UsageThrottleSevenDayHardPct = dto.UsageThrottleSevenDayHardPct, UsageThrottleSevenDayHardPct = dto.UsageThrottleSevenDayHardPct,
AutoContinueOnUsageLimit = dto.AutoContinueOnUsageLimit,
}); });
} }
+3
View File
@@ -156,6 +156,9 @@ builder.Services.AddHostedService<PrimeScheduler>();
builder.Services.AddSingleton<IRefineRunner, RefineRunner>(); builder.Services.AddSingleton<IRefineRunner, RefineRunner>();
builder.Services.AddSingleton<IRefineBroadcaster>(sp => sp.GetRequiredService<HubBroadcaster>()); builder.Services.AddSingleton<IRefineBroadcaster>(sp => sp.GetRequiredService<HubBroadcaster>());
// "Continue on session limit reset" toggle — depends on UsageState, registered below.
builder.Services.AddSingleton<UsageLimitAutoContinueCoordinator>();
// QueueService: singleton + hosted service (same instance). // QueueService: singleton + hosted service (same instance).
builder.Services.AddSingleton<QueueService>(); builder.Services.AddSingleton<QueueService>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<QueueService>()); builder.Services.AddHostedService(sp => sp.GetRequiredService<QueueService>());
+35 -1
View File
@@ -26,11 +26,13 @@ public sealed class QueueService : BackgroundService
private readonly IUsageGate _usageGate; private readonly IUsageGate _usageGate;
private readonly UsageState _usageState; private readonly UsageState _usageState;
private readonly HubBroadcaster _broadcaster; private readonly HubBroadcaster _broadcaster;
private readonly UsageLimitAutoContinueCoordinator _usageLimitAutoContinue;
private readonly object _lock = new(); private readonly object _lock = new();
private readonly Dictionary<string, QueueSlotState> _queueSlots = new(); private readonly Dictionary<string, QueueSlotState> _queueSlots = new();
private bool _usageGateBlocked; private bool _usageGateBlocked;
private int? _lastEffectiveSlots; private int? _lastEffectiveSlots;
private DateTimeOffset? _scheduledResetWakeAt;
public QueueService( public QueueService(
IDbContextFactory<ClaudeDoDbContext> dbFactory, IDbContextFactory<ClaudeDoDbContext> dbFactory,
@@ -44,7 +46,8 @@ public sealed class QueueService : BackgroundService
RunCancellationRegistry runCancels, RunCancellationRegistry runCancels,
IUsageGate usageGate, IUsageGate usageGate,
UsageState usageState, UsageState usageState,
HubBroadcaster broadcaster) HubBroadcaster broadcaster,
UsageLimitAutoContinueCoordinator usageLimitAutoContinue)
{ {
_dbFactory = dbFactory; _dbFactory = dbFactory;
_runner = runner; _runner = runner;
@@ -58,6 +61,7 @@ public sealed class QueueService : BackgroundService
_usageGate = usageGate; _usageGate = usageGate;
_usageState = usageState; _usageState = usageState;
_broadcaster = broadcaster; _broadcaster = broadcaster;
_usageLimitAutoContinue = usageLimitAutoContinue;
} }
public IReadOnlyList<(string slot, string taskId, DateTime startedAt)> GetActive() public IReadOnlyList<(string slot, string taskId, DateTime startedAt)> GetActive()
@@ -130,6 +134,14 @@ public sealed class QueueService : BackgroundService
var gateDecision = await _usageGate.EvaluateAsync(stoppingToken); var gateDecision = await _usageGate.EvaluateAsync(stoppingToken);
await ReportUsageGateTransitionAsync(gateDecision); await ReportUsageGateTransitionAsync(gateDecision);
// "Continue on session limit reset" toggle: schedule an exact wake for the reset
// time (instead of only relying on the 30s backstop above) and fire any pending
// auto-continues. No-ops entirely while the toggle is off or no reset time is known.
var scheduledWakeAt = await _usageLimitAutoContinue.GetScheduledWakeAtAsync(stoppingToken);
if (scheduledWakeAt is not null)
ScheduleResetWake(scheduledWakeAt.Value, stoppingToken);
await _usageLimitAutoContinue.RunAsync(stoppingToken);
// Only queue refill is gated. Runs already in flight (RunNow, ContinueTask, // Only queue refill is gated. Runs already in flight (RunNow, ContinueTask,
// interactive sessions, planning, daily prep) keep going regardless. // interactive sessions, planning, daily prep) keep going regardless.
if (!gateDecision.IsBlocked) if (!gateDecision.IsBlocked)
@@ -210,6 +222,28 @@ public sealed class QueueService : BackgroundService
return false; return false;
} }
// Wakes the queue loop exactly at the usage-limit reset time instead of leaving it to the
// (up to 30s late) backstop — schedules once per distinct target and is a no-op for a target
// already scheduled or already in the past (this tick's own check already covers "now").
private void ScheduleResetWake(DateTimeOffset resetsAt, CancellationToken ct)
{
if (_scheduledResetWakeAt == resetsAt) return;
_scheduledResetWakeAt = resetsAt;
var delay = resetsAt - DateTimeOffset.UtcNow;
if (delay <= TimeSpan.Zero) return;
_ = Task.Run(async () =>
{
try
{
await Task.Delay(delay, ct);
_waker.Wake();
}
catch (OperationCanceledException) { }
}, ct);
}
private async Task ReportUsageGateTransitionAsync(UsageGateDecision decision) private async Task ReportUsageGateTransitionAsync(UsageGateDecision decision)
{ {
if (decision.IsBlocked == _usageGateBlocked) return; if (decision.IsBlocked == _usageGateBlocked) return;
@@ -0,0 +1,99 @@
using ClaudeDo.Data;
using ClaudeDo.Data.Models;
using ClaudeDo.Data.Repositories;
using ClaudeDo.Worker.Hub;
using ClaudeDo.Worker.Usage;
using Microsoft.EntityFrameworkCore;
namespace ClaudeDo.Worker.Queue;
/// <summary>
/// The "Continue on session limit reset" toggle (<see cref="AppSettingsEntity.AutoContinueOnUsageLimit"/>).
/// Off by default and fails open on every missing signal — no snapshot, no reset time — never a
/// timer, never a crash, never a change from today's behaviour. Called once per
/// <see cref="QueueService"/> tick (both from its wake and its 30s backstop):
/// <see cref="GetScheduledWakeAtAsync"/> tells the caller when to schedule an exact wake so a
/// gate-blocked queue doesn't wait out the backstop, and <see cref="RunAsync"/> fires exactly one
/// <c>ContinueTask</c> per eligible task once the window has actually reset.
/// </summary>
public sealed class UsageLimitAutoContinueCoordinator
{
public const string ContinuePrompt =
"The Anthropic usage limit window that stopped this run has since reset. Continue the task from where the previous run left off.";
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
private readonly UsageState _usageState;
private readonly OverrideSlotService _override;
private readonly HubBroadcaster _broadcaster;
private readonly ILogger<UsageLimitAutoContinueCoordinator> _logger;
public UsageLimitAutoContinueCoordinator(
IDbContextFactory<ClaudeDoDbContext> dbFactory, UsageState usageState, OverrideSlotService overrideSlot,
HubBroadcaster broadcaster, ILogger<UsageLimitAutoContinueCoordinator> logger)
{
_dbFactory = dbFactory;
_usageState = usageState;
_override = overrideSlot;
_broadcaster = broadcaster;
_logger = logger;
}
/// <summary>The reset time <see cref="QueueService"/> should schedule an exact wake for, or
/// null when the toggle is off or no reset time is known yet — fail open, no timer.</summary>
public async Task<DateTimeOffset?> GetScheduledWakeAtAsync(CancellationToken ct)
{
var settings = await TryReadSettingsAsync(ct);
if (settings is null || !settings.AutoContinueOnUsageLimit) return null;
return _usageState.Snapshot?.FiveHour?.ResetsAt;
}
/// <summary>Fires exactly one <c>ContinueTask</c> per task failed on the usage limit, once the
/// 5h window has actually reset. No-ops (never throws) when the toggle is off, no reset time
/// is known, or the window hasn't reset yet.</summary>
public async Task RunAsync(CancellationToken ct)
{
var settings = await TryReadSettingsAsync(ct);
if (settings is null || !settings.AutoContinueOnUsageLimit) return;
var resetsAt = _usageState.Snapshot?.FiveHour?.ResetsAt;
if (resetsAt is null || DateTimeOffset.UtcNow < resetsAt.Value) return;
List<TaskEntity> candidates;
using (var context = _dbFactory.CreateDbContext())
candidates = await new TaskRepository(context).GetUsageLimitAutoContinueCandidatesAsync(ct);
foreach (var task in candidates)
{
bool claimed;
using (var context = _dbFactory.CreateDbContext())
claimed = await new TaskRepository(context).TryClaimUsageLimitAutoContinueAsync(task.Id, DateTime.UtcNow, ct);
if (!claimed) continue; // already fired for this task — never twice.
try
{
await _override.ContinueTask(task.Id, ContinuePrompt);
await _broadcaster.WorkerLog(
$"Auto-continuing #{task.Number} \"{task.Title}\" after usage-limit reset",
WorkerLogLevel.Info, DateTime.UtcNow);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to auto-continue task {TaskId} after usage-limit reset", task.Id);
}
}
}
private async Task<AppSettingsEntity?> TryReadSettingsAsync(CancellationToken ct)
{
try
{
using var context = _dbFactory.CreateDbContext();
return await new AppSettingsRepository(context).GetAsync(ct);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "UsageLimitAutoContinueCoordinator: failed to read settings; skipping tick");
return null;
}
}
}
+18 -6
View File
@@ -1,4 +1,5 @@
using System.Text.Json; using System.Text.Json;
using System.Text.RegularExpressions;
using ClaudeDo.Data; using ClaudeDo.Data;
using ClaudeDo.Data.Models; using ClaudeDo.Data.Models;
using ClaudeDo.Data.Repositories; using ClaudeDo.Data.Repositories;
@@ -159,13 +160,13 @@ public sealed class TaskRunner
else else
{ {
await MarkFailed(task.Id, task.Number, task.Title, slot, retryResult.ErrorMarkdown, retryResult.TurnCount, await MarkFailed(task.Id, task.Number, task.Title, slot, retryResult.ErrorMarkdown, retryResult.TurnCount,
retryConfig.MaxTurns, ClassifyFailureReason(retryResult.TerminalReason)); retryConfig.MaxTurns, ClassifyFailureReason(retryResult.TerminalReason, retryResult.ErrorMarkdown));
} }
} }
else else
{ {
await MarkFailed(task.Id, task.Number, task.Title, slot, result.ErrorMarkdown, result.TurnCount, await MarkFailed(task.Id, task.Number, task.Title, slot, result.ErrorMarkdown, result.TurnCount,
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason)); resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason, result.ErrorMarkdown));
} }
} }
@@ -276,7 +277,7 @@ public sealed class TaskRunner
else else
{ {
await MarkFailed(taskId, task.Number, task.Title, slot, result.ErrorMarkdown, result.TurnCount, await MarkFailed(taskId, task.Number, task.Title, slot, result.ErrorMarkdown, result.TurnCount,
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason)); resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason, result.ErrorMarkdown));
} }
await _broadcaster.TaskUpdated(taskId); await _broadcaster.TaskUpdated(taskId);
@@ -603,13 +604,24 @@ public sealed class TaskRunner
internal static bool IsWriteTool(string toolName) => WriteTools.Contains(toolName); internal static bool IsWriteTool(string toolName) => WriteTools.Contains(toolName);
/// <summary>Classifies the CLI's raw <c>terminal_reason</c> into the small, MCP-facing enum // The CLI has no dedicated terminal_reason for a session/account usage-limit stop — it comes
/// (<c>max_turns|timeout|error</c>) get_task/batch_get_tasks report as failureReason. // back as a plain "api_error" with the provider's own text (session/rate limit + reset time)
// riding in the result/error text (see ClaudeProcess.RunAsync). Matched against the rendered
// failure message (BuildFailureMarkdown's output), never against Status alone.
private static readonly Regex UsageLimitPattern = new(
"usage limit|session limit|rate.?limit", RegexOptions.IgnoreCase | RegexOptions.Compiled);
internal static bool IsUsageLimitMessage(string? text) => !string.IsNullOrEmpty(text) && UsageLimitPattern.IsMatch(text);
/// <summary>Classifies the CLI's raw <c>terminal_reason</c> (plus, for "api_error", the
/// rendered failure message) into the small, MCP-facing enum
/// (<c>max_turns|timeout|usage_limit|error</c>) get_task/batch_get_tasks report as failureReason.
/// "cancelled" is set explicitly at the call sites that know it (there's no CLI signal for it).</summary> /// "cancelled" is set explicitly at the call sites that know it (there's no CLI signal for it).</summary>
internal static string ClassifyFailureReason(string? terminalReason) => terminalReason switch internal static string ClassifyFailureReason(string? terminalReason, string? messageText = null) => terminalReason switch
{ {
"max_turns" => "max_turns", "max_turns" => "max_turns",
"timeout" => "timeout", "timeout" => "timeout",
"api_error" when IsUsageLimitMessage(messageText) => "usage_limit",
_ => "error", _ => "error",
}; };
@@ -77,8 +77,10 @@ public sealed class AddSubtaskToolTests : IDisposable
var picker = new ClaudeDo.Worker.Queue.QueuePicker(dbFactory); var picker = new ClaudeDo.Worker.Queue.QueuePicker(dbFactory);
var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance); var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance);
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels); var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels);
var usageState = new UsageState();
var queue = new QueueService(dbFactory, runner, cfg, NullLogger<QueueService>.Instance, waker, picker, overrideSlot, state, runCancels, var queue = new QueueService(dbFactory, runner, cfg, NullLogger<QueueService>.Instance, waker, picker, overrideSlot, state, runCancels,
new FakeUsageGate(), new UsageState(), broadcaster); new FakeUsageGate(), usageState, broadcaster,
new UsageLimitAutoContinueCoordinator(dbFactory, usageState, overrideSlot, broadcaster, NullLogger<UsageLimitAutoContinueCoordinator>.Instance));
var maintenance = new WorktreeMaintenanceService(dbFactory, git, NullLogger<WorktreeMaintenanceService>.Instance); var maintenance = new WorktreeMaintenanceService(dbFactory, git, NullLogger<WorktreeMaintenanceService>.Instance);
var merge = new TaskMergeService(dbFactory, git, broadcaster, state, new VerifyCommandRunner(), NullLogger<TaskMergeService>.Instance); var merge = new TaskMergeService(dbFactory, git, broadcaster, state, new VerifyCommandRunner(), NullLogger<TaskMergeService>.Instance);
var aggregator = new PlanningAggregator(dbFactory, git, NullLogger<PlanningAggregator>.Instance); var aggregator = new PlanningAggregator(dbFactory, git, NullLogger<PlanningAggregator>.Instance);
+3 -1
View File
@@ -111,9 +111,11 @@ public sealed class BatchMcpToolsTests : IDisposable
NullLogger<TaskRunner>.Instance, state, new TaskRunTokenRegistry(), new AttachmentStore(), new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader()); NullLogger<TaskRunner>.Instance, state, new TaskRunTokenRegistry(), new AttachmentStore(), new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader());
var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance); var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance);
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels); var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels);
var usageState = new UsageState();
return new QueueService(dbFactory, runner, cfg, NullLogger<QueueService>.Instance, return new QueueService(dbFactory, runner, cfg, NullLogger<QueueService>.Instance,
new QueueWaker(), new QueuePicker(dbFactory), overrideSlot, state, runCancels, new QueueWaker(), new QueuePicker(dbFactory), overrideSlot, state, runCancels,
new FakeUsageGate(), new UsageState(), broadcaster); new FakeUsageGate(), usageState, broadcaster,
new UsageLimitAutoContinueCoordinator(dbFactory, usageState, overrideSlot, broadcaster, NullLogger<UsageLimitAutoContinueCoordinator>.Instance));
} }
[Fact] [Fact]
@@ -178,8 +178,10 @@ public sealed class ExternalMcpServiceTests : IDisposable
var picker = new ClaudeDo.Worker.Queue.QueuePicker(dbFactory); var picker = new ClaudeDo.Worker.Queue.QueuePicker(dbFactory);
var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance); var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance);
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels); var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels);
var usageState = new UsageState();
return new QueueService(dbFactory, runner, cfg, NullLogger<QueueService>.Instance, waker, picker, overrideSlot, state, runCancels, return new QueueService(dbFactory, runner, cfg, NullLogger<QueueService>.Instance, waker, picker, overrideSlot, state, runCancels,
new FakeUsageGate(), new UsageState(), broadcaster); new FakeUsageGate(), usageState, broadcaster,
new UsageLimitAutoContinueCoordinator(dbFactory, usageState, overrideSlot, broadcaster, NullLogger<UsageLimitAutoContinueCoordinator>.Instance));
} }
[Fact] [Fact]
@@ -83,9 +83,11 @@ public sealed class LifecycleMcpToolsTests : IDisposable
NullLogger<TaskRunner>.Instance, state, new TaskRunTokenRegistry(), new AttachmentStore(), new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader()); NullLogger<TaskRunner>.Instance, state, new TaskRunTokenRegistry(), new AttachmentStore(), new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader());
var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance); var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance);
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels); var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels);
var usageState = new UsageState();
return new QueueService(dbFactory, runner, cfg, NullLogger<QueueService>.Instance, return new QueueService(dbFactory, runner, cfg, NullLogger<QueueService>.Instance,
new QueueWaker(), new QueuePicker(dbFactory), overrideSlot, state, runCancels, new QueueWaker(), new QueuePicker(dbFactory), overrideSlot, state, runCancels,
new FakeUsageGate(), new UsageState(), broadcaster); new FakeUsageGate(), usageState, broadcaster,
new UsageLimitAutoContinueCoordinator(dbFactory, usageState, overrideSlot, broadcaster, NullLogger<UsageLimitAutoContinueCoordinator>.Instance));
} }
private async Task<TaskEntity> SeedTaskAsync( private async Task<TaskEntity> SeedTaskAsync(
@@ -60,8 +60,10 @@ public sealed class QueueStateMcpToolsTests : IDisposable
new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader()); new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader());
var picker = new QueuePicker(dbFactory); var picker = new QueuePicker(dbFactory);
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels); var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels);
var resolvedUsageState = usageState ?? new UsageState();
var queue = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, new QueueWaker(), picker, var queue = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, new QueueWaker(), picker,
overrideSlot, built.State, built.RunCancels, new FakeUsageGate(), usageState ?? new UsageState(), broadcaster); overrideSlot, built.State, built.RunCancels, new FakeUsageGate(), resolvedUsageState, broadcaster,
new UsageLimitAutoContinueCoordinator(dbFactory, resolvedUsageState, overrideSlot, broadcaster, NullLogger<UsageLimitAutoContinueCoordinator>.Instance));
return (queue, new QueueStateMcpTools(queue, dbFactory)); return (queue, new QueueStateMcpTools(queue, dbFactory));
} }
@@ -0,0 +1,207 @@
using ClaudeDo.Data;
using ClaudeDo.Data.Models;
using ClaudeDo.Data.Repositories;
using ClaudeDo.Worker.Config;
using ClaudeDo.Worker.Hub;
using ClaudeDo.Worker.Queue;
using ClaudeDo.Worker.Runner;
using ClaudeDo.Worker.Tests.Infrastructure;
using ClaudeDo.Worker.Usage;
using Microsoft.Extensions.Logging.Abstractions;
using TaskStatus = ClaudeDo.Data.Models.TaskStatus;
namespace ClaudeDo.Worker.Tests.Queue;
/// Covers the "Continue on session limit reset" toggle's coordinator in isolation from
/// QueueService's own timer plumbing (see QueueService.ScheduleResetWake for that part).
public sealed class UsageLimitAutoContinueCoordinatorTests : IDisposable
{
private readonly DbFixture _db = new();
private readonly string _tempDir;
private readonly WorkerConfig _cfg;
public UsageLimitAutoContinueCoordinatorTests()
{
_tempDir = Path.Combine(Path.GetTempPath(), $"cd_usagelimit_{Guid.NewGuid():N}");
Directory.CreateDirectory(_tempDir);
_cfg = new WorkerConfig { SandboxRoot = _tempDir, LogRoot = _tempDir };
}
public void Dispose() { _db.Dispose(); try { Directory.Delete(_tempDir, true); } catch { } }
private (UsageLimitAutoContinueCoordinator Coordinator, UsageState UsageState, FakeClaudeProcess Process) BuildCoordinator()
{
var dbFactory = _db.CreateFactory();
var state = TaskStateServiceBuilder.Build(dbFactory).State;
var wt = new WorktreeManager(new ClaudeDo.Data.Git.GitService(), dbFactory, _cfg, NullLogger<WorktreeManager>.Instance);
var fake = new FakeClaudeProcess();
var runner = new TaskRunner(fake, dbFactory, new HubBroadcaster(new CapturingHubContext()), wt,
new ClaudeArgsBuilder(), _cfg, NullLogger<TaskRunner>.Instance, state, new TaskRunTokenRegistry(),
new AttachmentStore(), new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader());
var runCancels = new RunCancellationRegistry(NullLogger<RunCancellationRegistry>.Instance);
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, runCancels);
var usageState = new UsageState();
var coordinator = new UsageLimitAutoContinueCoordinator(
dbFactory, usageState, overrideSlot,
new HubBroadcaster(new CapturingHubContext()), NullLogger<UsageLimitAutoContinueCoordinator>.Instance);
return (coordinator, usageState, fake);
}
private async Task<string> SeedUsageLimitTaskAsync(bool withSessionId = true)
{
var listId = Guid.NewGuid().ToString();
var taskId = Guid.NewGuid().ToString();
using (var ctx = _db.CreateContext())
{
ctx.Lists.Add(new ListEntity { Id = listId, Name = "L", CreatedAt = DateTime.UtcNow });
ctx.Tasks.Add(new TaskEntity
{
Id = taskId, ListId = listId, Title = "T", Status = TaskStatus.Failed,
FailureReason = "usage_limit", CreatedAt = DateTime.UtcNow, FinishedAt = DateTime.UtcNow,
});
if (withSessionId)
{
ctx.TaskRuns.Add(new TaskRunEntity
{
Id = Guid.NewGuid().ToString(), TaskId = taskId, RunNumber = 1, IsRetry = false, Prompt = "p",
LogPath = Path.Combine(_tempDir, "log.ndjson"), StartedAt = DateTime.UtcNow,
FinishedAt = DateTime.UtcNow, SessionId = "sess-1", ExitCode = 1,
});
}
await ctx.SaveChangesAsync();
}
return taskId;
}
private async Task EnableToggleAsync()
{
using var ctx = _db.CreateContext();
var repo = new AppSettingsRepository(ctx);
var settings = await repo.GetAsync();
settings.AutoContinueOnUsageLimit = true;
await repo.UpdateAsync(settings);
}
private async Task<TaskEntity?> PollUntilCalledAsync(string taskId, FakeClaudeProcess fake)
{
var deadline = DateTime.UtcNow.AddSeconds(10);
while (DateTime.UtcNow < deadline && fake.CallCount == 0)
await Task.Delay(25);
await Task.Delay(150); // let the fire-and-forget continuation fully settle
using var ctx = _db.CreateContext();
return await new TaskRepository(ctx).GetByIdAsync(taskId);
}
[Fact]
public async Task Toggle_Off_Never_Schedules_Or_Continues()
{
var (coordinator, usageState, fake) = BuildCoordinator();
var taskId = await SeedUsageLimitTaskAsync();
usageState.ReportSuccess(new UsageSnapshot(
new UsageBucket(85, DateTimeOffset.UtcNow.AddSeconds(-1)), null, Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
var wakeAt = await coordinator.GetScheduledWakeAtAsync(default);
await coordinator.RunAsync(default);
await Task.Delay(150);
Assert.Null(wakeAt);
Assert.Equal(0, fake.CallCount);
using var verify = _db.CreateContext();
var task = await new TaskRepository(verify).GetByIdAsync(taskId);
Assert.Null(task!.UsageLimitAutoContinuedAt);
}
[Fact]
public async Task Toggle_On_No_Snapshot_Yet_Schedules_Nothing_And_Does_Not_Crash()
{
var (coordinator, _, fake) = BuildCoordinator();
await SeedUsageLimitTaskAsync();
await EnableToggleAsync();
var wakeAt = await coordinator.GetScheduledWakeAtAsync(default);
await coordinator.RunAsync(default); // must not throw despite no usage snapshot
Assert.Null(wakeAt);
Assert.Equal(0, fake.CallCount);
}
[Fact]
public async Task Toggle_On_Reset_In_Future_Schedules_Wake_But_Does_Not_Continue_Yet()
{
var (coordinator, usageState, fake) = BuildCoordinator();
await SeedUsageLimitTaskAsync();
await EnableToggleAsync();
var resetsAt = DateTimeOffset.UtcNow.AddMinutes(5);
usageState.ReportSuccess(new UsageSnapshot(
new UsageBucket(85, resetsAt), null, Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
var wakeAt = await coordinator.GetScheduledWakeAtAsync(default);
await coordinator.RunAsync(default);
await Task.Delay(150);
Assert.Equal(resetsAt, wakeAt);
Assert.Equal(0, fake.CallCount);
}
[Fact]
public async Task Toggle_On_Reset_Already_Passed_Continues_Exactly_Once()
{
var (coordinator, usageState, fake) = BuildCoordinator();
var taskId = await SeedUsageLimitTaskAsync();
await EnableToggleAsync();
usageState.ReportSuccess(new UsageSnapshot(
new UsageBucket(85, DateTimeOffset.UtcNow.AddSeconds(-1)), null, Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
await coordinator.RunAsync(default);
var task = await PollUntilCalledAsync(taskId, fake);
Assert.Equal(1, fake.CallCount);
Assert.NotNull(task!.UsageLimitAutoContinuedAt);
// A second tick (e.g. the next backstop) must not fire it again.
await coordinator.RunAsync(default);
await Task.Delay(150);
Assert.Equal(1, fake.CallCount);
}
[Fact]
public async Task Missing_Reset_Time_Never_Continues_And_Never_Throws()
{
var (coordinator, usageState, fake) = BuildCoordinator();
await SeedUsageLimitTaskAsync();
await EnableToggleAsync();
// A snapshot exists, but this bucket has no reset time — fail open.
usageState.ReportSuccess(new UsageSnapshot(
new UsageBucket(85, null), null, Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
var wakeAt = await coordinator.GetScheduledWakeAtAsync(default);
await coordinator.RunAsync(default);
await Task.Delay(150);
Assert.Null(wakeAt);
Assert.Equal(0, fake.CallCount);
}
[Fact]
public async Task Non_UsageLimit_Failure_Is_Never_Touched()
{
var (coordinator, usageState, fake) = BuildCoordinator();
var listId = Guid.NewGuid().ToString();
var taskId = Guid.NewGuid().ToString();
using (var ctx = _db.CreateContext())
{
ctx.Lists.Add(new ListEntity { Id = listId, Name = "L", CreatedAt = DateTime.UtcNow });
ctx.Tasks.Add(new TaskEntity { Id = taskId, ListId = listId, Title = "T", Status = TaskStatus.Failed,
FailureReason = "error", CreatedAt = DateTime.UtcNow });
await ctx.SaveChangesAsync();
}
await EnableToggleAsync();
usageState.ReportSuccess(new UsageSnapshot(
new UsageBucket(85, DateTimeOffset.UtcNow.AddSeconds(-1)), null, Array.Empty<UsageLimitRow>(), DateTime.UtcNow));
await coordinator.RunAsync(default);
await Task.Delay(150);
Assert.Equal(0, fake.CallCount);
}
}
@@ -99,6 +99,29 @@ public class FailureDiagnosisTests
{ {
Assert.Equal(expected, TaskRunner.ClassifyFailureReason(terminalReason)); Assert.Equal(expected, TaskRunner.ClassifyFailureReason(terminalReason));
} }
[Theory]
[InlineData("You've hit your session limit resets 1pm (Europe/Berlin)")]
[InlineData("Claude usage limit reached, resets at 3pm")]
[InlineData("rate limited, please retry later")]
public void ClassifyFailureReason_ApiError_With_Limit_Text_Returns_UsageLimit(string messageText)
{
Assert.Equal("usage_limit", TaskRunner.ClassifyFailureReason("api_error", messageText));
}
[Fact]
public void ClassifyFailureReason_ApiError_With_Generic_Text_Stays_Error()
{
Assert.Equal("error", TaskRunner.ClassifyFailureReason("api_error", "Internal server error, please retry"));
}
[Theory]
[InlineData("max_turns")]
[InlineData("timeout")]
public void ClassifyFailureReason_Ignores_Limit_Text_For_NonApiError_Reasons(string terminalReason)
{
Assert.Equal(terminalReason, TaskRunner.ClassifyFailureReason(terminalReason, "usage limit reached"));
}
} }
public sealed class FailureDiagnosisEndToEndTests : IDisposable public sealed class FailureDiagnosisEndToEndTests : IDisposable
@@ -282,4 +305,32 @@ public sealed class FailureDiagnosisEndToEndTests : IDisposable
Assert.Equal("error", task.FailureReason); Assert.Equal("error", task.FailureReason);
Assert.NotEqual("max_turns", task.FailureReason); Assert.NotEqual("max_turns", task.FailureReason);
} }
[Fact]
public async Task Usage_Limit_Run_Sets_FailureReason_UsageLimit()
{
var dbFactory = _db.CreateFactory();
using (var ctx = _db.CreateContext())
{
ctx.Lists.Add(new ListEntity { Id = "l1", Name = "L", WorkingDir = null, CreatedAt = DateTime.UtcNow });
ctx.Tasks.Add(new TaskEntity { Id = "t1", ListId = "l1", Title = "T",
Status = TaskStatus.Running, CreatedAt = DateTime.UtcNow });
await ctx.SaveChangesAsync();
}
var fake = new FakeClaudeProcess((_, _, _, _, _) => Task.FromResult(new RunResult
{
ExitCode = 1,
TerminalReason = "api_error",
ResultMarkdown = "You've hit your session limit resets 1pm (Europe/Berlin)",
}));
var runner = MakeRunner(dbFactory, fake);
using (var ctx = _db.CreateContext())
await runner.RunAsync((await new TaskRepository(ctx).GetByIdAsync("t1"))!, "slot-1", default, alreadyClaimed: true);
using var verify = _db.CreateContext();
var task = await new TaskRepository(verify).GetByIdAsync("t1");
Assert.Equal(TaskStatus.Failed, task!.Status);
Assert.Equal("usage_limit", task.FailureReason);
}
} }
@@ -65,8 +65,10 @@ public sealed class QueueServiceSlotFailureTests : IDisposable
new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader()); new FakeSessionSkillSeeder(), new FakeTranscriptUsageReader());
var waker = new QueueWaker(); var waker = new QueueWaker();
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels); var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels);
var usageState = new UsageState();
var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, waker, picker, var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, waker, picker,
overrideSlot, built.State, built.RunCancels, new FakeUsageGate(), new UsageState(), broadcaster); overrideSlot, built.State, built.RunCancels, new FakeUsageGate(), usageState, broadcaster,
new UsageLimitAutoContinueCoordinator(dbFactory, usageState, overrideSlot, broadcaster, NullLogger<UsageLimitAutoContinueCoordinator>.Instance));
return (service, built.Hub, waker); return (service, built.Hub, waker);
} }
@@ -60,8 +60,10 @@ public sealed class QueueServiceSlotGuardTests : IDisposable
_waker = new QueueWaker(); _waker = new QueueWaker();
var picker = new QueuePicker(dbFactory); var picker = new QueuePicker(dbFactory);
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels); var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels);
var usageState = new UsageState();
var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, _waker, picker, overrideSlot, state, built.RunCancels, var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, _waker, picker, overrideSlot, state, built.RunCancels,
new FakeUsageGate(), new UsageState(), broadcaster); new FakeUsageGate(), usageState, broadcaster,
new UsageLimitAutoContinueCoordinator(dbFactory, usageState, overrideSlot, broadcaster, NullLogger<UsageLimitAutoContinueCoordinator>.Instance));
return (service, fake); return (service, fake);
} }
@@ -69,8 +69,10 @@ public sealed class QueueServiceTests : IDisposable
var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels); var overrideSlot = new OverrideSlotService(dbFactory, runner, NullLogger<OverrideSlotService>.Instance, built.RunCancels);
_usageGate = usageGate ?? new FakeUsageGate(); _usageGate = usageGate ?? new FakeUsageGate();
_runCancels = built.RunCancels; _runCancels = built.RunCancels;
var resolvedUsageState = usageState ?? new UsageState();
var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, _waker, picker, overrideSlot, state, built.RunCancels, var service = new QueueService(dbFactory, runner, _cfg, NullLogger<QueueService>.Instance, _waker, picker, overrideSlot, state, built.RunCancels,
_usageGate, usageState ?? new UsageState(), broadcaster); _usageGate, resolvedUsageState, broadcaster,
new UsageLimitAutoContinueCoordinator(dbFactory, resolvedUsageState, overrideSlot, broadcaster, NullLogger<UsageLimitAutoContinueCoordinator>.Instance));
return (service, fake); return (service, fake);
} }