feat(worker): Toggle "Continue on session limit reset" für Usage-Limit-Abbrüche

Klassifiziert einen echten Usage-Limit-Abbruch als eigene FailureReason
"usage_limit" (TaskRunner.ClassifyFailureReason: nur bei terminal_reason
"api_error" plus einem Limit-Muster im gerenderten Fehlertext, nicht an
Status==Failed allein). Neuer Toggle AutoContinueOnUsageLimit (app_settings,
Default aus) unter Settings → General → "Usage limit stop":

- UsageLimitAutoContinueCoordinator feuert pro Task genau einmal ContinueTask
  über OverrideSlotService, sobald das 5h-Fenster (UsageState.Snapshot.FiveHour
  .ResetsAt) tatsächlich zurückgesetzt ist; ein persistenter Marker
  (TaskEntity.UsageLimitAutoContinuedAt) verhindert einen zweiten Anlauf bei
  einem erneuten Limit-Treffer.
- QueueService schedult zusätzlich einen exakten Wake-Timer auf den
  Reset-Zeitpunkt, statt nur auf den 30s-Backstop zu warten.
- Fail-open durchgängig: kein Snapshot/keine Reset-Zeit → kein Timer, kein
  Continue, kein Throw. Toggle aus ändert das heutige Verhalten nicht.

Migration AddUsageLimitAutoContinue fügt beide Spalten hinzu; die von
`dotnet ef migrations add` mitgescaffoldete leere UpdateData auf app_settings
(columns/values: []) erzeugte ungültiges SQL ("near WHERE") und wurde entfernt
— TaskNumberMigrationTests deckte das über den vollen Migrate()-Pfad auf.
This commit is contained in:
mika kuns
2026-08-21 18:43:02 +02:00
parent 290dd1b614
commit 07dd75700d
30 changed files with 1505 additions and 19 deletions
+5 -2
View File
@@ -55,7 +55,8 @@ public record AppSettingsDto(
int UsageThrottleFiveHourSoftPct = 50,
int UsageThrottleFiveHourHardPct = 65,
int UsageThrottleSevenDaySoftPct = 50,
int UsageThrottleSevenDayHardPct = 65);
int UsageThrottleSevenDayHardPct = 65,
bool AutoContinueOnUsageLimit = false);
// Per-model run defaults (effort + turn budget) edited in Settings -> General.
public record ModelPresetDto(string Model, string Effort, int MaxTurns);
@@ -471,7 +472,8 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
row.UsageThrottleFiveHourSoftPct,
row.UsageThrottleFiveHourHardPct,
row.UsageThrottleSevenDaySoftPct,
row.UsageThrottleSevenDayHardPct);
row.UsageThrottleSevenDayHardPct,
row.AutoContinueOnUsageLimit);
}
public async Task UpdateAppSettings(AppSettingsDto dto)
@@ -506,6 +508,7 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
UsageThrottleFiveHourHardPct = dto.UsageThrottleFiveHourHardPct,
UsageThrottleSevenDaySoftPct = dto.UsageThrottleSevenDaySoftPct,
UsageThrottleSevenDayHardPct = dto.UsageThrottleSevenDayHardPct,
AutoContinueOnUsageLimit = dto.AutoContinueOnUsageLimit,
});
}
+3
View File
@@ -156,6 +156,9 @@ builder.Services.AddHostedService<PrimeScheduler>();
builder.Services.AddSingleton<IRefineRunner, RefineRunner>();
builder.Services.AddSingleton<IRefineBroadcaster>(sp => sp.GetRequiredService<HubBroadcaster>());
// "Continue on session limit reset" toggle — depends on UsageState, registered below.
builder.Services.AddSingleton<UsageLimitAutoContinueCoordinator>();
// QueueService: singleton + hosted service (same instance).
builder.Services.AddSingleton<QueueService>();
builder.Services.AddHostedService(sp => sp.GetRequiredService<QueueService>());
+35 -1
View File
@@ -26,11 +26,13 @@ public sealed class QueueService : BackgroundService
private readonly IUsageGate _usageGate;
private readonly UsageState _usageState;
private readonly HubBroadcaster _broadcaster;
private readonly UsageLimitAutoContinueCoordinator _usageLimitAutoContinue;
private readonly object _lock = new();
private readonly Dictionary<string, QueueSlotState> _queueSlots = new();
private bool _usageGateBlocked;
private int? _lastEffectiveSlots;
private DateTimeOffset? _scheduledResetWakeAt;
public QueueService(
IDbContextFactory<ClaudeDoDbContext> dbFactory,
@@ -44,7 +46,8 @@ public sealed class QueueService : BackgroundService
RunCancellationRegistry runCancels,
IUsageGate usageGate,
UsageState usageState,
HubBroadcaster broadcaster)
HubBroadcaster broadcaster,
UsageLimitAutoContinueCoordinator usageLimitAutoContinue)
{
_dbFactory = dbFactory;
_runner = runner;
@@ -58,6 +61,7 @@ public sealed class QueueService : BackgroundService
_usageGate = usageGate;
_usageState = usageState;
_broadcaster = broadcaster;
_usageLimitAutoContinue = usageLimitAutoContinue;
}
public IReadOnlyList<(string slot, string taskId, DateTime startedAt)> GetActive()
@@ -130,6 +134,14 @@ public sealed class QueueService : BackgroundService
var gateDecision = await _usageGate.EvaluateAsync(stoppingToken);
await ReportUsageGateTransitionAsync(gateDecision);
// "Continue on session limit reset" toggle: schedule an exact wake for the reset
// time (instead of only relying on the 30s backstop above) and fire any pending
// auto-continues. No-ops entirely while the toggle is off or no reset time is known.
var scheduledWakeAt = await _usageLimitAutoContinue.GetScheduledWakeAtAsync(stoppingToken);
if (scheduledWakeAt is not null)
ScheduleResetWake(scheduledWakeAt.Value, stoppingToken);
await _usageLimitAutoContinue.RunAsync(stoppingToken);
// Only queue refill is gated. Runs already in flight (RunNow, ContinueTask,
// interactive sessions, planning, daily prep) keep going regardless.
if (!gateDecision.IsBlocked)
@@ -210,6 +222,28 @@ public sealed class QueueService : BackgroundService
return false;
}
// Wakes the queue loop exactly at the usage-limit reset time instead of leaving it to the
// (up to 30s late) backstop — schedules once per distinct target and is a no-op for a target
// already scheduled or already in the past (this tick's own check already covers "now").
private void ScheduleResetWake(DateTimeOffset resetsAt, CancellationToken ct)
{
if (_scheduledResetWakeAt == resetsAt) return;
_scheduledResetWakeAt = resetsAt;
var delay = resetsAt - DateTimeOffset.UtcNow;
if (delay <= TimeSpan.Zero) return;
_ = Task.Run(async () =>
{
try
{
await Task.Delay(delay, ct);
_waker.Wake();
}
catch (OperationCanceledException) { }
}, ct);
}
private async Task ReportUsageGateTransitionAsync(UsageGateDecision decision)
{
if (decision.IsBlocked == _usageGateBlocked) return;
@@ -0,0 +1,99 @@
using ClaudeDo.Data;
using ClaudeDo.Data.Models;
using ClaudeDo.Data.Repositories;
using ClaudeDo.Worker.Hub;
using ClaudeDo.Worker.Usage;
using Microsoft.EntityFrameworkCore;
namespace ClaudeDo.Worker.Queue;
/// <summary>
/// The "Continue on session limit reset" toggle (<see cref="AppSettingsEntity.AutoContinueOnUsageLimit"/>).
/// Off by default and fails open on every missing signal — no snapshot, no reset time — never a
/// timer, never a crash, never a change from today's behaviour. Called once per
/// <see cref="QueueService"/> tick (both from its wake and its 30s backstop):
/// <see cref="GetScheduledWakeAtAsync"/> tells the caller when to schedule an exact wake so a
/// gate-blocked queue doesn't wait out the backstop, and <see cref="RunAsync"/> fires exactly one
/// <c>ContinueTask</c> per eligible task once the window has actually reset.
/// </summary>
public sealed class UsageLimitAutoContinueCoordinator
{
public const string ContinuePrompt =
"The Anthropic usage limit window that stopped this run has since reset. Continue the task from where the previous run left off.";
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
private readonly UsageState _usageState;
private readonly OverrideSlotService _override;
private readonly HubBroadcaster _broadcaster;
private readonly ILogger<UsageLimitAutoContinueCoordinator> _logger;
public UsageLimitAutoContinueCoordinator(
IDbContextFactory<ClaudeDoDbContext> dbFactory, UsageState usageState, OverrideSlotService overrideSlot,
HubBroadcaster broadcaster, ILogger<UsageLimitAutoContinueCoordinator> logger)
{
_dbFactory = dbFactory;
_usageState = usageState;
_override = overrideSlot;
_broadcaster = broadcaster;
_logger = logger;
}
/// <summary>The reset time <see cref="QueueService"/> should schedule an exact wake for, or
/// null when the toggle is off or no reset time is known yet — fail open, no timer.</summary>
public async Task<DateTimeOffset?> GetScheduledWakeAtAsync(CancellationToken ct)
{
var settings = await TryReadSettingsAsync(ct);
if (settings is null || !settings.AutoContinueOnUsageLimit) return null;
return _usageState.Snapshot?.FiveHour?.ResetsAt;
}
/// <summary>Fires exactly one <c>ContinueTask</c> per task failed on the usage limit, once the
/// 5h window has actually reset. No-ops (never throws) when the toggle is off, no reset time
/// is known, or the window hasn't reset yet.</summary>
public async Task RunAsync(CancellationToken ct)
{
var settings = await TryReadSettingsAsync(ct);
if (settings is null || !settings.AutoContinueOnUsageLimit) return;
var resetsAt = _usageState.Snapshot?.FiveHour?.ResetsAt;
if (resetsAt is null || DateTimeOffset.UtcNow < resetsAt.Value) return;
List<TaskEntity> candidates;
using (var context = _dbFactory.CreateDbContext())
candidates = await new TaskRepository(context).GetUsageLimitAutoContinueCandidatesAsync(ct);
foreach (var task in candidates)
{
bool claimed;
using (var context = _dbFactory.CreateDbContext())
claimed = await new TaskRepository(context).TryClaimUsageLimitAutoContinueAsync(task.Id, DateTime.UtcNow, ct);
if (!claimed) continue; // already fired for this task — never twice.
try
{
await _override.ContinueTask(task.Id, ContinuePrompt);
await _broadcaster.WorkerLog(
$"Auto-continuing #{task.Number} \"{task.Title}\" after usage-limit reset",
WorkerLogLevel.Info, DateTime.UtcNow);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to auto-continue task {TaskId} after usage-limit reset", task.Id);
}
}
}
private async Task<AppSettingsEntity?> TryReadSettingsAsync(CancellationToken ct)
{
try
{
using var context = _dbFactory.CreateDbContext();
return await new AppSettingsRepository(context).GetAsync(ct);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "UsageLimitAutoContinueCoordinator: failed to read settings; skipping tick");
return null;
}
}
}
+18 -6
View File
@@ -1,4 +1,5 @@
using System.Text.Json;
using System.Text.RegularExpressions;
using ClaudeDo.Data;
using ClaudeDo.Data.Models;
using ClaudeDo.Data.Repositories;
@@ -159,13 +160,13 @@ public sealed class TaskRunner
else
{
await MarkFailed(task.Id, task.Number, task.Title, slot, retryResult.ErrorMarkdown, retryResult.TurnCount,
retryConfig.MaxTurns, ClassifyFailureReason(retryResult.TerminalReason));
retryConfig.MaxTurns, ClassifyFailureReason(retryResult.TerminalReason, retryResult.ErrorMarkdown));
}
}
else
{
await MarkFailed(task.Id, task.Number, task.Title, slot, result.ErrorMarkdown, result.TurnCount,
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason));
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason, result.ErrorMarkdown));
}
}
@@ -276,7 +277,7 @@ public sealed class TaskRunner
else
{
await MarkFailed(taskId, task.Number, task.Title, slot, result.ErrorMarkdown, result.TurnCount,
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason));
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason, result.ErrorMarkdown));
}
await _broadcaster.TaskUpdated(taskId);
@@ -603,13 +604,24 @@ public sealed class TaskRunner
internal static bool IsWriteTool(string toolName) => WriteTools.Contains(toolName);
/// <summary>Classifies the CLI's raw <c>terminal_reason</c> into the small, MCP-facing enum
/// (<c>max_turns|timeout|error</c>) get_task/batch_get_tasks report as failureReason.
// The CLI has no dedicated terminal_reason for a session/account usage-limit stop — it comes
// back as a plain "api_error" with the provider's own text (session/rate limit + reset time)
// riding in the result/error text (see ClaudeProcess.RunAsync). Matched against the rendered
// failure message (BuildFailureMarkdown's output), never against Status alone.
private static readonly Regex UsageLimitPattern = new(
"usage limit|session limit|rate.?limit", RegexOptions.IgnoreCase | RegexOptions.Compiled);
internal static bool IsUsageLimitMessage(string? text) => !string.IsNullOrEmpty(text) && UsageLimitPattern.IsMatch(text);
/// <summary>Classifies the CLI's raw <c>terminal_reason</c> (plus, for "api_error", the
/// rendered failure message) into the small, MCP-facing enum
/// (<c>max_turns|timeout|usage_limit|error</c>) get_task/batch_get_tasks report as failureReason.
/// "cancelled" is set explicitly at the call sites that know it (there's no CLI signal for it).</summary>
internal static string ClassifyFailureReason(string? terminalReason) => terminalReason switch
internal static string ClassifyFailureReason(string? terminalReason, string? messageText = null) => terminalReason switch
{
"max_turns" => "max_turns",
"timeout" => "timeout",
"api_error" when IsUsageLimitMessage(messageText) => "usage_limit",
_ => "error",
};