feat(worker): Toggle "Continue on session limit reset" für Usage-Limit-Abbrüche

Klassifiziert einen echten Usage-Limit-Abbruch als eigene FailureReason
"usage_limit" (TaskRunner.ClassifyFailureReason: nur bei terminal_reason
"api_error" plus einem Limit-Muster im gerenderten Fehlertext, nicht an
Status==Failed allein). Neuer Toggle AutoContinueOnUsageLimit (app_settings,
Default aus) unter Settings → General → "Usage limit stop":

- UsageLimitAutoContinueCoordinator feuert pro Task genau einmal ContinueTask
  über OverrideSlotService, sobald das 5h-Fenster (UsageState.Snapshot.FiveHour
  .ResetsAt) tatsächlich zurückgesetzt ist; ein persistenter Marker
  (TaskEntity.UsageLimitAutoContinuedAt) verhindert einen zweiten Anlauf bei
  einem erneuten Limit-Treffer.
- QueueService schedult zusätzlich einen exakten Wake-Timer auf den
  Reset-Zeitpunkt, statt nur auf den 30s-Backstop zu warten.
- Fail-open durchgängig: kein Snapshot/keine Reset-Zeit → kein Timer, kein
  Continue, kein Throw. Toggle aus ändert das heutige Verhalten nicht.

Migration AddUsageLimitAutoContinue fügt beide Spalten hinzu; die von
`dotnet ef migrations add` mitgescaffoldete leere UpdateData auf app_settings
(columns/values: []) erzeugte ungültiges SQL ("near WHERE") und wurde entfernt
— TaskNumberMigrationTests deckte das über den vollen Migrate()-Pfad auf.
This commit is contained in:
mika kuns
2026-08-21 18:43:02 +02:00
parent 290dd1b614
commit 07dd75700d
30 changed files with 1505 additions and 19 deletions
+35 -1
View File
@@ -26,11 +26,13 @@ public sealed class QueueService : BackgroundService
private readonly IUsageGate _usageGate;
private readonly UsageState _usageState;
private readonly HubBroadcaster _broadcaster;
private readonly UsageLimitAutoContinueCoordinator _usageLimitAutoContinue;
private readonly object _lock = new();
private readonly Dictionary<string, QueueSlotState> _queueSlots = new();
private bool _usageGateBlocked;
private int? _lastEffectiveSlots;
private DateTimeOffset? _scheduledResetWakeAt;
public QueueService(
IDbContextFactory<ClaudeDoDbContext> dbFactory,
@@ -44,7 +46,8 @@ public sealed class QueueService : BackgroundService
RunCancellationRegistry runCancels,
IUsageGate usageGate,
UsageState usageState,
HubBroadcaster broadcaster)
HubBroadcaster broadcaster,
UsageLimitAutoContinueCoordinator usageLimitAutoContinue)
{
_dbFactory = dbFactory;
_runner = runner;
@@ -58,6 +61,7 @@ public sealed class QueueService : BackgroundService
_usageGate = usageGate;
_usageState = usageState;
_broadcaster = broadcaster;
_usageLimitAutoContinue = usageLimitAutoContinue;
}
public IReadOnlyList<(string slot, string taskId, DateTime startedAt)> GetActive()
@@ -130,6 +134,14 @@ public sealed class QueueService : BackgroundService
var gateDecision = await _usageGate.EvaluateAsync(stoppingToken);
await ReportUsageGateTransitionAsync(gateDecision);
// "Continue on session limit reset" toggle: schedule an exact wake for the reset
// time (instead of only relying on the 30s backstop above) and fire any pending
// auto-continues. No-ops entirely while the toggle is off or no reset time is known.
var scheduledWakeAt = await _usageLimitAutoContinue.GetScheduledWakeAtAsync(stoppingToken);
if (scheduledWakeAt is not null)
ScheduleResetWake(scheduledWakeAt.Value, stoppingToken);
await _usageLimitAutoContinue.RunAsync(stoppingToken);
// Only queue refill is gated. Runs already in flight (RunNow, ContinueTask,
// interactive sessions, planning, daily prep) keep going regardless.
if (!gateDecision.IsBlocked)
@@ -210,6 +222,28 @@ public sealed class QueueService : BackgroundService
return false;
}
// Wakes the queue loop exactly at the usage-limit reset time instead of leaving it to the
// (up to 30s late) backstop — schedules once per distinct target and is a no-op for a target
// already scheduled or already in the past (this tick's own check already covers "now").
private void ScheduleResetWake(DateTimeOffset resetsAt, CancellationToken ct)
{
if (_scheduledResetWakeAt == resetsAt) return;
_scheduledResetWakeAt = resetsAt;
var delay = resetsAt - DateTimeOffset.UtcNow;
if (delay <= TimeSpan.Zero) return;
_ = Task.Run(async () =>
{
try
{
await Task.Delay(delay, ct);
_waker.Wake();
}
catch (OperationCanceledException) { }
}, ct);
}
private async Task ReportUsageGateTransitionAsync(UsageGateDecision decision)
{
if (decision.IsBlocked == _usageGateBlocked) return;
@@ -0,0 +1,99 @@
using ClaudeDo.Data;
using ClaudeDo.Data.Models;
using ClaudeDo.Data.Repositories;
using ClaudeDo.Worker.Hub;
using ClaudeDo.Worker.Usage;
using Microsoft.EntityFrameworkCore;
namespace ClaudeDo.Worker.Queue;
/// <summary>
/// The "Continue on session limit reset" toggle (<see cref="AppSettingsEntity.AutoContinueOnUsageLimit"/>).
/// Off by default and fails open on every missing signal — no snapshot, no reset time — never a
/// timer, never a crash, never a change from today's behaviour. Called once per
/// <see cref="QueueService"/> tick (both from its wake and its 30s backstop):
/// <see cref="GetScheduledWakeAtAsync"/> tells the caller when to schedule an exact wake so a
/// gate-blocked queue doesn't wait out the backstop, and <see cref="RunAsync"/> fires exactly one
/// <c>ContinueTask</c> per eligible task once the window has actually reset.
/// </summary>
public sealed class UsageLimitAutoContinueCoordinator
{
public const string ContinuePrompt =
"The Anthropic usage limit window that stopped this run has since reset. Continue the task from where the previous run left off.";
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
private readonly UsageState _usageState;
private readonly OverrideSlotService _override;
private readonly HubBroadcaster _broadcaster;
private readonly ILogger<UsageLimitAutoContinueCoordinator> _logger;
public UsageLimitAutoContinueCoordinator(
IDbContextFactory<ClaudeDoDbContext> dbFactory, UsageState usageState, OverrideSlotService overrideSlot,
HubBroadcaster broadcaster, ILogger<UsageLimitAutoContinueCoordinator> logger)
{
_dbFactory = dbFactory;
_usageState = usageState;
_override = overrideSlot;
_broadcaster = broadcaster;
_logger = logger;
}
/// <summary>The reset time <see cref="QueueService"/> should schedule an exact wake for, or
/// null when the toggle is off or no reset time is known yet — fail open, no timer.</summary>
public async Task<DateTimeOffset?> GetScheduledWakeAtAsync(CancellationToken ct)
{
var settings = await TryReadSettingsAsync(ct);
if (settings is null || !settings.AutoContinueOnUsageLimit) return null;
return _usageState.Snapshot?.FiveHour?.ResetsAt;
}
/// <summary>Fires exactly one <c>ContinueTask</c> per task failed on the usage limit, once the
/// 5h window has actually reset. No-ops (never throws) when the toggle is off, no reset time
/// is known, or the window hasn't reset yet.</summary>
public async Task RunAsync(CancellationToken ct)
{
var settings = await TryReadSettingsAsync(ct);
if (settings is null || !settings.AutoContinueOnUsageLimit) return;
var resetsAt = _usageState.Snapshot?.FiveHour?.ResetsAt;
if (resetsAt is null || DateTimeOffset.UtcNow < resetsAt.Value) return;
List<TaskEntity> candidates;
using (var context = _dbFactory.CreateDbContext())
candidates = await new TaskRepository(context).GetUsageLimitAutoContinueCandidatesAsync(ct);
foreach (var task in candidates)
{
bool claimed;
using (var context = _dbFactory.CreateDbContext())
claimed = await new TaskRepository(context).TryClaimUsageLimitAutoContinueAsync(task.Id, DateTime.UtcNow, ct);
if (!claimed) continue; // already fired for this task — never twice.
try
{
await _override.ContinueTask(task.Id, ContinuePrompt);
await _broadcaster.WorkerLog(
$"Auto-continuing #{task.Number} \"{task.Title}\" after usage-limit reset",
WorkerLogLevel.Info, DateTime.UtcNow);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to auto-continue task {TaskId} after usage-limit reset", task.Id);
}
}
}
private async Task<AppSettingsEntity?> TryReadSettingsAsync(CancellationToken ct)
{
try
{
using var context = _dbFactory.CreateDbContext();
return await new AppSettingsRepository(context).GetAsync(ct);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "UsageLimitAutoContinueCoordinator: failed to read settings; skipping tick");
return null;
}
}
}