feat(worker): Toggle "Continue on session limit reset" für Usage-Limit-Abbrüche
Klassifiziert einen echten Usage-Limit-Abbruch als eigene FailureReason
"usage_limit" (TaskRunner.ClassifyFailureReason: nur bei terminal_reason
"api_error" plus einem Limit-Muster im gerenderten Fehlertext, nicht an
Status==Failed allein). Neuer Toggle AutoContinueOnUsageLimit (app_settings,
Default aus) unter Settings → General → "Usage limit stop":
- UsageLimitAutoContinueCoordinator feuert pro Task genau einmal ContinueTask
über OverrideSlotService, sobald das 5h-Fenster (UsageState.Snapshot.FiveHour
.ResetsAt) tatsächlich zurückgesetzt ist; ein persistenter Marker
(TaskEntity.UsageLimitAutoContinuedAt) verhindert einen zweiten Anlauf bei
einem erneuten Limit-Treffer.
- QueueService schedult zusätzlich einen exakten Wake-Timer auf den
Reset-Zeitpunkt, statt nur auf den 30s-Backstop zu warten.
- Fail-open durchgängig: kein Snapshot/keine Reset-Zeit → kein Timer, kein
Continue, kein Throw. Toggle aus ändert das heutige Verhalten nicht.
Migration AddUsageLimitAutoContinue fügt beide Spalten hinzu; die von
`dotnet ef migrations add` mitgescaffoldete leere UpdateData auf app_settings
(columns/values: []) erzeugte ungültiges SQL ("near WHERE") und wurde entfernt
— TaskNumberMigrationTests deckte das über den vollen Migrate()-Pfad auf.
This commit is contained in:
@@ -26,11 +26,13 @@ public sealed class QueueService : BackgroundService
|
||||
private readonly IUsageGate _usageGate;
|
||||
private readonly UsageState _usageState;
|
||||
private readonly HubBroadcaster _broadcaster;
|
||||
private readonly UsageLimitAutoContinueCoordinator _usageLimitAutoContinue;
|
||||
|
||||
private readonly object _lock = new();
|
||||
private readonly Dictionary<string, QueueSlotState> _queueSlots = new();
|
||||
private bool _usageGateBlocked;
|
||||
private int? _lastEffectiveSlots;
|
||||
private DateTimeOffset? _scheduledResetWakeAt;
|
||||
|
||||
public QueueService(
|
||||
IDbContextFactory<ClaudeDoDbContext> dbFactory,
|
||||
@@ -44,7 +46,8 @@ public sealed class QueueService : BackgroundService
|
||||
RunCancellationRegistry runCancels,
|
||||
IUsageGate usageGate,
|
||||
UsageState usageState,
|
||||
HubBroadcaster broadcaster)
|
||||
HubBroadcaster broadcaster,
|
||||
UsageLimitAutoContinueCoordinator usageLimitAutoContinue)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_runner = runner;
|
||||
@@ -58,6 +61,7 @@ public sealed class QueueService : BackgroundService
|
||||
_usageGate = usageGate;
|
||||
_usageState = usageState;
|
||||
_broadcaster = broadcaster;
|
||||
_usageLimitAutoContinue = usageLimitAutoContinue;
|
||||
}
|
||||
|
||||
public IReadOnlyList<(string slot, string taskId, DateTime startedAt)> GetActive()
|
||||
@@ -130,6 +134,14 @@ public sealed class QueueService : BackgroundService
|
||||
var gateDecision = await _usageGate.EvaluateAsync(stoppingToken);
|
||||
await ReportUsageGateTransitionAsync(gateDecision);
|
||||
|
||||
// "Continue on session limit reset" toggle: schedule an exact wake for the reset
|
||||
// time (instead of only relying on the 30s backstop above) and fire any pending
|
||||
// auto-continues. No-ops entirely while the toggle is off or no reset time is known.
|
||||
var scheduledWakeAt = await _usageLimitAutoContinue.GetScheduledWakeAtAsync(stoppingToken);
|
||||
if (scheduledWakeAt is not null)
|
||||
ScheduleResetWake(scheduledWakeAt.Value, stoppingToken);
|
||||
await _usageLimitAutoContinue.RunAsync(stoppingToken);
|
||||
|
||||
// Only queue refill is gated. Runs already in flight (RunNow, ContinueTask,
|
||||
// interactive sessions, planning, daily prep) keep going regardless.
|
||||
if (!gateDecision.IsBlocked)
|
||||
@@ -210,6 +222,28 @@ public sealed class QueueService : BackgroundService
|
||||
return false;
|
||||
}
|
||||
|
||||
// Wakes the queue loop exactly at the usage-limit reset time instead of leaving it to the
|
||||
// (up to 30s late) backstop — schedules once per distinct target and is a no-op for a target
|
||||
// already scheduled or already in the past (this tick's own check already covers "now").
|
||||
private void ScheduleResetWake(DateTimeOffset resetsAt, CancellationToken ct)
|
||||
{
|
||||
if (_scheduledResetWakeAt == resetsAt) return;
|
||||
_scheduledResetWakeAt = resetsAt;
|
||||
|
||||
var delay = resetsAt - DateTimeOffset.UtcNow;
|
||||
if (delay <= TimeSpan.Zero) return;
|
||||
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(delay, ct);
|
||||
_waker.Wake();
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
}, ct);
|
||||
}
|
||||
|
||||
private async Task ReportUsageGateTransitionAsync(UsageGateDecision decision)
|
||||
{
|
||||
if (decision.IsBlocked == _usageGateBlocked) return;
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
using ClaudeDo.Data;
|
||||
using ClaudeDo.Data.Models;
|
||||
using ClaudeDo.Data.Repositories;
|
||||
using ClaudeDo.Worker.Hub;
|
||||
using ClaudeDo.Worker.Usage;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ClaudeDo.Worker.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// The "Continue on session limit reset" toggle (<see cref="AppSettingsEntity.AutoContinueOnUsageLimit"/>).
|
||||
/// Off by default and fails open on every missing signal — no snapshot, no reset time — never a
|
||||
/// timer, never a crash, never a change from today's behaviour. Called once per
|
||||
/// <see cref="QueueService"/> tick (both from its wake and its 30s backstop):
|
||||
/// <see cref="GetScheduledWakeAtAsync"/> tells the caller when to schedule an exact wake so a
|
||||
/// gate-blocked queue doesn't wait out the backstop, and <see cref="RunAsync"/> fires exactly one
|
||||
/// <c>ContinueTask</c> per eligible task once the window has actually reset.
|
||||
/// </summary>
|
||||
public sealed class UsageLimitAutoContinueCoordinator
|
||||
{
|
||||
public const string ContinuePrompt =
|
||||
"The Anthropic usage limit window that stopped this run has since reset. Continue the task from where the previous run left off.";
|
||||
|
||||
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||
private readonly UsageState _usageState;
|
||||
private readonly OverrideSlotService _override;
|
||||
private readonly HubBroadcaster _broadcaster;
|
||||
private readonly ILogger<UsageLimitAutoContinueCoordinator> _logger;
|
||||
|
||||
public UsageLimitAutoContinueCoordinator(
|
||||
IDbContextFactory<ClaudeDoDbContext> dbFactory, UsageState usageState, OverrideSlotService overrideSlot,
|
||||
HubBroadcaster broadcaster, ILogger<UsageLimitAutoContinueCoordinator> logger)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_usageState = usageState;
|
||||
_override = overrideSlot;
|
||||
_broadcaster = broadcaster;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>The reset time <see cref="QueueService"/> should schedule an exact wake for, or
|
||||
/// null when the toggle is off or no reset time is known yet — fail open, no timer.</summary>
|
||||
public async Task<DateTimeOffset?> GetScheduledWakeAtAsync(CancellationToken ct)
|
||||
{
|
||||
var settings = await TryReadSettingsAsync(ct);
|
||||
if (settings is null || !settings.AutoContinueOnUsageLimit) return null;
|
||||
return _usageState.Snapshot?.FiveHour?.ResetsAt;
|
||||
}
|
||||
|
||||
/// <summary>Fires exactly one <c>ContinueTask</c> per task failed on the usage limit, once the
|
||||
/// 5h window has actually reset. No-ops (never throws) when the toggle is off, no reset time
|
||||
/// is known, or the window hasn't reset yet.</summary>
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var settings = await TryReadSettingsAsync(ct);
|
||||
if (settings is null || !settings.AutoContinueOnUsageLimit) return;
|
||||
|
||||
var resetsAt = _usageState.Snapshot?.FiveHour?.ResetsAt;
|
||||
if (resetsAt is null || DateTimeOffset.UtcNow < resetsAt.Value) return;
|
||||
|
||||
List<TaskEntity> candidates;
|
||||
using (var context = _dbFactory.CreateDbContext())
|
||||
candidates = await new TaskRepository(context).GetUsageLimitAutoContinueCandidatesAsync(ct);
|
||||
|
||||
foreach (var task in candidates)
|
||||
{
|
||||
bool claimed;
|
||||
using (var context = _dbFactory.CreateDbContext())
|
||||
claimed = await new TaskRepository(context).TryClaimUsageLimitAutoContinueAsync(task.Id, DateTime.UtcNow, ct);
|
||||
if (!claimed) continue; // already fired for this task — never twice.
|
||||
|
||||
try
|
||||
{
|
||||
await _override.ContinueTask(task.Id, ContinuePrompt);
|
||||
await _broadcaster.WorkerLog(
|
||||
$"Auto-continuing #{task.Number} \"{task.Title}\" after usage-limit reset",
|
||||
WorkerLogLevel.Info, DateTime.UtcNow);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to auto-continue task {TaskId} after usage-limit reset", task.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<AppSettingsEntity?> TryReadSettingsAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var context = _dbFactory.CreateDbContext();
|
||||
return await new AppSettingsRepository(context).GetAsync(ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "UsageLimitAutoContinueCoordinator: failed to read settings; skipping tick");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user