feat(worker): Toggle "Continue on session limit reset" für Usage-Limit-Abbrüche
Klassifiziert einen echten Usage-Limit-Abbruch als eigene FailureReason
"usage_limit" (TaskRunner.ClassifyFailureReason: nur bei terminal_reason
"api_error" plus einem Limit-Muster im gerenderten Fehlertext, nicht an
Status==Failed allein). Neuer Toggle AutoContinueOnUsageLimit (app_settings,
Default aus) unter Settings → General → "Usage limit stop":
- UsageLimitAutoContinueCoordinator feuert pro Task genau einmal ContinueTask
über OverrideSlotService, sobald das 5h-Fenster (UsageState.Snapshot.FiveHour
.ResetsAt) tatsächlich zurückgesetzt ist; ein persistenter Marker
(TaskEntity.UsageLimitAutoContinuedAt) verhindert einen zweiten Anlauf bei
einem erneuten Limit-Treffer.
- QueueService schedult zusätzlich einen exakten Wake-Timer auf den
Reset-Zeitpunkt, statt nur auf den 30s-Backstop zu warten.
- Fail-open durchgängig: kein Snapshot/keine Reset-Zeit → kein Timer, kein
Continue, kein Throw. Toggle aus ändert das heutige Verhalten nicht.
Migration AddUsageLimitAutoContinue fügt beide Spalten hinzu; die von
`dotnet ef migrations add` mitgescaffoldete leere UpdateData auf app_settings
(columns/values: []) erzeugte ungültiges SQL ("near WHERE") und wurde entfernt
— TaskNumberMigrationTests deckte das über den vollen Migrate()-Pfad auf.
This commit is contained in:
@@ -55,7 +55,8 @@ public record AppSettingsDto(
|
||||
int UsageThrottleFiveHourSoftPct = 50,
|
||||
int UsageThrottleFiveHourHardPct = 65,
|
||||
int UsageThrottleSevenDaySoftPct = 50,
|
||||
int UsageThrottleSevenDayHardPct = 65);
|
||||
int UsageThrottleSevenDayHardPct = 65,
|
||||
bool AutoContinueOnUsageLimit = false);
|
||||
|
||||
// Per-model run defaults (effort + turn budget) edited in Settings -> General.
|
||||
public record ModelPresetDto(string Model, string Effort, int MaxTurns);
|
||||
@@ -471,7 +472,8 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
|
||||
row.UsageThrottleFiveHourSoftPct,
|
||||
row.UsageThrottleFiveHourHardPct,
|
||||
row.UsageThrottleSevenDaySoftPct,
|
||||
row.UsageThrottleSevenDayHardPct);
|
||||
row.UsageThrottleSevenDayHardPct,
|
||||
row.AutoContinueOnUsageLimit);
|
||||
}
|
||||
|
||||
public async Task UpdateAppSettings(AppSettingsDto dto)
|
||||
@@ -506,6 +508,7 @@ public sealed class WorkerHub : Microsoft.AspNetCore.SignalR.Hub
|
||||
UsageThrottleFiveHourHardPct = dto.UsageThrottleFiveHourHardPct,
|
||||
UsageThrottleSevenDaySoftPct = dto.UsageThrottleSevenDaySoftPct,
|
||||
UsageThrottleSevenDayHardPct = dto.UsageThrottleSevenDayHardPct,
|
||||
AutoContinueOnUsageLimit = dto.AutoContinueOnUsageLimit,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -156,6 +156,9 @@ builder.Services.AddHostedService<PrimeScheduler>();
|
||||
builder.Services.AddSingleton<IRefineRunner, RefineRunner>();
|
||||
builder.Services.AddSingleton<IRefineBroadcaster>(sp => sp.GetRequiredService<HubBroadcaster>());
|
||||
|
||||
// "Continue on session limit reset" toggle — depends on UsageState, registered below.
|
||||
builder.Services.AddSingleton<UsageLimitAutoContinueCoordinator>();
|
||||
|
||||
// QueueService: singleton + hosted service (same instance).
|
||||
builder.Services.AddSingleton<QueueService>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<QueueService>());
|
||||
|
||||
@@ -26,11 +26,13 @@ public sealed class QueueService : BackgroundService
|
||||
private readonly IUsageGate _usageGate;
|
||||
private readonly UsageState _usageState;
|
||||
private readonly HubBroadcaster _broadcaster;
|
||||
private readonly UsageLimitAutoContinueCoordinator _usageLimitAutoContinue;
|
||||
|
||||
private readonly object _lock = new();
|
||||
private readonly Dictionary<string, QueueSlotState> _queueSlots = new();
|
||||
private bool _usageGateBlocked;
|
||||
private int? _lastEffectiveSlots;
|
||||
private DateTimeOffset? _scheduledResetWakeAt;
|
||||
|
||||
public QueueService(
|
||||
IDbContextFactory<ClaudeDoDbContext> dbFactory,
|
||||
@@ -44,7 +46,8 @@ public sealed class QueueService : BackgroundService
|
||||
RunCancellationRegistry runCancels,
|
||||
IUsageGate usageGate,
|
||||
UsageState usageState,
|
||||
HubBroadcaster broadcaster)
|
||||
HubBroadcaster broadcaster,
|
||||
UsageLimitAutoContinueCoordinator usageLimitAutoContinue)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_runner = runner;
|
||||
@@ -58,6 +61,7 @@ public sealed class QueueService : BackgroundService
|
||||
_usageGate = usageGate;
|
||||
_usageState = usageState;
|
||||
_broadcaster = broadcaster;
|
||||
_usageLimitAutoContinue = usageLimitAutoContinue;
|
||||
}
|
||||
|
||||
public IReadOnlyList<(string slot, string taskId, DateTime startedAt)> GetActive()
|
||||
@@ -130,6 +134,14 @@ public sealed class QueueService : BackgroundService
|
||||
var gateDecision = await _usageGate.EvaluateAsync(stoppingToken);
|
||||
await ReportUsageGateTransitionAsync(gateDecision);
|
||||
|
||||
// "Continue on session limit reset" toggle: schedule an exact wake for the reset
|
||||
// time (instead of only relying on the 30s backstop above) and fire any pending
|
||||
// auto-continues. No-ops entirely while the toggle is off or no reset time is known.
|
||||
var scheduledWakeAt = await _usageLimitAutoContinue.GetScheduledWakeAtAsync(stoppingToken);
|
||||
if (scheduledWakeAt is not null)
|
||||
ScheduleResetWake(scheduledWakeAt.Value, stoppingToken);
|
||||
await _usageLimitAutoContinue.RunAsync(stoppingToken);
|
||||
|
||||
// Only queue refill is gated. Runs already in flight (RunNow, ContinueTask,
|
||||
// interactive sessions, planning, daily prep) keep going regardless.
|
||||
if (!gateDecision.IsBlocked)
|
||||
@@ -210,6 +222,28 @@ public sealed class QueueService : BackgroundService
|
||||
return false;
|
||||
}
|
||||
|
||||
// Wakes the queue loop exactly at the usage-limit reset time instead of leaving it to the
|
||||
// (up to 30s late) backstop — schedules once per distinct target and is a no-op for a target
|
||||
// already scheduled or already in the past (this tick's own check already covers "now").
|
||||
private void ScheduleResetWake(DateTimeOffset resetsAt, CancellationToken ct)
|
||||
{
|
||||
if (_scheduledResetWakeAt == resetsAt) return;
|
||||
_scheduledResetWakeAt = resetsAt;
|
||||
|
||||
var delay = resetsAt - DateTimeOffset.UtcNow;
|
||||
if (delay <= TimeSpan.Zero) return;
|
||||
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(delay, ct);
|
||||
_waker.Wake();
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
}, ct);
|
||||
}
|
||||
|
||||
private async Task ReportUsageGateTransitionAsync(UsageGateDecision decision)
|
||||
{
|
||||
if (decision.IsBlocked == _usageGateBlocked) return;
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
using ClaudeDo.Data;
|
||||
using ClaudeDo.Data.Models;
|
||||
using ClaudeDo.Data.Repositories;
|
||||
using ClaudeDo.Worker.Hub;
|
||||
using ClaudeDo.Worker.Usage;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ClaudeDo.Worker.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// The "Continue on session limit reset" toggle (<see cref="AppSettingsEntity.AutoContinueOnUsageLimit"/>).
|
||||
/// Off by default and fails open on every missing signal — no snapshot, no reset time — never a
|
||||
/// timer, never a crash, never a change from today's behaviour. Called once per
|
||||
/// <see cref="QueueService"/> tick (both from its wake and its 30s backstop):
|
||||
/// <see cref="GetScheduledWakeAtAsync"/> tells the caller when to schedule an exact wake so a
|
||||
/// gate-blocked queue doesn't wait out the backstop, and <see cref="RunAsync"/> fires exactly one
|
||||
/// <c>ContinueTask</c> per eligible task once the window has actually reset.
|
||||
/// </summary>
|
||||
public sealed class UsageLimitAutoContinueCoordinator
|
||||
{
|
||||
public const string ContinuePrompt =
|
||||
"The Anthropic usage limit window that stopped this run has since reset. Continue the task from where the previous run left off.";
|
||||
|
||||
private readonly IDbContextFactory<ClaudeDoDbContext> _dbFactory;
|
||||
private readonly UsageState _usageState;
|
||||
private readonly OverrideSlotService _override;
|
||||
private readonly HubBroadcaster _broadcaster;
|
||||
private readonly ILogger<UsageLimitAutoContinueCoordinator> _logger;
|
||||
|
||||
public UsageLimitAutoContinueCoordinator(
|
||||
IDbContextFactory<ClaudeDoDbContext> dbFactory, UsageState usageState, OverrideSlotService overrideSlot,
|
||||
HubBroadcaster broadcaster, ILogger<UsageLimitAutoContinueCoordinator> logger)
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
_usageState = usageState;
|
||||
_override = overrideSlot;
|
||||
_broadcaster = broadcaster;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>The reset time <see cref="QueueService"/> should schedule an exact wake for, or
|
||||
/// null when the toggle is off or no reset time is known yet — fail open, no timer.</summary>
|
||||
public async Task<DateTimeOffset?> GetScheduledWakeAtAsync(CancellationToken ct)
|
||||
{
|
||||
var settings = await TryReadSettingsAsync(ct);
|
||||
if (settings is null || !settings.AutoContinueOnUsageLimit) return null;
|
||||
return _usageState.Snapshot?.FiveHour?.ResetsAt;
|
||||
}
|
||||
|
||||
/// <summary>Fires exactly one <c>ContinueTask</c> per task failed on the usage limit, once the
|
||||
/// 5h window has actually reset. No-ops (never throws) when the toggle is off, no reset time
|
||||
/// is known, or the window hasn't reset yet.</summary>
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var settings = await TryReadSettingsAsync(ct);
|
||||
if (settings is null || !settings.AutoContinueOnUsageLimit) return;
|
||||
|
||||
var resetsAt = _usageState.Snapshot?.FiveHour?.ResetsAt;
|
||||
if (resetsAt is null || DateTimeOffset.UtcNow < resetsAt.Value) return;
|
||||
|
||||
List<TaskEntity> candidates;
|
||||
using (var context = _dbFactory.CreateDbContext())
|
||||
candidates = await new TaskRepository(context).GetUsageLimitAutoContinueCandidatesAsync(ct);
|
||||
|
||||
foreach (var task in candidates)
|
||||
{
|
||||
bool claimed;
|
||||
using (var context = _dbFactory.CreateDbContext())
|
||||
claimed = await new TaskRepository(context).TryClaimUsageLimitAutoContinueAsync(task.Id, DateTime.UtcNow, ct);
|
||||
if (!claimed) continue; // already fired for this task — never twice.
|
||||
|
||||
try
|
||||
{
|
||||
await _override.ContinueTask(task.Id, ContinuePrompt);
|
||||
await _broadcaster.WorkerLog(
|
||||
$"Auto-continuing #{task.Number} \"{task.Title}\" after usage-limit reset",
|
||||
WorkerLogLevel.Info, DateTime.UtcNow);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to auto-continue task {TaskId} after usage-limit reset", task.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<AppSettingsEntity?> TryReadSettingsAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var context = _dbFactory.CreateDbContext();
|
||||
return await new AppSettingsRepository(context).GetAsync(ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "UsageLimitAutoContinueCoordinator: failed to read settings; skipping tick");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using ClaudeDo.Data;
|
||||
using ClaudeDo.Data.Models;
|
||||
using ClaudeDo.Data.Repositories;
|
||||
@@ -159,13 +160,13 @@ public sealed class TaskRunner
|
||||
else
|
||||
{
|
||||
await MarkFailed(task.Id, task.Number, task.Title, slot, retryResult.ErrorMarkdown, retryResult.TurnCount,
|
||||
retryConfig.MaxTurns, ClassifyFailureReason(retryResult.TerminalReason));
|
||||
retryConfig.MaxTurns, ClassifyFailureReason(retryResult.TerminalReason, retryResult.ErrorMarkdown));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await MarkFailed(task.Id, task.Number, task.Title, slot, result.ErrorMarkdown, result.TurnCount,
|
||||
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason));
|
||||
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason, result.ErrorMarkdown));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,7 +277,7 @@ public sealed class TaskRunner
|
||||
else
|
||||
{
|
||||
await MarkFailed(taskId, task.Number, task.Title, slot, result.ErrorMarkdown, result.TurnCount,
|
||||
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason));
|
||||
resolvedConfig.MaxTurns, ClassifyFailureReason(result.TerminalReason, result.ErrorMarkdown));
|
||||
}
|
||||
|
||||
await _broadcaster.TaskUpdated(taskId);
|
||||
@@ -603,13 +604,24 @@ public sealed class TaskRunner
|
||||
|
||||
internal static bool IsWriteTool(string toolName) => WriteTools.Contains(toolName);
|
||||
|
||||
/// <summary>Classifies the CLI's raw <c>terminal_reason</c> into the small, MCP-facing enum
|
||||
/// (<c>max_turns|timeout|error</c>) get_task/batch_get_tasks report as failureReason.
|
||||
// The CLI has no dedicated terminal_reason for a session/account usage-limit stop — it comes
|
||||
// back as a plain "api_error" with the provider's own text (session/rate limit + reset time)
|
||||
// riding in the result/error text (see ClaudeProcess.RunAsync). Matched against the rendered
|
||||
// failure message (BuildFailureMarkdown's output), never against Status alone.
|
||||
private static readonly Regex UsageLimitPattern = new(
|
||||
"usage limit|session limit|rate.?limit", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
internal static bool IsUsageLimitMessage(string? text) => !string.IsNullOrEmpty(text) && UsageLimitPattern.IsMatch(text);
|
||||
|
||||
/// <summary>Classifies the CLI's raw <c>terminal_reason</c> (plus, for "api_error", the
|
||||
/// rendered failure message) into the small, MCP-facing enum
|
||||
/// (<c>max_turns|timeout|usage_limit|error</c>) get_task/batch_get_tasks report as failureReason.
|
||||
/// "cancelled" is set explicitly at the call sites that know it (there's no CLI signal for it).</summary>
|
||||
internal static string ClassifyFailureReason(string? terminalReason) => terminalReason switch
|
||||
internal static string ClassifyFailureReason(string? terminalReason, string? messageText = null) => terminalReason switch
|
||||
{
|
||||
"max_turns" => "max_turns",
|
||||
"timeout" => "timeout",
|
||||
"api_error" when IsUsageLimitMessage(messageText) => "usage_limit",
|
||||
_ => "error",
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user