releases/ClaudeDo
2
0
Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases 25 Wiki Activity

refactor(usage): drop the transcript aggregation path in favour of TokenTracker

Browse Source
This commit is contained in:
mika kuns
2026-08-24 13:49:49 +02:00
parent 44337d4b94
commit 40879705ac
6 changed files with 23 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ClaudeDo.Worker/Usage/Interfaces/ITranscriptUsageReader.cs
-3
View File
@@ -2,9 +2,6 @@ namespace ClaudeDo.Worker.Usage.Interfaces;
public interface ITranscriptUsageReader
{
Task<IReadOnlyList<UsageAggregateRow>> ReadAsync(
DateOnly start, DateOnly end, CancellationToken ct = default);
/// <summary>Cumulative raw token totals for one session's transcript file
/// (located by <c>{sessionId}.jsonl</c> under the projects root), or null when
/// no matching transcript file can be found or read.</summary>
src/ClaudeDo.Worker/Usage/TranscriptUsageReader.cs
+5 -82
View File
@@ -9,63 +9,13 @@ namespace ClaudeDo.Worker.Usage;
public sealed class TranscriptUsageReader : ITranscriptUsageReader
{
private readonly string _projectsRoot;
private readonly string _centralRoot;
private readonly string _sandboxRoot;
private readonly ConcurrentDictionary<string, FileCacheEntry> _cache = new();
// cfg is unused since the model/scope aggregation moved to TokenTracker, but the parameter
// stays so the DI registration and every existing call site keep working.
public TranscriptUsageReader(WorkerConfig cfg, string? projectsRoot = null)
{
_projectsRoot = projectsRoot ?? Paths.Expand("~/.claude/projects");
_centralRoot = NormalizePath(cfg.CentralWorktreeRoot);
_sandboxRoot = NormalizePath(cfg.SandboxRoot);
}
public Task<IReadOnlyList<UsageAggregateRow>> ReadAsync(
DateOnly start, DateOnly end, CancellationToken ct = default)
{
var seenKeys = new HashSet<string>();
var buckets = new Dictionary<(DateOnly Date, string Model, UsageScope Scope), Accumulator>();
if (Directory.Exists(_projectsRoot))
{
// A transcript last written before the window began cannot hold a record inside it, so
// it is skipped unread — that is what keeps a 7-day range off the full history (hundreds
// of MB). One day of slack absorbs local-vs-UTC skew between mtime and record stamps.
var mtimeCutoff = start.ToDateTime(TimeOnly.MinValue).AddDays(-1);
foreach (var file in new DirectoryInfo(_projectsRoot).EnumerateFiles("*.jsonl", SearchOption.AllDirectories))
{
ct.ThrowIfCancellationRequested();
if (file.LastWriteTime < mtimeCutoff) continue;
foreach (var record in GetOrReadFile(file))
{
if (record.Date < start || record.Date > end) continue;
if (!seenKeys.Add(record.DedupeKey)) continue;
var key = (record.Date, record.Model, record.Scope);
if (!buckets.TryGetValue(key, out var acc))
{
acc = new Accumulator();
buckets[key] = acc;
}
acc.Input += record.InputTokens;
acc.Output += record.OutputTokens;
acc.CacheRead += record.CacheReadTokens;
acc.CacheCreation += record.CacheCreationTokens;
acc.Messages++;
}
}
}
var rows = buckets
.Select(kv => new UsageAggregateRow(
kv.Key.Date, kv.Key.Model, kv.Key.Scope,
kv.Value.Input, kv.Value.Output, kv.Value.CacheRead, kv.Value.CacheCreation, kv.Value.Messages))
.OrderBy(r => r.Date).ThenBy(r => r.Model).ThenBy(r => r.Scope)
.ToList();
return Task.FromResult<IReadOnlyList<UsageAggregateRow>>(rows);
}
public Task<SessionUsageTotals?> ReadSessionTotalsAsync(string sessionId, CancellationToken ct = default)
@@ -130,7 +80,8 @@ public sealed class TranscriptUsageReader : ITranscriptUsageReader
if (!root.TryGetProperty("message", out var msg) || msg.ValueKind != JsonValueKind.Object) continue;
if (!msg.TryGetProperty("model", out var modelEl) || modelEl.ValueKind != JsonValueKind.String) continue;
var date = DateOnly.FromDateTime(ts.LocalDateTime);
// The timestamp and model are still parsed above: they gate out non-assistant
// lines and <synthetic> models, which is right for the session totals too.
var model = modelEl.GetString()!;
if (model == "<synthetic>") continue;
@@ -143,54 +94,26 @@ public sealed class TranscriptUsageReader : ITranscriptUsageReader
cacheCreation = GetLong(usage, "cache_creation_input_tokens");
}
var cwd = TryGetString(root, "cwd") ?? "";
var dedupeKey = TryGetString(root, "requestId")
?? TryGetString(msg, "id")
?? Guid.NewGuid().ToString();
records.Add(new UsageMessageRecord(
date, model, ResolveScope(cwd), input, output, cacheRead, cacheCreation, dedupeKey));
records.Add(new UsageMessageRecord(input, output, cacheRead, cacheCreation, dedupeKey));
}
}
return records;
}
private UsageScope ResolveScope(string cwd)
{
var norm = NormalizePath(cwd);
if (norm.Length == 0) return UsageScope.Other;
// Sibling-strategy worktrees are placed next to whatever repo they belong to
// (no single root path), but always under a literal ".claudedo-worktrees" segment.
if (norm.Split('\\').Any(seg => seg == ".claudedo-worktrees")) return UsageScope.ClaudeDo;
if (IsUnderRoot(norm, _centralRoot)) return UsageScope.ClaudeDo;
if (IsUnderRoot(norm, _sandboxRoot)) return UsageScope.ClaudeDo;
return UsageScope.Other;
}
private static bool IsUnderRoot(string normPath, string normRoot) =>
normRoot.Length > 0 && (normPath == normRoot || normPath.StartsWith(normRoot + "\\", StringComparison.Ordinal));
private static string? TryGetString(JsonElement obj, string prop) =>
obj.TryGetProperty(prop, out var el) && el.ValueKind == JsonValueKind.String ? el.GetString() : null;
private static long GetLong(JsonElement obj, string prop) =>
obj.TryGetProperty(prop, out var el) && el.TryGetInt64(out var v) ? v : 0;
private static string NormalizePath(string p) =>
(p ?? "").Replace('/', '\\').TrimEnd('\\').ToLowerInvariant();
private sealed class Accumulator
{
public long Input, Output, CacheRead, CacheCreation;
public int Messages;
}
private sealed record FileCacheEntry(long Length, DateTime LastWriteUtc, List<UsageMessageRecord> Records);
private sealed record UsageMessageRecord(
DateOnly Date, string Model, UsageScope Scope,
long InputTokens, long OutputTokens, long CacheReadTokens, long CacheCreationTokens,
string DedupeKey);
}
src/ClaudeDo.Worker/Usage/UsageModels.cs
-16
View File
@@ -17,22 +17,6 @@ public sealed record UsageSnapshot(
IReadOnlyList<UsageLimitRow> Limits,
DateTime FetchedAtUtc);
public enum UsageScope
{
ClaudeDo,
Other,
}
public sealed record UsageAggregateRow(
DateOnly Date,
string Model,
UsageScope Scope,
long InputTokens,
long OutputTokens,
long CacheReadTokens,
long CacheCreationTokens,
int Messages);
/// <summary>Cumulative raw token usage for one Claude Code session (all its transcript
/// lines to date), synthetic messages excluded. Not a per-run delta.</summary>
public sealed record SessionUsageTotals(
tests/ClaudeDo.Worker.Tests/Infrastructure/FakeTranscriptUsageReader.cs
-4
View File
@@ -9,10 +9,6 @@ public sealed class FakeTranscriptUsageReader : ITranscriptUsageReader
public void SetTotals(string sessionId, SessionUsageTotals totals) => _totalsBySession[sessionId] = totals;
public Task<IReadOnlyList<UsageAggregateRow>> ReadAsync(
DateOnly start, DateOnly end, CancellationToken ct = default) =>
Task.FromResult<IReadOnlyList<UsageAggregateRow>>(Array.Empty<UsageAggregateRow>());
public Task<SessionUsageTotals?> ReadSessionTotalsAsync(string sessionId, CancellationToken ct = default) =>
Task.FromResult(_totalsBySession.TryGetValue(sessionId, out var totals) ? totals : null);
}
tests/ClaudeDo.Worker.Tests/Runner/RunUsagePersistenceTests.cs
-4
View File
@@ -155,10 +155,6 @@ public sealed class RunUsagePersistenceTests : IDisposable
private sealed class ThrowingUsageReader : ITranscriptUsageReader
{
public Task<IReadOnlyList<UsageAggregateRow>> ReadAsync(
DateOnly start, DateOnly end, CancellationToken ct = default) =>
throw new IOException("boom");
public Task<SessionUsageTotals?> ReadSessionTotalsAsync(string sessionId, CancellationToken ct = default) =>
throw new IOException("boom");
}
tests/ClaudeDo.Worker.Tests/Usage/TranscriptUsageReaderTests.cs
+18 -172
View File
@@ -4,6 +4,10 @@ using ClaudeDo.Worker.Usage;
namespace ClaudeDo.Worker.Tests.Usage;
/// <summary>
/// Covers what is left of the reader after the model/scope aggregation moved to TokenTracker:
/// the per-session totals that back <c>task_runs</c>' per-run token delta.
/// </summary>
public class TranscriptUsageReaderTests : IDisposable
{
private readonly string _root;
@@ -57,178 +61,6 @@ public class TranscriptUsageReaderTests : IDisposable
private TranscriptUsageReader MakeReader() =>
new(_cfg, Path.Combine(_root, "projects"));
[Fact]
public async Task Aggregates_By_Date_And_Model_Including_Cache_Tokens()
{
WriteSession("proj", "s.jsonl",
AssistantLine(@"C:\Dev\App", "2026-06-01T08:00:00Z", "claude-sonnet-5", 10, 20, 3, 1),
AssistantLine(@"C:\Dev\App", "2026-06-01T09:00:00Z", "claude-sonnet-5", 5, 6, 1, 0),
AssistantLine(@"C:\Dev\App", "2026-06-02T08:00:00Z", "claude-opus-5", 100, 200, 30, 10));
var reader = MakeReader();
var result = await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3));
Assert.Equal(2, result.Count);
var day1 = Assert.Single(result, r => r.Model == "claude-sonnet-5");
Assert.Equal(new DateOnly(2026, 6, 1), day1.Date);
Assert.Equal(15, day1.InputTokens);
Assert.Equal(26, day1.OutputTokens);
Assert.Equal(4, day1.CacheReadTokens);
Assert.Equal(1, day1.CacheCreationTokens);
Assert.Equal(2, day1.Messages);
var day2 = Assert.Single(result, r => r.Model == "claude-opus-5");
Assert.Equal(new DateOnly(2026, 6, 2), day2.Date);
Assert.Equal(100, day2.InputTokens);
Assert.Equal(1, day2.Messages);
}
[Fact]
public async Task Same_RequestId_Across_Files_Counts_Once()
{
WriteSession("proj-a", "s1.jsonl",
AssistantLine(@"C:\Dev\App", "2026-06-01T08:00:00Z", "claude-sonnet-5", 10, 20, 0, 0, requestId: "req-1"));
WriteSession("proj-b", "s2.jsonl",
AssistantLine(@"C:\Dev\App", "2026-06-01T08:00:00Z", "claude-sonnet-5", 10, 20, 0, 0, requestId: "req-1"));
var reader = MakeReader();
var result = await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3));
var row = Assert.Single(result);
Assert.Equal(1, row.Messages);
Assert.Equal(10, row.InputTokens);
}
[Fact]
public async Task Scope_Split_Covers_Sibling_Central_And_Sandbox_Roots()
{
var siblingCwd = Path.Combine(_root, "some-repo", ".claudedo-worktrees", "list-slug", "task-id");
var centralCwd = Path.Combine(_cfg.CentralWorktreeRoot, "list-slug", "task-id");
var sandboxCwd = Path.Combine(_cfg.SandboxRoot, "task-id");
var otherCwd = Path.Combine(_root, "some-repo");
WriteSession("proj", "s.jsonl",
AssistantLine(siblingCwd, "2026-06-01T08:00:00Z", "claude-sonnet-5", 1, 1, 0, 0, requestId: "sibling"),
AssistantLine(centralCwd, "2026-06-01T08:00:00Z", "claude-sonnet-5", 1, 1, 0, 0, requestId: "central"),
AssistantLine(sandboxCwd, "2026-06-01T08:00:00Z", "claude-sonnet-5", 1, 1, 0, 0, requestId: "sandbox"),
AssistantLine(otherCwd, "2026-06-01T08:00:00Z", "claude-sonnet-5", 1, 1, 0, 0, requestId: "other"));
var reader = MakeReader();
var result = await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3));
Assert.Equal(2, result.Count);
var claudeDo = Assert.Single(result, r => r.Scope == UsageScope.ClaudeDo);
Assert.Equal(3, claudeDo.Messages);
var other = Assert.Single(result, r => r.Scope == UsageScope.Other);
Assert.Equal(1, other.Messages);
}
[Fact]
public async Task Date_Filter_Excludes_Lines_Outside_Window()
{
WriteSession("proj", "s.jsonl",
AssistantLine(@"C:\Dev\App", "2026-05-01T08:00:00Z", "claude-sonnet-5", 1, 1, 0, 0),
AssistantLine(@"C:\Dev\App", "2026-06-02T08:00:00Z", "claude-sonnet-5", 5, 5, 0, 0),
AssistantLine(@"C:\Dev\App", "2026-07-01T08:00:00Z", "claude-sonnet-5", 1, 1, 0, 0));
var reader = MakeReader();
var result = await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3));
var row = Assert.Single(result);
Assert.Equal(5, row.InputTokens);
Assert.Equal(1, row.Messages);
}
[Fact]
public async Task Files_Last_Written_Before_The_Window_Are_Not_Read()
{
// Deliberate heuristic: a transcript whose mtime predates the window cannot contain a
// record inside it, so it is skipped unread. Here the content would match the window —
// proving the file was never opened, which is what keeps a 7-day range off the full history.
var path = WriteSession("proj", "old.jsonl",
AssistantLine(@"C:\Dev\App", "2026-06-02T08:00:00Z", "claude-sonnet-5", 5, 5, 0, 0));
File.SetLastWriteTime(path, new DateTime(2026, 5, 1, 12, 0, 0));
var reader = MakeReader();
var result = await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3));
Assert.Empty(result);
}
[Fact]
public async Task File_Written_On_The_Window_Start_Day_Is_Still_Read()
{
var path = WriteSession("proj", "edge.jsonl",
AssistantLine(@"C:\Dev\App", "2026-06-01T08:00:00Z", "claude-sonnet-5", 5, 5, 0, 0));
File.SetLastWriteTime(path, new DateTime(2026, 6, 1, 0, 5, 0));
var reader = MakeReader();
var result = await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3));
Assert.Single(result);
}
[Fact]
public async Task Malformed_Line_Does_Not_Abort_The_Run()
{
WriteSession("proj", "s.jsonl",
"this is not json",
AssistantLine(@"C:\Dev\App", "2026-06-01T08:00:00Z", "claude-sonnet-5", 5, 5, 0, 0));
var reader = MakeReader();
var result = await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3));
var row = Assert.Single(result);
Assert.Equal(1, row.Messages);
}
[Fact]
public async Task Cache_Skips_Unchanged_File_And_Picks_Up_Appended_Lines()
{
var path = WriteSession("proj", "s.jsonl",
AssistantLine(@"C:\Dev\App", "2026-06-01T08:00:00Z", "claude-sonnet-5", 5, 5, 0, 0, requestId: "req-a"));
var reader = MakeReader();
var first = Assert.Single(await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3)));
Assert.Equal(1, first.Messages);
// Overwrite with different content but the SAME length and mtime: if the reader honored
// the cache it must still return the ORIGINAL aggregate, proving it did not re-read the file.
var originalBytes = File.ReadAllBytes(path);
var originalWriteUtc = File.GetLastWriteTimeUtc(path);
var tamperedLine = AssistantLine(@"C:\Dev\App", "2026-06-01T08:00:00Z", "claude-sonnet-5", 9, 9, 0, 0, requestId: "req-9");
// Pad/truncate to the EXACT same byte length as the original file so the (length, mtime)
// cache key still matches — the reader must then serve the cached (stale) aggregate.
var tamperedPadded = tamperedLine.Length + 1 <= originalBytes.Length
? tamperedLine.PadRight(originalBytes.Length - 1) + "\n"
: tamperedLine[..(originalBytes.Length - 1)] + "\n";
var tamperedBytes = System.Text.Encoding.UTF8.GetBytes(tamperedPadded);
Assert.Equal(originalBytes.Length, tamperedBytes.Length);
File.WriteAllBytes(path, tamperedBytes);
File.SetLastWriteTimeUtc(path, originalWriteUtc);
var stale = Assert.Single(await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3)));
Assert.Equal(5, stale.InputTokens);
// Now really append a new line: length/mtime change, so the file must be re-read.
File.AppendAllLines(path, new[]
{
AssistantLine(@"C:\Dev\App", "2026-06-01T09:00:00Z", "claude-sonnet-5", 3, 3, 0, 0, requestId: "req-b"),
});
var updated = Assert.Single(await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3)));
Assert.Equal(2, updated.Messages);
}
[Fact]
public async Task Missing_ProjectsRoot_Returns_Empty_Without_Throwing()
{
var reader = new TranscriptUsageReader(_cfg, Path.Combine(_root, "does-not-exist"));
var result = await reader.ReadAsync(new DateOnly(2026, 6, 1), new DateOnly(2026, 6, 3));
Assert.Empty(result);
}
[Fact]
public async Task ReadSessionTotalsAsync_Sums_All_Assistant_Messages_In_The_Session_File()
{
@@ -276,6 +108,20 @@ public class TranscriptUsageReaderTests : IDisposable
Assert.Equal(20, totals.OutputTokens);
}
[Fact]
public async Task ReadSessionTotalsAsync_Malformed_Line_Does_Not_Abort_The_Read()
{
WriteSession("proj", "sess-4.jsonl",
"this is not json",
AssistantLine(@"C:\Dev\App", "2026-06-01T08:00:00Z", "claude-sonnet-5", 5, 5, 0, 0, requestId: "r1"));
var reader = MakeReader();
var totals = await reader.ReadSessionTotalsAsync("sess-4");
Assert.NotNull(totals);
Assert.Equal(5, totals!.InputTokens);
}
[Fact]
public async Task ReadSessionTotalsAsync_Returns_Null_When_No_Matching_Transcript_File()
{