Files
ClaudeDo/tests/ClaudeDo.Data.Tests/PromptFilesTests.cs
T
mika kuns 315bea7cf9 fix(prompts): correct five prompt claims that contradicted the tool allowlists
Audited all 12 prompt kinds against the code they drive. Every real defect sat on
the boundary between prompt text and the --allowedTools the launcher passes.

- Planning: "Use nothing else" after a six-tool list forbade the brainstorming
  Skill the same prompt demands two paragraphs earlier. WindowsTerminalLauncher
  allowlists mcp__claudedo__*,Read,Grep,Glob,WebFetch,WebSearch,Skill -- name them,
  and tell the planner to ground subtasks in the repo with Read/Grep/Glob.
- System: SuggestImprovement is only allowlisted when ParentTaskId is null and
  PlanningPhase is None, and TaskRunMcpService throws for any child, but this
  prompt reaches every run. Planning children were told to use a tool they lack.
- MergeHelperExecute: derived "effective max-turns" from task/list/preset by hand,
  which misses TaskRunner's MaxTurnsCeiling clamp. Call get_effective_run_config
  instead -- built for exactly this and reports the clamp.
- MergeHelperExecute: quoted the override-slot error as the raw lowercase throw
  rather than the string ExternalMcpService actually surfaces.
- Refine: listed Read/Grep/Glob unconditionally though RefinePrompt.BuildArgs only
  appends them when a repo is available.

Two findings deliberately left open, both needing a code decision rather than a
prompt edit: the System prompt's worktree claim is false for a list without a
WorkingDir (task runs in a plain sandbox dir), and 'fable' is missing from both the
prompt's cost ordering and ModelRegistry.ByCostAscending.
2026-08-06 22:47:41 +02:00

342 lines
14 KiB
C#

using ClaudeDo.Data;
namespace ClaudeDo.Data.Tests;
public class PromptFilesTests
{
[Fact]
public void RenderTemplate_replaces_known_tokens()
{
var outp = PromptFiles.RenderTemplate(
"Plan for {date}, cap {maxTasks}.",
new Dictionary<string, string> { ["date"] = "2026-06-04", ["maxTasks"] = "5" });
Assert.Equal("Plan for 2026-06-04, cap 5.", outp);
}
[Fact]
public void RenderTemplate_leaves_unknown_braces_intact()
{
var outp = PromptFiles.RenderTemplate(
"## {Wochentag}, {dd.MM.yyyy} — {start}",
new Dictionary<string, string> { ["start"] = "01.06.2026" });
Assert.Equal("## {Wochentag}, {dd.MM.yyyy} — 01.06.2026", outp);
}
[Fact]
public void RenderTemplate_does_not_substitute_tokens_that_appear_inside_a_value()
{
// A sharpened task description can legitimately contain "{repo}" (a config snippet, a
// path placeholder). Substitution must be a single pass over the TEMPLATE, so injected
// values are never rescanned.
var outp = PromptFiles.RenderTemplate(
"Repo: {repo}\n\n{tasks}",
new Dictionary<string, string>
{
["repo"] = "C:\\real\\repo",
["tasks"] = "- Fix the {repo} placeholder in the config template",
});
Assert.Equal("Repo: C:\\real\\repo\n\n- Fix the {repo} placeholder in the config template", outp);
}
[Fact]
public void RenderTemplate_result_is_independent_of_dictionary_order()
{
const string template = "Scope: {scope}\nRepo: {repo}\n\n{tasks}";
var task = "- Document {scope} and {repo} tokens";
var tasksLast = PromptFiles.RenderTemplate(template, new Dictionary<string, string>
{
["scope"] = "List: Bugs", ["repo"] = "C:\\repo", ["tasks"] = task,
});
var tasksFirst = PromptFiles.RenderTemplate(template, new Dictionary<string, string>
{
["tasks"] = task, ["repo"] = "C:\\repo", ["scope"] = "List: Bugs",
});
Assert.Equal(tasksLast, tasksFirst);
Assert.Contains("- Document {scope} and {repo} tokens", tasksLast);
}
[Fact]
public void DefaultFor_system_mentions_blocked_marker_and_scope()
{
var d = PromptFiles.DefaultFor(PromptKind.System);
Assert.Contains("CLAUDEDO_BLOCKED:", d);
Assert.Contains("unattended", d, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void DefaultFor_planning_initial_has_title_and_description_tokens()
{
var d = PromptFiles.DefaultFor(PromptKind.PlanningInitial);
Assert.Contains("{title}", d);
Assert.Contains("{description}", d);
}
[Fact]
public void PathFor_planning_is_planning_system_file()
{
Assert.EndsWith("planning-system.md", PromptFiles.PathFor(PromptKind.Planning));
}
[Fact]
public void PathFor_merge_helper_kinds_map_to_their_files()
{
Assert.EndsWith("merge-helper-triage.md", PromptFiles.PathFor(PromptKind.MergeHelperTriage));
Assert.EndsWith("merge-helper-execute.md", PromptFiles.PathFor(PromptKind.MergeHelperExecute));
Assert.EndsWith("merge-helper-initial.md", PromptFiles.PathFor(PromptKind.MergeHelperInitial));
Assert.EndsWith("merge-helper-handoff.md", PromptFiles.PathFor(PromptKind.MergeHelperHandoff));
}
[Fact]
public void DefaultFor_merge_helper_triage_covers_phases_0_to_2_only()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperTriage);
Assert.False(string.IsNullOrWhiteSpace(d));
Assert.Contains("## Phase 0", d);
Assert.Contains("## Phase 1", d);
Assert.Contains("## Phase 2", d);
// The run/review/merge phases belong to the handoff session's prompt, not this one —
// carrying them here is what made the handoff session redo dedupe work.
Assert.DoesNotContain("## Phase 3", d);
Assert.DoesNotContain("## Phase 4", d);
Assert.DoesNotContain("## Phase 5", d);
}
[Fact]
public void DefaultFor_merge_helper_execute_covers_phases_3_to_5_only()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperExecute);
Assert.False(string.IsNullOrWhiteSpace(d));
Assert.Contains("## Phase 3", d);
Assert.Contains("## Phase 4", d);
Assert.Contains("## Phase 5", d);
Assert.DoesNotContain("## Phase 0", d);
Assert.DoesNotContain("## Phase 1", d);
Assert.DoesNotContain("## Phase 2", d);
}
[Fact]
public void DefaultFor_merge_helper_triage_names_the_tools_its_phases_need()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperTriage);
Assert.Contains("batch_get_tasks", d); // phase 0
Assert.Contains("update_task", d); // phase 1 + 2
Assert.Contains("handoff_list_handler", d); // handoff
// Triage never runs or merges anything.
Assert.DoesNotContain("review_task", d);
Assert.DoesNotContain("continue_merge", d);
Assert.DoesNotContain("preview_merge_set", d);
}
[Fact]
public void DefaultFor_merge_helper_execute_names_the_tools_its_phases_need()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperExecute);
Assert.Contains("get_app_settings", d); // phase 3
Assert.Contains("update_task_status", d); // phase 3
Assert.Contains("wait_for_task_change", d); // phase 3
Assert.Contains("preview_merge_set", d); // phase 4
Assert.Contains("review_task", d); // phase 4
Assert.Contains("continue_merge", d); // phase 4
Assert.DoesNotContain("run_task_now(", d); // single override slot — must not batch-start
}
[Fact]
public void DefaultFor_merge_helper_execute_waits_with_the_real_server_side_cap()
{
// TaskWaitMcpTools.MaxTimeoutSeconds is 900 and every ClaudeDo launcher sets
// MCP_TOOL_TIMEOUT=930000ms. The prompt used to say 170 -- a leftover from the retired
// 200000ms era -- which burned ~5x the turns on re-waiting.
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperExecute);
Assert.Contains("timeoutSeconds 900", d);
Assert.DoesNotContain("170", d);
}
[Fact]
public void DefaultFor_merge_helper_execute_waits_through_waiting_for_children()
{
// Without the flag, a parent with children reports "changed" as soon as it reaches
// WaitingForChildren, so the handler would advance to review/merge while children run.
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperExecute);
Assert.Contains("treatWaitingForChildrenAsBusy=true", d);
Assert.Contains("WaitingForChildren", d);
}
[Fact]
public void DefaultFor_merge_helper_execute_keeps_the_shared_checkout_git_guard()
{
// The handler session never gets PromptKind.System, so this is the ONLY place the
// never-`git add -A`-in-a-shared-checkout rule reaches a list-handler run.
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperExecute);
Assert.Contains("git add -- <the resolved paths>", d);
Assert.Contains("NEVER `git add -A`", d);
Assert.Contains("Never use raw `git merge`", d);
}
[Fact]
public void DefaultFor_merge_helper_only_asks_dedupe_questions_when_a_candidate_exists()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperTriage);
Assert.Contains("move straight to Phase 2", d);
Assert.Contains("do not ask the user to confirm the absence of duplicates", d);
Assert.Contains("Cancel nothing without an explicit answer", d);
Assert.DoesNotContain("whenever you are unsure", d);
}
[Fact]
public void DefaultFor_merge_helper_phase0_treats_brief_as_primary_source()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperTriage);
Assert.Contains("primary source", d, StringComparison.OrdinalIgnoreCase);
Assert.Contains("batch_get_tasks", d);
Assert.Contains("Do not act on any single task before you have read", d);
}
[Fact]
public void DefaultFor_merge_helper_allows_splitting_bundled_tasks_in_phase_2()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperTriage);
Assert.Contains("Propose splitting it to the user", d);
Assert.Contains("add_task/add_subtask", d);
Assert.Contains("do not invent requirements", d);
}
[Fact]
public void DefaultFor_merge_helper_checks_effective_max_turns_before_queuing()
{
// Must delegate to get_effective_run_config rather than re-deriving the
// task/list/preset chain: TaskRunner.ResolveMaxTurns clamps the result to
// AppSettings.MaxTurnsCeiling, so a hand-derived number can exceed what actually runs.
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperExecute);
Assert.Contains("get_effective_run_config", d);
Assert.Contains("clamped", d);
Assert.Contains("set_task_config", d);
Assert.DoesNotContain("get_list_config", d);
}
[Fact]
public void DefaultFor_merge_helper_execute_quotes_the_override_slot_error_verbatim()
{
// ExternalMcpService.RunTaskNow rewraps OverrideSlotService's raw lowercase throw.
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperExecute);
Assert.Contains("Override slot busy. Try again later.", d);
}
[Fact]
public void DefaultFor_system_scopes_suggest_improvement_to_top_level_tasks()
{
// TaskRunner only allowlists SuggestImprovement when ParentTaskId is null AND
// PlanningPhase is None, and TaskRunMcpService throws for any child -- but this
// prompt reaches every run, including planning children.
var d = PromptFiles.DefaultFor(PromptKind.System);
Assert.Contains("SuggestImprovement", d);
Assert.Contains("standalone top-level task", d);
Assert.Contains("improvements are one layer deep", d);
}
[Fact]
public void DefaultFor_planning_names_every_allowlisted_tool_and_claims_no_exclusivity()
{
// WindowsTerminalLauncher allowlists mcp__claudedo__*,Read,Grep,Glob,WebFetch,
// WebSearch,Skill. The prompt used to say "Use nothing else" right after telling the
// session to invoke the brainstorming skill -- forbidding its own first instruction.
var d = PromptFiles.DefaultFor(PromptKind.Planning);
Assert.DoesNotContain("Use nothing else", d);
Assert.Contains("Skill", d);
Assert.Contains("Grep", d);
Assert.Contains("WebSearch", d);
Assert.Contains("Write, Edit or Bash", d);
}
[Fact]
public void DefaultFor_refine_gates_the_read_tools_on_an_available_repo()
{
// RefinePrompt.BuildArgs only appends Read/Grep/Glob when canReadRepo is true.
var d = PromptFiles.DefaultFor(PromptKind.Refine);
Assert.Contains("only when a repository is available", d);
}
[Fact]
public void DefaultFor_merge_helper_checks_file_collisions_before_merge_order()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperExecute);
Assert.Contains("Default to the order the brief lists them", d);
Assert.Contains("tell the user which tasks collide", d);
Assert.Contains("NORMAL case, not a failure", d);
}
[Fact]
public void DefaultFor_merge_helper_initial_has_repo_token()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperInitial);
Assert.Contains("{repo}", d);
}
[Fact]
public void DefaultFor_merge_helper_initial_has_scope_and_tasks_tokens()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperInitial);
Assert.False(string.IsNullOrWhiteSpace(d));
Assert.Contains("{scope}", d);
Assert.Contains("{tasks}", d);
}
[Fact]
public void RenderTemplate_merge_helper_initial_substitutes_scope_and_tasks()
{
var outp = PromptFiles.RenderTemplate(
PromptFiles.DefaultFor(PromptKind.MergeHelperInitial),
new Dictionary<string, string> { ["scope"] = "All lists", ["tasks"] = "- [Idle] T1" });
Assert.Contains("Scope: All lists", outp);
Assert.Contains("- [Idle] T1", outp);
Assert.DoesNotContain("{scope}", outp);
Assert.DoesNotContain("{tasks}", outp);
}
[Fact]
public void DefaultFor_merge_helper_handoff_has_scope_repo_and_tasks_tokens()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperHandoff);
Assert.False(string.IsNullOrWhiteSpace(d));
Assert.Contains("{scope}", d);
Assert.Contains("{repo}", d);
Assert.Contains("{tasks}", d);
}
[Fact]
public void DefaultFor_merge_helper_handoff_points_at_phase_3()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperHandoff);
Assert.Contains("phase 3", d, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void RenderTemplate_merge_helper_handoff_substitutes_scope_repo_and_tasks()
{
var outp = PromptFiles.RenderTemplate(
PromptFiles.DefaultFor(PromptKind.MergeHelperHandoff),
new Dictionary<string, string>
{
["scope"] = "List: Bugs",
["repo"] = "C:\\repo",
["tasks"] = "- [WaitingForReview] T1 (id: abc)",
});
Assert.Contains("Scope: List: Bugs", outp);
Assert.Contains("Repo: C:\\repo", outp);
Assert.Contains("- [WaitingForReview] T1 (id: abc)", outp);
Assert.DoesNotContain("{scope}", outp);
Assert.DoesNotContain("{repo}", outp);
Assert.DoesNotContain("{tasks}", outp);
}
[Fact]
public void DefaultFor_merge_helper_tells_the_session_to_hand_off_after_phase_2()
{
var d = PromptFiles.DefaultFor(PromptKind.MergeHelperTriage);
Assert.Contains("handoff_list_handler", d);
Assert.Contains("do not continue into phase 3 yourself", d, StringComparison.OrdinalIgnoreCase);
}
}