Coverage for scripts/check_command_contracts.py: 100%
149 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-31 06:18 +0000
1#!/usr/bin/env python3
2"""Check that the prose commands are executable — the integration test for markdown.
4Every file in ``commands/`` and ``prompts/`` is instructions a model executes at
5runtime. The bundled scripts are covered by unit tests, but the **prose that drives
6them** was previously unverified, and that is where the expensive failures have come
7from: a command referring to a script that no longer exists, passing a flag that was
8renamed, or invoking a slash command that was removed. Those surface only when a user
9runs the command, mid-task, and the model improvises around the breakage.
11This closes that gap by treating each command as a contract and checking it against
12the code it actually calls:
141. **Frontmatter** — a command declares ``description``, ``argument-hint`` and
15 ``allowed-tools``, and the block **parses**. That second half matters: five of seven
16 commands once shipped frontmatter YAML could not read, because a description
17 contained an unquoted ``": "`` — which YAML takes as a nested mapping. The key check
18 was a substring search, so it passed happily on a file no parser would accept.
192. **Bash blocks parse** — every fenced ``bash`` block is valid shell (``bash -n``),
20 with ``<placeholder>`` spans neutralised first since they aren't real syntax.
213. **Scripts exist** — every ``scripts/<name>.py`` a block invokes is shipped.
224. **Flags exist** — every ``--flag`` passed to a bundled script is one that script's
23 ``argparse`` actually accepts. This is the check that catches a renamed CLI, the
24 most likely silent breakage.
255. **Slash commands exist** — every ``/rhiza:<name>`` referenced resolves to a real
26 command, so a removed one can't linger in another command's prose.
276. **allowed-tools covers the binaries used** — a command that runs ``git`` in a block
28 must be permitted to, or the user gets a permission prompt mid-flow.
30Usage:
31 uv run --python 3.12 --no-project python \
32 scripts/check_command_contracts.py [--root DIR]
34Exits 0 when every contract holds, 1 (listing each violation) otherwise.
35"""
37from __future__ import annotations
39import argparse
40import re
41import shutil
42import subprocess # nosec B404
43import sys
44from pathlib import Path
46_FRONTMATTER_KEYS = ("description", "argument-hint", "allowed-tools")
47_BASH_BLOCK = re.compile(r"```bash\n(.*?)```", re.S)
48# The script path is usually quoted — `"${CLAUDE_PLUGIN_ROOT}/scripts/x.py" --flag` —
49# so the closing quote must be consumed before the arguments, or the argument capture
50# comes back empty and every flag goes unchecked.
51_SCRIPT_CALL = re.compile(r"scripts/([a-z_]+)\.py[\"']?((?:\s+[^\n`]*)?)")
52_SLASH_COMMAND = re.compile(r"/rhiza:([a-z-]+)")
53# The documented way one command delegates to another. Case-insensitive: the phrase
54# is often capitalised at the start of a sentence or a bullet.
55_SKILL_INVOCATION = re.compile(r"invoke the `([a-z-]+)` command via the Skill tool", re.IGNORECASE)
56_ADD_ARGUMENT = re.compile(r"add_argument\(")
57_FLAG = re.compile(r"--[a-zA-Z][a-zA-Z0-9-]*")
58# `Bash(git*)` and friends, from a command's allowed-tools line.
59_ALLOWED_BASH = re.compile(r"Bash\(([a-zA-Z0-9_.-]+)\*?\)")
61# Placeholders that are prose, not shell: `<github|gitlab>`, `<BODY>`, `<TARGET>`.
62_ANGLE_PLACEHOLDER = re.compile(r"<[A-Za-z][A-Za-z0-9|_ -]*>")
64# Binaries a block may call without being declared: shell builtins and control words.
65_SHELL_BUILTINS = frozenset(
66 {"cd", "echo", "test", "if", "then", "else", "fi", "for", "do", "done", "printf", "export"}
67)
70def frontmatter(text: str) -> str | None:
71 """Return the YAML frontmatter block, or None when the file has none."""
72 if not text.startswith("---\n"):
73 return None
74 end = text.find("\n---\n", 4)
75 return None if end == -1 else text[4:end]
78def script_flags(path: Path) -> set[str]:
79 """Return every ``--flag`` the script at *path* declares via argparse.
81 Scans a window after each ``add_argument(`` rather than parsing the call, which
82 handles both the single-line and multi-line styles used across the scripts without
83 importing them (importing would run module-level code).
84 """
85 source = path.read_text()
86 flags: set[str] = set()
87 for match in _ADD_ARGUMENT.finditer(source):
88 window = source[match.end() : match.end() + 400]
89 # Stop at the next add_argument so a long call can't leak into the next one.
90 nxt = window.find("add_argument(")
91 if nxt != -1:
92 window = window[:nxt]
93 flags |= set(_FLAG.findall(window))
94 return flags
97def bash_blocks(text: str) -> list[str]:
98 """Return the fenced ``bash`` blocks in *text*."""
99 return _BASH_BLOCK.findall(text)
102def unquoted_mapping_colon(value: str) -> bool:
103 """Does *value* contain a `: ` that YAML would read as a nested mapping?
105 A plain (unquoted) YAML scalar may not contain ``": "``. Writing
106 ``description: procedures under prompts/: install-uv`` therefore makes the whole
107 frontmatter unparseable — and five of seven commands shipped exactly that, because
108 the key check below was a substring search that never tried to parse anything.
109 """
110 stripped = value.strip()
111 if not stripped or stripped[0] in "'\"|>": # quoted or a block scalar: fine
112 return False
113 return ": " in stripped
116def parse_frontmatter(block: str) -> tuple[dict[str, str], list[str]]:
117 """Parse a simple ``key: value`` frontmatter block; return (mapping, problems).
119 Deliberately a strict subset rather than a YAML library: the scripts are
120 stdlib-only, and the failure being guarded against is precisely a value that a real
121 YAML parser would choke on.
122 """
123 mapping: dict[str, str] = {}
124 problems: list[str] = []
125 for number, line in enumerate(block.splitlines(), start=2):
126 if not line.strip() or line.startswith("#"):
127 continue
128 if line[0].isspace(): # a continuation of the previous value
129 continue
130 if ":" not in line:
131 problems.append(f"line {number} is not `key: value`: {line[:60]!r}")
132 continue
133 key, _, value = line.partition(":")
134 mapping[key.strip()] = value.strip()
135 if unquoted_mapping_colon(value):
136 problems.append(
137 f"`{key.strip()}` contains an unquoted `: `, which YAML reads as a "
138 "nested mapping — quote the value or rewrite the colon"
139 )
140 return mapping, problems
143def check_frontmatter(rel: str, text: str, *, is_command: bool) -> list[str]:
144 """Rule 1: commands declare parseable frontmatter; procedures declare none."""
145 block = frontmatter(text)
146 if not is_command:
147 if block is not None:
148 return [f"{rel}: has frontmatter, but a procedure is not invocable"]
149 return []
150 if block is None:
151 return [f"{rel}: missing frontmatter"]
153 mapping, problems = parse_frontmatter(block)
154 violations = [f"{rel}: {problem}" for problem in problems]
155 violations += [
156 f"{rel}: frontmatter has no `{key}`" for key in _FRONTMATTER_KEYS if key not in mapping
157 ]
158 return violations
161def check_bash_syntax(rel: str, blocks: list[str]) -> list[str]:
162 """Rule 2: every bash block is syntactically valid shell."""
163 bash = shutil.which("bash")
164 if bash is None: # pragma: no cover - bash is present everywhere this runs
165 return []
166 violations = []
167 for i, block in enumerate(blocks, 1):
168 cleaned = _ANGLE_PLACEHOLDER.sub("PLACEHOLDER", block)
169 result = subprocess.run( # nosec B603
170 [bash, "-n"], input=cleaned, capture_output=True, text=True, check=False
171 )
172 if result.returncode != 0:
173 detail = result.stderr.strip().splitlines()[-1] if result.stderr.strip() else "?"
174 violations.append(f"{rel}: bash block {i} is not valid shell — {detail}")
175 return violations
178def check_script_calls(rel: str, blocks: list[str], scripts_dir: Path) -> list[str]:
179 """Rules 3 and 4: invoked scripts exist, and their flags are real.
181 Backslash continuations are joined first. Nearly every invocation in this prose is
182 wrapped across lines, so without joining, only the first line's flags are seen —
183 and a bad flag on a later line would pass unnoticed.
184 """
185 violations = []
186 for raw in blocks:
187 block = raw.replace("\\\n", " ")
188 for name, args in _SCRIPT_CALL.findall(block):
189 path = scripts_dir / f"{name}.py"
190 if not path.is_file():
191 violations.append(f"{rel}: invokes scripts/{name}.py, which does not exist")
192 continue
193 declared = script_flags(path)
194 for flag in _FLAG.findall(args):
195 if flag not in declared:
196 violations.append(
197 f"{rel}: passes {flag} to scripts/{name}.py, which does not accept it"
198 )
199 return violations
202def check_slash_commands(rel: str, text: str, commands_dir: Path) -> list[str]:
203 """Rule 5: every command we tell the model to *invoke* exists.
205 Only invocations are checked, not mentions. Prose legitimately refers to retired
206 commands to explain history ("the view the retired ``/rhiza:tree`` gave",
207 "absorbs ``/rhiza:validate``"), and flagging those would push authors to delete
208 useful context. What must not dangle is an instruction to run something.
209 """
210 invoked = set(_SKILL_INVOCATION.findall(text))
211 for block in bash_blocks(text):
212 invoked |= set(_SLASH_COMMAND.findall(block))
213 return [
214 f"{rel}: tells the model to invoke `{name}`, which is not a command"
215 for name in sorted(invoked)
216 if not (commands_dir / f"{name}.md").is_file()
217 ]
220def _leading_binaries(blocks: list[str]) -> set[str]:
221 """Return the binary invoked at the start of each command line across *blocks*.
223 Backslash continuations are joined first: without that, the second line of a
224 wrapped invocation looks like a command of its own and every flag reads as a
225 binary name.
226 """
227 found: set[str] = set()
228 for block in blocks:
229 joined = block.replace("\\\n", " ")
230 for raw in joined.splitlines():
231 line = raw.strip()
232 if not line or line.startswith("#"):
233 continue
234 word = line.split()[0]
235 # Skip variable assignments (`BRANCH=...`) and anything flag-shaped.
236 if word.startswith("-") or "=" in word:
237 continue
238 found.add(word)
239 return found
242def check_allowed_tools(rel: str, text: str, blocks: list[str]) -> list[str]:
243 """Rule 6: a command may run the binaries its blocks actually call."""
244 block_text = frontmatter(text)
245 if block_text is None:
246 return []
247 line = next((ln for ln in block_text.splitlines() if ln.startswith("allowed-tools:")), None)
248 if line is None:
249 return []
250 permitted = set(_ALLOWED_BASH.findall(line))
251 return [
252 f"{rel}: runs `{binary}` but allowed-tools has no Bash({binary}*)"
253 for binary in sorted(_leading_binaries(blocks))
254 if binary not in permitted and binary not in _SHELL_BUILTINS
255 ]
258def check_contracts(root: Path) -> list[str]:
259 """Run every rule over the plugin at *root*; return all violations."""
260 commands_dir, prompts_dir, scripts_dir = root / "commands", root / "prompts", root / "scripts"
261 violations: list[str] = []
263 for directory, is_command in ((commands_dir, True), (prompts_dir, False)):
264 if not directory.is_dir():
265 continue
266 for path in sorted(directory.glob("*.md")):
267 rel = f"{directory.name}/{path.name}"
268 text = path.read_text()
269 blocks = bash_blocks(text)
270 violations += check_frontmatter(rel, text, is_command=is_command)
271 violations += check_bash_syntax(rel, blocks)
272 violations += check_script_calls(rel, blocks, scripts_dir)
273 violations += check_slash_commands(rel, text, commands_dir)
274 if is_command:
275 violations += check_allowed_tools(rel, text, blocks)
276 return violations
279def main(argv: list[str] | None = None) -> int:
280 """Entry point: check every command contract and return an exit code."""
281 parser = argparse.ArgumentParser(description="Check the plugin's prose command contracts.")
282 parser.add_argument("--root", default=".", help="Plugin root (default: current directory).")
283 args = parser.parse_args(argv)
285 root = Path(args.root).resolve()
286 violations = check_contracts(root)
287 if violations:
288 print("Command-contract check failed:", file=sys.stderr)
289 for violation in violations:
290 print(f" ✗ {violation}", file=sys.stderr)
291 return 1
293 count = len(list((root / "commands").glob("*.md"))) + len(list((root / "prompts").glob("*.md")))
294 print(f"command contracts hold ({count} file(s) checked)")
295 return 0
298if __name__ == "__main__":
299 raise SystemExit(main())