Coverage for plugin/scripts/_doc_examples_readme.py: 100%
118 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-09-15 14:46 +0000
1#!/usr/bin/env python3
2"""The README half of `check_doc_examples.py`: fenced blocks, checked by language.
4`markdownlint` checks that a fence is well-formed *markdown* and says nothing about
5whether the shell inside it parses or the Python inside it runs. A README's examples are
6the first thing a newcomer executes and the last thing anyone re-reads, so they rot
7silently — which is why the rhiza template ships `test_readme.py` and
8`test_readme_validation.py` into every managed repo. This module is that pair, minus the
9`.rhiza/` directory, so an unmanaged repo gets the same check.
11The conventions are the template's, deliberately: the `+RHIZA_SKIP` fence flag is spelled
12the same, directory trees and comment-only blocks are skipped the same way, and `python`
13fences are diffed against the following ```result``` block. A repo that adopts rhiza later
14keeps whatever verdict it had here.
16**Shell fences are parsed, never executed** — under any flag. A README's shell is
17routinely destructive-adjacent (`make clean`, `git push`, `rm -rf`), and a fence that
18cannot parse is a documentation bug whether or not anyone runs it.
19"""
21from __future__ import annotations
23import re
24import shutil
25import subprocess # nosec B404
26import sys
27from pathlib import Path
28from typing import Any, NamedTuple
30# A fenced block: the info string (language plus any flags) and the body. The closing
31# fence has to start a line, so an indented fence inside a block doesn't end it early.
32_FENCE = re.compile(r"^```([^\n`]*)\n(.*?)^```", re.S | re.M)
34# The template's own marker for a fence that is illustrative rather than runnable, spelled
35# exactly as `.rhiza/tests/test_readme.py` spells it. Sharing the spelling is the point: a
36# fence a repo opted out of upstream is opted out here too.
37SKIP_FLAG = "+RHIZA_SKIP"
39# Box-drawing characters mean the fence is a directory tree wearing a `bash` label.
40_TREE_MARKERS = ("├──", "└──", "│")
42_SHELL_LANGS = frozenset({"bash", "sh", "shell", "zsh"})
43_PYTHON_LANGS = frozenset({"python", "py"})
44_RESULT_LANG = "result"
47class Fence(NamedTuple):
48 """One fenced code block: its language, its flags, its body and where it starts."""
50 language: str
51 flags: str
52 body: str
53 line: int
56def fences(text: str) -> list[Fence]:
57 """Return every fenced block in *text*, in document order."""
58 found: list[Fence] = []
59 for match in _FENCE.finditer(text):
60 language, _, flags = match.group(1).strip().partition(" ")
61 line = text.count("\n", 0, match.start()) + 1
62 found.append(Fence(language.lower(), flags.strip(), match.group(2), line))
63 return found
66def should_skip(flags: str) -> bool:
67 """Is this fence marked `+RHIZA_SKIP`?"""
68 return SKIP_FLAG in flags
71def shell_skip_reason(body: str) -> str | None:
72 """Why *body* isn't shell worth parsing, or None when it is.
74 Two shapes wear a ``bash`` label without being runnable shell: a directory tree drawn
75 with box characters, and a block of nothing but comments. Either would pass or fail
76 ``bash -n`` for reasons that say nothing about the documentation.
77 """
78 if any(marker in body for marker in _TREE_MARKERS):
79 return "directory tree, not shell"
80 lines = [line.strip() for line in body.splitlines() if line.strip()]
81 if not [line for line in lines if not line.startswith("#")]:
82 return "comments only"
83 return None
86def last_line(text: str, fallback: str) -> str:
87 """Return the last non-empty line of *text*, or *fallback* when there is none."""
88 stripped = text.strip()
89 return stripped.splitlines()[-1] if stripped else fallback
92def check_shell(fence: Fence) -> tuple[str, str]:
93 """Parse a shell fence with ``bash -n``; return (status, detail).
95 Parsing only — never execution. See this module's docstring for why.
96 """
97 bash = shutil.which("bash")
98 if bash is None: # pragma: no cover - bash is present everywhere this runs
99 return "skipped", "bash not available"
100 result = subprocess.run( # nosec B603
101 [bash, "-n"], input=fence.body, capture_output=True, text=True, check=False
102 )
103 if result.returncode == 0:
104 return "ok", ""
105 return "failed", last_line(result.stderr, "bash -n reported a syntax error")
108def check_python(fence: Fence) -> tuple[str, str]:
109 """Compile a Python fence without running it; return (status, detail)."""
110 try:
111 compile(fence.body, f"<readme:{fence.line}>", "exec")
112 except SyntaxError as exc:
113 return "failed", f"{exc.msg} (line {exc.lineno})"
114 return "ok", ""
117def check_fence(fence: Fence) -> tuple[str, str]:
118 """Check one fence as far as its language allows; return (status, detail)."""
119 if should_skip(fence.flags):
120 return "skipped", f"{SKIP_FLAG} on the fence"
121 if fence.language in _SHELL_LANGS:
122 reason = shell_skip_reason(fence.body)
123 return ("skipped", reason) if reason else check_shell(fence)
124 if fence.language in _PYTHON_LANGS:
125 return check_python(fence)
126 if fence.language == _RESULT_LANG:
127 return "skipped", "expected output for a python fence"
128 if not fence.language:
129 return "untagged", "no language on the fence — nothing can check it"
130 return "skipped", f"`{fence.language}` fences are not checkable"
133def run_python_fences(readme: Path, blocks: list[Fence]) -> dict[str, Any]:
134 """Execute the README's Python fences and diff the output against its ``result`` blocks.
136 The fences are concatenated and run as one program, exactly as the template's
137 `test_readme_validation.py` does it: a README's examples are usually one session split
138 across prose, so running each in isolation would break every one that builds on the
139 last.
141 **With no ``result`` block, only the exit status is asserted.** The template compares
142 against the empty string there, which fails any example that prints — right for a repo
143 whose README is expected to carry them, wrong as a general rule, and this script runs
144 against repos that never adopted the convention. Undocumented output is a note, not a
145 failure.
146 """
147 code = "".join(
148 f.body for f in blocks if f.language in _PYTHON_LANGS and not should_skip(f.flags)
149 )
150 expected = "".join(f.body for f in blocks if f.language == _RESULT_LANG)
151 if not code.strip():
152 return {"ran": False, "violations": [], "notes": ["no executable python fence"]}
154 result = subprocess.run( # nosec B603
155 [sys.executable, "-c", code],
156 cwd=str(readme.parent),
157 capture_output=True,
158 text=True,
159 check=False,
160 )
161 violations: list[str] = []
162 notes: list[str] = []
163 matched: bool | None = None
164 if result.returncode != 0:
165 detail = last_line(result.stderr, "no stderr")
166 violations.append(f"{readme.name}: python fences exited {result.returncode} — {detail}")
167 elif expected.strip():
168 matched = result.stdout.strip() == expected.strip()
169 if not matched:
170 violations.append(
171 f"{readme.name}: python fence output does not match its ```result``` block "
172 f"(expected {expected.strip()[:60]!r}, got {result.stdout.strip()[:60]!r})"
173 )
174 else:
175 notes.append("python fences ran, but no ```result``` block documents their output")
176 return {
177 "ran": True,
178 "returncode": result.returncode,
179 "matched": matched,
180 "violations": violations,
181 "notes": notes,
182 }
185def _untagged_note(checked: list[dict[str, Any]]) -> list[str]:
186 """Name the fences that carry no language, which nothing can check."""
187 untagged = [block for block in checked if block["status"] == "untagged"]
188 if not untagged:
189 return []
190 return [
191 f"{len(untagged)} fence(s) carry no language, so nothing can check them: "
192 + ", ".join(f"line {block['line']}" for block in untagged)
193 ]
196def readme_report(readme: Path, *, run: bool) -> dict[str, Any]:
197 """Check every fence in *readme*; return a summary dict."""
198 if not readme.is_file():
199 return {
200 "path": str(readme),
201 "present": False,
202 "blocks": [],
203 "violations": [],
204 "notes": [f"no {readme.name} — README examples are out of scope, not failing"],
205 }
207 checked: list[dict[str, Any]] = []
208 violations: list[str] = []
209 blocks = fences(readme.read_text(encoding="utf-8"))
210 for fence in blocks:
211 status, detail = check_fence(fence)
212 checked.append(
213 {
214 "line": fence.line,
215 "language": fence.language or "(none)",
216 "status": status,
217 "detail": detail,
218 }
219 )
220 if status == "failed":
221 violations.append(f"{readme.name}:{fence.line}: {fence.language} fence — {detail}")
223 report: dict[str, Any] = {
224 "path": str(readme),
225 "present": True,
226 "blocks": checked,
227 "violations": violations,
228 "notes": _untagged_note(checked),
229 }
230 if run:
231 execution = run_python_fences(readme, blocks)
232 report["execution"] = execution
233 violations.extend(execution["violations"])
234 report["notes"].extend(execution["notes"])
235 return report
238def print_report(readme: dict[str, Any]) -> None:
239 """Print the README half of a report as text."""
240 if not readme["present"]:
241 print(f"{'unavailable':<12} README fences ({readme['path']} is missing)")
242 return
243 counts: dict[str, int] = {}
244 for block in readme["blocks"]:
245 counts[block["status"]] = counts.get(block["status"], 0) + 1
246 tally = ", ".join(f"{count} {status}" for status, count in sorted(counts.items()))
247 print(f"{'readme':<12} {readme['path']}: {len(readme['blocks'])} fence(s) — {tally or 'none'}")
248 for block in readme["blocks"]:
249 detail = f" — {block['detail']}" if block["detail"] else ""
250 print(f"{block['status']:<12} {readme['path']}:{block['line']} {block['language']}{detail}")
251 execution = readme.get("execution")
252 if execution is not None and execution["ran"]:
253 print(
254 f"{'ran':<12} python fences exited {execution['returncode']}, "
255 f"output matched: {execution['matched']}"
256 )