Coverage for plugin/scripts/_doc_examples_readme.py: 100%
170 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-10-26 04:21 +0000
1#!/usr/bin/env python3
2"""The README half of `check_doc_examples.py`: fenced blocks, checked by language.
4`markdownlint` checks that a fence is well-formed *markdown* and says nothing about
5whether the shell inside it parses or the Python inside it runs. A README's examples are
6the first thing a newcomer executes and the last thing anyone re-reads, so they rot
7silently — which is why the rhiza template ships `test_readme.py` and
8`test_readme_validation.py` into every managed repo. This module is that pair, minus the
9`.rhiza/` directory, so an unmanaged repo gets the same check.
11The conventions are the template's, deliberately: the `+RHIZA_SKIP` fence flag is spelled
12the same, directory trees and comment-only blocks are skipped the same way, and `pycon`
13fences are doctest transcripts — `>>>`/`...` prompts with the expected output inline,
14every unskipped one in the file joined in document order and run as one doctest under
15`ELLIPSIS`. A repo that adopts rhiza later keeps whatever verdict it had here.
17The older convention — a `python` fence diffed against the ```result``` block after it —
18is still understood, so a README that hasn't migrated keeps its verdict. New examples
19should be `pycon`: the output sits next to the line that produced it, and a mismatch
20names the example rather than the whole program.
22**Shell fences are parsed, never executed** — under any flag. A README's shell is
23routinely destructive-adjacent (`make clean`, `git push`, `rm -rf`), and a fence that
24cannot parse is a documentation bug whether or not anyone runs it.
25"""
27from __future__ import annotations
29import doctest
30import json
31import re
32import shutil
33import subprocess # nosec B404
34import sys
35from pathlib import Path
36from typing import Any, NamedTuple
38# A fenced block: the info string (language plus any flags) and the body. The closing
39# fence has to start a line, so an indented fence inside a block doesn't end it early.
40_FENCE = re.compile(r"^```([^\n`]*)\n(.*?)^```", re.S | re.M)
42# The template's own marker for a fence that is illustrative rather than runnable, spelled
43# exactly as `.rhiza/tests/test_readme.py` spells it. Sharing the spelling is the point: a
44# fence a repo opted out of upstream is opted out here too.
45SKIP_FLAG = "+RHIZA_SKIP"
47# Box-drawing characters mean the fence is a directory tree wearing a `bash` label.
48_TREE_MARKERS = ("├──", "└──", "│")
50_SHELL_LANGS = frozenset({"bash", "sh", "shell", "zsh"})
51_PYTHON_LANGS = frozenset({"python", "py"})
52_PYCON_LANG = "pycon"
53_RESULT_LANG = "result"
55# What the child process runs for the `pycon` fences: the joined transcript arrives on
56# stdin, one doctest runs over it, and a JSON verdict goes back on stdout. A subprocess
57# rather than an in-process run for the same reason the `python` fences get one — a
58# README's examples import the repo's code and may mutate `sys.path`, and neither should
59# leak into the checker.
60_DOCTEST_CHILD = """
61import doctest, io, json, sys
62text = sys.stdin.read()
63test = doctest.DocTestParser().get_doctest(
64 text, {"__name__": "__main__"}, sys.argv[1], sys.argv[1], 0
65)
66buf = io.StringIO()
67runner = doctest.DocTestRunner(optionflags=doctest.ELLIPSIS)
68result = runner.run(test, out=buf.write, clear_globs=True)
69print()
70print(json.dumps({"failed": result.failed, "attempted": result.attempted,
71 "report": buf.getvalue()}))
72"""
75class Fence(NamedTuple):
76 """One fenced code block: its language, its flags, its body and where it starts."""
78 language: str
79 flags: str
80 body: str
81 line: int
84def fences(text: str) -> list[Fence]:
85 """Return every fenced block in *text*, in document order."""
86 found: list[Fence] = []
87 for match in _FENCE.finditer(text):
88 language, _, flags = match.group(1).strip().partition(" ")
89 line = text.count("\n", 0, match.start()) + 1
90 found.append(Fence(language.lower(), flags.strip(), match.group(2), line))
91 return found
94def should_skip(flags: str) -> bool:
95 """Is this fence marked `+RHIZA_SKIP`?"""
96 return SKIP_FLAG in flags
99def shell_skip_reason(body: str) -> str | None:
100 """Why *body* isn't shell worth parsing, or None when it is.
102 Two shapes wear a ``bash`` label without being runnable shell: a directory tree drawn
103 with box characters, and a block of nothing but comments. Either would pass or fail
104 ``bash -n`` for reasons that say nothing about the documentation.
105 """
106 if any(marker in body for marker in _TREE_MARKERS):
107 return "directory tree, not shell"
108 lines = [line.strip() for line in body.splitlines() if line.strip()]
109 if not [line for line in lines if not line.startswith("#")]:
110 return "comments only"
111 return None
114def last_line(text: str, fallback: str) -> str:
115 """Return the last non-empty line of *text*, or *fallback* when there is none."""
116 stripped = text.strip()
117 return stripped.splitlines()[-1] if stripped else fallback
120def check_shell(fence: Fence) -> tuple[str, str]:
121 """Parse a shell fence with ``bash -n``; return (status, detail).
123 Parsing only — never execution. See this module's docstring for why.
124 """
125 bash = shutil.which("bash")
126 if bash is None: # pragma: no cover - bash is present everywhere this runs
127 return "skipped", "bash not available"
128 result = subprocess.run( # nosec B603
129 [bash, "-n"], input=fence.body, capture_output=True, text=True, check=False
130 )
131 if result.returncode == 0:
132 return "ok", ""
133 return "failed", last_line(result.stderr, "bash -n reported a syntax error")
136def check_python(fence: Fence) -> tuple[str, str]:
137 """Compile a Python fence without running it; return (status, detail)."""
138 try:
139 compile(fence.body, f"<readme:{fence.line}>", "exec")
140 except SyntaxError as exc:
141 return "failed", f"{exc.msg} (line {exc.lineno})"
142 return "ok", ""
145def check_pycon(fence: Fence) -> tuple[str, str]:
146 """Parse a ``pycon`` transcript and compile each example without running it.
148 Syntax only, like `check_python`: a transcript whose prompts are malformed or whose
149 source doesn't compile is a documentation bug before anyone executes it.
150 """
151 try:
152 examples = doctest.DocTestParser().get_examples(fence.body, f"<readme:{fence.line}>")
153 except ValueError as exc:
154 return "failed", str(exc)
155 if not examples:
156 return "skipped", "no `>>>` prompt in the fence"
157 for example in examples:
158 try:
159 compile(example.source, f"<readme:{fence.line}>", "single")
160 except SyntaxError as exc:
161 where = fence.line + 1 + example.lineno
162 return "failed", f"{exc.msg} (README line {where})"
163 return "ok", ""
166def check_fence(fence: Fence) -> tuple[str, str]:
167 """Check one fence as far as its language allows; return (status, detail)."""
168 if should_skip(fence.flags):
169 return "skipped", f"{SKIP_FLAG} on the fence"
170 if fence.language in _SHELL_LANGS:
171 reason = shell_skip_reason(fence.body)
172 return ("skipped", reason) if reason else check_shell(fence)
173 if fence.language in _PYTHON_LANGS:
174 return check_python(fence)
175 if fence.language == _PYCON_LANG:
176 return check_pycon(fence)
177 if fence.language == _RESULT_LANG:
178 return "skipped", "expected output for a python fence"
179 if not fence.language:
180 return "untagged", "no language on the fence — nothing can check it"
181 return "skipped", f"`{fence.language}` fences are not checkable"
184def run_python_fences(readme: Path, blocks: list[Fence]) -> dict[str, Any]:
185 """Execute the README's Python fences and diff the output against its ``result`` blocks.
187 The fences are concatenated and run as one program, exactly as the template's
188 `test_readme_validation.py` does it: a README's examples are usually one session split
189 across prose, so running each in isolation would break every one that builds on the
190 last.
192 **With no ``result`` block, only the exit status is asserted.** The template compares
193 against the empty string there, which fails any example that prints — right for a repo
194 whose README is expected to carry them, wrong as a general rule, and this script runs
195 against repos that never adopted the convention. Undocumented output is a note, not a
196 failure.
197 """
198 code = "".join(
199 f.body for f in blocks if f.language in _PYTHON_LANGS and not should_skip(f.flags)
200 )
201 expected = "".join(f.body for f in blocks if f.language == _RESULT_LANG)
202 if not code.strip():
203 return {"ran": False, "violations": [], "notes": ["no executable python fence"]}
205 result = subprocess.run( # nosec B603
206 [sys.executable, "-c", code],
207 cwd=str(readme.parent),
208 capture_output=True,
209 text=True,
210 check=False,
211 )
212 violations: list[str] = []
213 notes: list[str] = []
214 matched: bool | None = None
215 if result.returncode != 0:
216 detail = last_line(result.stderr, "no stderr")
217 violations.append(f"{readme.name}: python fences exited {result.returncode} — {detail}")
218 elif expected.strip():
219 matched = result.stdout.strip() == expected.strip()
220 if not matched:
221 violations.append(
222 f"{readme.name}: python fence output does not match its ```result``` block "
223 f"(expected {expected.strip()[:60]!r}, got {result.stdout.strip()[:60]!r})"
224 )
225 else:
226 notes.append("python fences ran, but no ```result``` block documents their output")
227 return {
228 "ran": True,
229 "returncode": result.returncode,
230 "matched": matched,
231 "violations": violations,
232 "notes": notes,
233 }
236def _first_failed_example(report: str) -> str:
237 """Return the source line of the first failing example in a doctest *report*.
239 That line, not the header doctest prints above it, is what a reader searches the
240 README for.
241 """
242 lines = report.splitlines()
243 for index, line in enumerate(lines[:-1]):
244 if line.strip() == "Failed example:":
245 return lines[index + 1].strip()
246 return last_line(report, "see the doctest report")
249def run_pycon_fences(readme: Path, blocks: list[Fence]) -> dict[str, Any]:
250 """Doctest the README's ``pycon`` fences as one transcript; return a summary dict.
252 Every unskipped ``pycon`` fence is joined in document order and run as a single
253 doctest with ``ELLIPSIS``, in a subprocess whose working directory is the README's
254 own — the same semantics as the template's `test_readme_validation.py`. One doctest
255 rather than one per fence, because a README's examples are one session split across
256 prose: the second fence uses the imports of the first.
257 """
258 text = "".join(
259 f.body if f.body.endswith("\n") else f.body + "\n"
260 for f in blocks
261 if f.language == _PYCON_LANG and not should_skip(f.flags)
262 )
263 if not doctest.DocTestParser().get_examples(text):
264 return {"ran": False, "violations": [], "notes": ["no executable pycon fence"]}
266 proc = subprocess.run( # nosec B603
267 [sys.executable, "-c", _DOCTEST_CHILD, readme.name],
268 input=text,
269 cwd=str(readme.parent),
270 capture_output=True,
271 text=True,
272 check=False,
273 )
274 try:
275 # The last line only: an example that writes to fd 1 behind doctest's back (a
276 # subprocess, say) would otherwise corrupt the verdict.
277 verdict = json.loads(last_line(proc.stdout, ""))
278 except ValueError:
279 detail = last_line(proc.stderr, "no stderr")
280 return {
281 "ran": True,
282 "failed": None,
283 "attempted": None,
284 "violations": [f"{readme.name}: pycon doctest exited {proc.returncode} — {detail}"],
285 "notes": [],
286 }
287 violations: list[str] = []
288 if verdict["failed"]:
289 violations.append(
290 f"{readme.name}: pycon fences — {verdict['failed']} of {verdict['attempted']} "
291 f"example(s) failed, first: {_first_failed_example(verdict['report'])}"
292 )
293 return {
294 "ran": True,
295 "failed": verdict["failed"],
296 "attempted": verdict["attempted"],
297 "report": verdict["report"],
298 "violations": violations,
299 "notes": [],
300 }
303def _untagged_note(checked: list[dict[str, Any]]) -> list[str]:
304 """Name the fences that carry no language, which nothing can check."""
305 untagged = [block for block in checked if block["status"] == "untagged"]
306 if not untagged:
307 return []
308 return [
309 f"{len(untagged)} fence(s) carry no language, so nothing can check them: "
310 + ", ".join(f"line {block['line']}" for block in untagged)
311 ]
314def readme_report(readme: Path, *, run: bool) -> dict[str, Any]:
315 """Check every fence in *readme*; return a summary dict."""
316 if not readme.is_file():
317 return {
318 "path": str(readme),
319 "present": False,
320 "blocks": [],
321 "violations": [],
322 "notes": [f"no {readme.name} — README examples are out of scope, not failing"],
323 }
325 checked: list[dict[str, Any]] = []
326 violations: list[str] = []
327 blocks = fences(readme.read_text(encoding="utf-8"))
328 for fence in blocks:
329 status, detail = check_fence(fence)
330 checked.append(
331 {
332 "line": fence.line,
333 "language": fence.language or "(none)",
334 "status": status,
335 "detail": detail,
336 }
337 )
338 if status == "failed":
339 violations.append(f"{readme.name}:{fence.line}: {fence.language} fence — {detail}")
341 report: dict[str, Any] = {
342 "path": str(readme),
343 "present": True,
344 "blocks": checked,
345 "violations": violations,
346 "notes": _untagged_note(checked),
347 }
348 if run:
349 _run_fences(readme, blocks, report)
350 return report
353def _run_fences(readme: Path, blocks: list[Fence], report: dict[str, Any]) -> None:
354 """Execute whichever convention *blocks* use, recording each under its own key.
356 ``pycon`` is run when present; the legacy ``python``/``result`` pair is run when any
357 ``python`` fence is present, or when there is no ``pycon`` fence either — so a README
358 with no examples at all still gets the one note that says so.
359 """
360 has_pycon = any(f.language == _PYCON_LANG for f in blocks)
361 if has_pycon:
362 report["doctest"] = run_pycon_fences(readme, blocks)
363 if not has_pycon or any(f.language in _PYTHON_LANGS for f in blocks):
364 report["execution"] = run_python_fences(readme, blocks)
365 for key in ("doctest", "execution"):
366 if key in report:
367 report["violations"].extend(report[key]["violations"])
368 report["notes"].extend(report[key]["notes"])
371def print_report(readme: dict[str, Any]) -> None:
372 """Print the README half of a report as text."""
373 if not readme["present"]:
374 print(f"{'unavailable':<12} README fences ({readme['path']} is missing)")
375 return
376 counts: dict[str, int] = {}
377 for block in readme["blocks"]:
378 counts[block["status"]] = counts.get(block["status"], 0) + 1
379 tally = ", ".join(f"{count} {status}" for status, count in sorted(counts.items()))
380 print(f"{'readme':<12} {readme['path']}: {len(readme['blocks'])} fence(s) — {tally or 'none'}")
381 for block in readme["blocks"]:
382 detail = f" — {block['detail']}" if block["detail"] else ""
383 print(f"{block['status']:<12} {readme['path']}:{block['line']} {block['language']}{detail}")
384 _print_runs(readme)
387def _print_runs(readme: dict[str, Any]) -> None:
388 """Print one line per execution pass that actually ran over the README's fences."""
389 execution = readme.get("execution")
390 if execution is not None and execution["ran"]:
391 print(
392 f"{'ran':<12} python fences exited {execution['returncode']}, "
393 f"output matched: {execution['matched']}"
394 )
395 doctest_run = readme.get("doctest")
396 if doctest_run is not None and doctest_run["ran"]:
397 print(
398 f"{'ran':<12} pycon fences as one doctest: "
399 f"{doctest_run['attempted']} example(s), {doctest_run['failed']} failed"
400 )