Coverage for plugin/scripts/check_doc_examples.py: 100%

1#!/usr/bin/env python3 

2"""Check a repo's *examples* — the doctests in its docstrings and the fences in its README. 

3 

4`/rhiza:quality` has always measured docstring **coverage** and never docstring **truth**. 

5`interrogate` answers "is there a docstring?"; nothing answered "does the example inside it 

6still evaluate to what it claims?". The same hole sits under the README: `markdownlint` 

7checks that a fence is well-formed markdown and says nothing about whether the shell inside 

8it parses or the Python inside it runs. Both are documentation that fails silently — it 

9keeps rendering long after it stopped being true, and the person it misleads is a newcomer 

10running the quickstart. 

11 

12The rhiza template already closes this in a *managed* repo: `.rhiza/tests/` ships 

13`test_docstrings.py`, `test_readme.py` and `test_readme_validation.py`, all of which 

14`make rhiza-test` runs. This script is that same trio for **any** repo — the degraded-mode 

15case above all, where there is no `.rhiza/` and so nothing checks any of it. Its 

16conventions mirror the template's, `+RHIZA_SKIP` included, so a repo that adopts rhiza 

17later keeps the verdict it had here. 

18 

19The two halves live in `_doc_examples_source.py` and `_doc_examples_readme.py`; this file 

20is the dispatcher, the combined verdict and the CLI. Each half degrades on its own: a repo 

21with no source root still gets its README checked, and vice versa. 

22 

23`--run` adds execution, and is opt-in for a reason: running examples means importing the 

24repo's modules and executing its README's Python, which runs whatever module-level code 

25they carry. That is the same trust boundary `make test` already crosses, but `/quality` is 

26an *assessment* command, so it is never crossed unasked. Shell fences are never executed 

27under any flag. 

28 

29Usage: 

30 uv run --python 3.12 --no-project python \ 

31 scripts/check_doc_examples.py [--target-dir DIR] [--source-root DIR] \ 

32 [--readme FILE] [--run] [--json] 

33 

34Exit codes: 

35 0 every example that could be checked holds 

36 1 an example is broken (bad syntax, a failing doctest, output that doesn't match) 

37 2 nothing was checkable — no source root and no README 

38""" 

39 

40from __future__ import annotations 

41 

42import argparse 

43import json 

44import sys 

45from pathlib import Path 

46from typing import Any 

47 

48sys.path.insert(0, str(Path(__file__).resolve().parent)) 

49from _doc_examples_readme import print_report as print_readme # noqa: E402 

50from _doc_examples_readme import readme_report # noqa: E402 

51from _doc_examples_source import docstring_report # noqa: E402 

52from _doc_examples_source import print_report as print_docstrings # noqa: E402 

53 

54EXIT_OK = 0 

55EXIT_VIOLATION = 1 

56EXIT_NOTHING = 2 

57 

58 

59def report(target: Path, source_root: str, readme: str, *, run: bool) -> dict[str, Any]: 

60 """Check both halves and return the combined summary, exit code included. 

61 

62 The distinction between exit 1 and exit 2 is the one that matters to a caller that 

63 scores: a broken example is a finding, while *nothing to check* is out-of-scope — the 

64 same rule an unavailable `make` target already follows in `/quality`. 

65 """ 

66 docs = docstring_report(target / source_root, run=run) 

67 readme_part = readme_report(target / readme, run=run) 

68 violations = list(docs["violations"]) + list(readme_part["violations"]) 

69 

70 if violations: 

71 exit_code = EXIT_VIOLATION 

72 elif not docs["present"] and not readme_part["present"]: 

73 exit_code = EXIT_NOTHING 

74 else: 

75 exit_code = EXIT_OK 

76 return { 

77 "docstrings": docs, 

78 "readme": readme_part, 

79 "violations": violations, 

80 "notes": list(docs["notes"]) + list(readme_part["notes"]), 

81 "exit_code": exit_code, 

82 } 

83 

84 

85def _parser() -> argparse.ArgumentParser: 

86 """Build the CLI parser.""" 

87 parser = argparse.ArgumentParser( 

88 description="Check a repo's doctest examples and README code fences.", 

89 ) 

90 parser.add_argument("--target-dir", default=".", help="Repository root (default: cwd).") 

91 parser.add_argument( 

92 "--source-root", 

93 default="src", 

94 help="Source root holding the docstrings (default: src; ask language_profile.py).", 

95 ) 

96 parser.add_argument( 

97 "--readme", default="README.md", help="README to check (default: README.md)." 

98 ) 

99 parser.add_argument( 

100 "--run", 

101 action="store_true", 

102 help="Execute the examples, not just parse them (imports the repo's modules).", 

103 ) 

104 parser.add_argument( 

105 "--json", dest="json_output", action="store_true", help="Emit the summary as JSON." 

106 ) 

107 return parser 

108 

109 

110def main(argv: list[str] | None = None) -> int: 

111 """Entry point: check the repo's examples and return an exit code.""" 

112 args = _parser().parse_args(argv) 

113 # Deliberately not resolved: every path in the report is quoted back as it was given, 

114 # so a run from the repo root prints `README.md:59` rather than an absolute path 

115 # nobody can paste into an editor. 

116 summary = report(Path(args.target_dir), args.source_root, args.readme, run=args.run) 

117 

118 if args.json_output: 

119 print(json.dumps(summary, indent=2)) 

120 else: 

121 print_docstrings(summary["docstrings"]) 

122 print_readme(summary["readme"]) 

123 for violation in summary["violations"]: 

124 print(f"violation {violation}", file=sys.stderr) 

125 for note in summary["notes"]: 

126 print(f"note {note}", file=sys.stderr) 

127 return int(summary["exit_code"]) 

128 

129 

130if __name__ == "__main__": 

131 raise SystemExit(main())