Coverage for plugin/scripts/_doc_examples_source.py: 100%

1#!/usr/bin/env python3 

2"""The docstring half of `check_doc_examples.py`: the doctests under a source root. 

3 

4`interrogate` answers "is there a docstring?". Nothing answered "does the example inside 

5it still evaluate to what it claims?" — and a docstring that lies is worse than one that 

6is missing, because it reads as verified. The rhiza template closes this in a managed repo 

7with `.rhiza/tests/test_docstrings.py`; this module is that check without the `.rhiza/` 

8directory. 

9 

10Two passes, and the split is what lets the first one work anywhere: 

11 

12* **Inventory** — `ast` reads every docstring and `doctest.DocTestParser` finds the 

13 examples. No import, so no dependency has to be installed, and a malformed example 

14 (inconsistent leading whitespace, a `>>>` with no output) is already a violation. It 

15 also reports *how many* examples exist, which is the number separating "12 examples, all 

16 passing" from the far more common "0 examples" — a silence that reads as a pass. 

17* **Execution** (opt-in) — each module is imported and handed to `doctest.testmod`, with 

18 the template's own option flags so a `...` means the same thing in both places. 

19 

20A module that cannot be imported is reported as **unmeasured**, never failed: a missing 

21third-party dependency in the ambient interpreter is a fact about the environment, not a 

22defect in the docstring. 

23""" 

24 

25from __future__ import annotations 

26 

27import ast 

28import contextlib 

29import doctest 

30import importlib 

31import io 

32import sys 

33from pathlib import Path 

34from typing import Any 

35 

36# Directories that are never a repo's own source: virtualenvs, build output, vendored 

37# code — plus `tests`, whose examples are not documentation and whose modules a scoring 

38# run should not import. A source root of `.` (what `language_profile.py` reports for a 

39# manifest-less repo) makes all of these reachable, which is why the list exists. 

40_SKIP_DIRS = frozenset( 

41 {".git", ".venv", "venv", ".tox", "__pycache__", "build", "dist", "node_modules", "tests"} 

42) 

43 

44# What the template's own doctest runner uses, so an example that passes there passes here. 

45_OPTIONFLAGS = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE 

46 

47_DOCUMENTABLE = (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) 

48 

49 

50def _node_name(node: ast.AST) -> str: 

51 """Return a readable name for the docstring's owner.""" 

52 return getattr(node, "name", "<module>") 

53 

54 

55def _node_line(node: ast.AST) -> int: 

56 """Return the line the docstring's owner starts on (1 for a module).""" 

57 return getattr(node, "lineno", 1) 

58 

59 

60def source_files(root: Path) -> list[Path]: 

61 """Return the repo's own `.py` files under *root*, skipping vendored and test trees.""" 

62 return sorted(p for p in root.rglob("*.py") if not (set(p.parts) & _SKIP_DIRS)) 

63 

64 

65def docstring_examples(path: Path) -> tuple[list[dict[str, Any]], list[str]]: 

66 """Return (example locations, problems) for every docstring in *path*. 

67 

68 No import happens here, so this pass works in an environment where the module's 

69 dependencies are absent — and a docstring whose example is malformed is a violation 

70 before anything is run. 

71 """ 

72 try: 

73 tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) 

74 except (SyntaxError, UnicodeDecodeError, ValueError) as exc: 

75 return [], [f"{path}: does not parse — {exc}"] 

76 

77 parser = doctest.DocTestParser() 

78 found: list[dict[str, Any]] = [] 

79 problems: list[str] = [] 

80 for node in ast.walk(tree): 

81 text = ast.get_docstring(node, clean=False) if isinstance(node, _DOCUMENTABLE) else None 

82 if not text: 

83 continue 

84 try: 

85 examples = parser.get_examples(text) 

86 except ValueError as exc: 

87 problems.append(f"{path}:{_node_line(node)}: malformed doctest — {exc}") 

88 continue 

89 if examples: 

90 found.append( 

91 { 

92 "file": str(path), 

93 "object": _node_name(node), 

94 "line": _node_line(node), 

95 "examples": len(examples), 

96 } 

97 ) 

98 return found, problems 

99 

100 

101def run_doctests(root: Path, paths: list[Path]) -> dict[str, Any]: 

102 """Import each module under *root*, run its doctests, and summarise the result. 

103 

104 An unimportable module is unmeasured rather than failed (see the module docstring). 

105 The count comes back so the caller can say so, instead of a partial run reading as a 

106 complete one. 

107 """ 

108 attempted = failed = 0 

109 failures: list[str] = [] 

110 unimportable: list[str] = [] 

111 # Resolved only here: the reports quote *root* as it was given, so a relative 

112 # `--target-dir` keeps the paths short, while an import path has to be absolute or it 

113 # would follow the process's cwd. 

114 import_path = str(root.resolve()) 

115 sys.path.insert(0, import_path) 

116 try: 

117 for path in paths: 

118 name = ".".join(path.relative_to(root).with_suffix("").parts).removesuffix(".__init__") 

119 try: 

120 module = importlib.import_module(name) 

121 except Exception as exc: # noqa: BLE001 - any module-level failure is "unmeasured" 

122 unimportable.append(f"{name}: {type(exc).__name__}: {exc}") 

123 continue 

124 buffer = io.StringIO() 

125 with contextlib.redirect_stdout(buffer): 

126 result = doctest.testmod(module, verbose=False, optionflags=_OPTIONFLAGS) 

127 attempted += result.attempted 

128 failed += result.failed 

129 if result.failed: 

130 failures.append( 

131 f"{name}: {result.failed}/{result.attempted} example(s) failed\n" 

132 f"{buffer.getvalue().strip()}" 

133 ) 

134 finally: 

135 sys.path.remove(import_path) 

136 return { 

137 "attempted": attempted, 

138 "failed": failed, 

139 "failures": failures, 

140 "unimportable": unimportable, 

141 } 

142 

143 

144def _notes(total: int, execution: dict[str, Any] | None) -> list[str]: 

145 """Build the notes for this half — the silences that would otherwise read as passes.""" 

146 if not total: 

147 return [ 

148 "no doctest examples found — docstring coverage says nothing about whether the " 

149 "docstrings are true, and here there is nothing to check" 

150 ] 

151 if execution is None: 

152 return [f"{total} example(s) found but not run — pass --run to execute them"] 

153 if execution["unimportable"]: 

154 return [ 

155 f"{len(execution['unimportable'])} module(s) could not be imported, so their " 

156 "examples are unmeasured, not passing — run this inside the project's own " 

157 "environment to measure them" 

158 ] 

159 return [] 

160 

161 

162def docstring_report(root: Path, *, run: bool) -> dict[str, Any]: 

163 """Inventory (and optionally run) every doctest example under *root*.""" 

164 if not root.is_dir(): 

165 return { 

166 "source_root": str(root), 

167 "present": False, 

168 "examples": 0, 

169 "locations": [], 

170 "violations": [], 

171 "notes": [f"no source root at {root} — docstring examples are out of scope"], 

172 } 

173 

174 paths = source_files(root) 

175 locations: list[dict[str, Any]] = [] 

176 violations: list[str] = [] 

177 for path in paths: 

178 found, problems = docstring_examples(path) 

179 locations.extend(found) 

180 violations.extend(problems) 

181 total = sum(int(item["examples"]) for item in locations) 

182 

183 execution: dict[str, Any] | None = None 

184 if run and total: 

185 execution = run_doctests(root, paths) 

186 violations.extend(execution["failures"]) 

187 

188 report: dict[str, Any] = { 

189 "source_root": str(root), 

190 "present": True, 

191 "files": len(paths), 

192 "examples": total, 

193 "locations": locations, 

194 "violations": violations, 

195 "notes": _notes(total, execution), 

196 } 

197 if execution is not None: 

198 report["execution"] = execution 

199 return report 

200 

201 

202def print_report(docs: dict[str, Any]) -> None: 

203 """Print the docstring half of a report as text.""" 

204 if not docs["present"]: 

205 print(f"{'unavailable':<12} docstring examples ({docs['source_root']} is not a directory)") 

206 return 

207 print( 

208 f"{'docstrings':<12} {docs['source_root']}: {docs['files']} file(s), " 

209 f"{docs['examples']} example(s) in {len(docs['locations'])} docstring(s)" 

210 ) 

211 for item in docs["locations"]: 

212 print( 

213 f"{'example':<12} {item['file']}:{item['line']} {item['object']} ({item['examples']})" 

214 ) 

215 execution = docs.get("execution") 

216 if execution is not None: 

217 print( 

218 f"{'ran':<12} {execution['attempted']} example(s), {execution['failed']} failed, " 

219 f"{len(execution['unimportable'])} module(s) unimportable" 

220 )