Coverage for plugin/scripts/check_test_layout.py: 100%

1#!/usr/bin/env python3 

2"""Check that the test layout mirrors the source layout. 

3 

4Enforces a strict test/source parity so tests are easy to locate and no test 

5drifts loose from what it covers: 

6 

7 * every source module ``<src>/…/xyz.py`` has a test file 

8 ``<tests>/…/test_xyz.py`` (nested packages are mirrored); 

9 * every top-level ``class A`` in a source module has a matching ``TestA`` 

10 class in that test file; 

11 * no test file lacks a corresponding source module (no orphan test files); 

12 * no ``Test*`` class lacks a corresponding source class (no orphan test 

13 classes). 

14 

15``__init__.py`` and ``conftest.py`` are ignored on both sides, and the 

16``tests/benchmarks/`` and ``tests/stress/`` trees are exempt entirely — those 

17hold benchmarks and stress tests that need not mirror a source module. Test 

18*functions* are unconstrained — the rules bind files and classes only. 

19 

20Repositories that deliberately organise tests by *behaviour* rather than 1:1 

21mirroring (and guarantee per-module coverage another way, e.g. a 100% coverage 

22gate) can opt out via a ``[tool.check_test_layout]`` table in ``pyproject.toml``:: 

23 

24 [tool.check_test_layout] 

25 enforce = false 

26 reason = "Tests are grouped by behaviour; coverage is enforced by pytest." 

27 

28``enforce = false`` requires a non-empty ``reason`` so the deviation is always 

29documented. The same table accepts ``exempt_dirs = [...]`` to extend the 

30built-in benchmarks/stress exemptions when parity *is* enforced. 

31 

32**Template-owned tests are exempt too, and not by name.** A rhiza sync writes 

33files into the consumer's tree and records every one of them in 

34``.rhiza/template.lock``'s ``files:`` list. Since v1.3.2 that list includes 

35``tests/test_rhiza_packaging.py`` — a test of *packaging metadata*, which mirrors 

36no source module and never will, so a freshly synced repo failed this check on a 

37file it did not write and cannot move (jebel-quant/rhiza#1489). The lock is the 

38machine-readable answer: any test file it tracks is skipped. That covers the next 

39synced test as well, which an allowlist of one filename would not, and it stays 

40narrow — a repo's *own* tests are still checked, unlike the ``enforce = false`` 

41escape hatch, which switches off the guarantee wholesale. 

42 

43Usage: 

44 uv run --python 3.12 --no-project python \ 

45 scripts/check_test_layout.py [--src DIR] [--tests DIR] [--config FILE] \ 

46 [--lock FILE] 

47 

48Exits 0 when the layout is clean (or parity is intentionally not enforced), 

491 (listing every violation) otherwise. 

50""" 

51 

52from __future__ import annotations 

53 

54import argparse 

55import ast 

56import sys 

57from collections.abc import Mapping 

58from pathlib import Path 

59 

60try: # py3.11+ 

61 import tomllib 

62except ModuleNotFoundError: # pragma: no cover - older interpreters 

63 try: 

64 import tomli as tomllib # type: ignore 

65 except ModuleNotFoundError: 

66 tomllib = None # type: ignore 

67 

68_IGNORED = {"__init__.py", "conftest.py"} 

69 

70# Top-level directories under the tests root that are exempt from parity by 

71# default: they hold benchmarks / stress tests that need not mirror a source 

72# module. A repo can extend this set via ``[tool.check_test_layout] exempt_dirs``. 

73_DEFAULT_EXEMPT_DIRS = {"benchmarks", "stress"} 

74 

75# Where a rhiza sync records the files it wrote. Read for its ``files:`` list only. 

76_DEFAULT_LOCK = ".rhiza/template.lock" 

77 

78 

79def _coerce_scalar(raw: str) -> object: 

80 """Coerce a TOML scalar/array literal to a Python value (fallback reader). 

81 

82 Handles the narrow subset the ``[tool.check_test_layout]`` table uses: 

83 quoted strings, ``true``/``false`` booleans, and single-line arrays of 

84 quoted strings. Everything else is returned as its stripped token. 

85 """ 

86 raw = raw.strip() 

87 if raw[:1] in {'"', "'"}: 

88 end = raw.find(raw[0], 1) 

89 return raw[1:end] if end != -1 else raw[1:] 

90 if raw.startswith("["): 

91 end = raw.find("]") 

92 inner = raw[1 : end if end != -1 else len(raw)] 

93 return [v for v in (_coerce_scalar(item) for item in inner.split(",")) if v != ""] 

94 bare = raw.split("#", 1)[0].strip() 

95 if bare == "true": 

96 return True 

97 if bare == "false": 

98 return False 

99 return bare 

100 

101 

102def _parse_flat_section(text: str, header: str) -> dict[str, object]: 

103 """Extract a single flat ``[header]`` table from TOML *text*. 

104 

105 A dependency-free fallback for interpreters without ``tomllib``/``tomli`` 

106 (the plugin runs under the ambient ``python3``, which may predate 3.11). 

107 It recognises only the flat ``key = value`` table this checker reads. 

108 """ 

109 want = f"[{header}]" 

110 out: dict[str, object] = {} 

111 in_section = False 

112 for line in text.splitlines(): 

113 stripped = line.strip() 

114 if not stripped or stripped.startswith("#"): 

115 continue 

116 if stripped.startswith("["): 

117 in_section = stripped == want 

118 continue 

119 if not in_section or "=" not in stripped: 

120 continue 

121 key, _, value = stripped.partition("=") 

122 out[key.strip()] = _coerce_scalar(value) 

123 return out 

124 

125 

126def _read_config(pyproject: Path) -> dict[str, object]: 

127 """Return the ``[tool.check_test_layout]`` table from *pyproject* (empty if absent). 

128 

129 Prefers ``tomllib``/``tomli`` when importable; otherwise uses the flat-table 

130 fallback so the opt-out is honoured regardless of interpreter version. 

131 """ 

132 if not pyproject.is_file(): 

133 return {} 

134 text = pyproject.read_text(encoding="utf-8") 

135 if tomllib is not None: 

136 try: 

137 data = tomllib.loads(text) 

138 except ValueError: 

139 return {} 

140 section = data.get("tool", {}).get("check_test_layout", {}) 

141 return section if isinstance(section, dict) else {} 

142 return _parse_flat_section(text, "tool.check_test_layout") 

143 

144 

145def _exempt_dirs(config: Mapping[str, object]) -> set[str]: 

146 """Return the exempt top-level test dirs: defaults plus any from *config*.""" 

147 dirs = set(_DEFAULT_EXEMPT_DIRS) 

148 extra = config.get("exempt_dirs") 

149 if isinstance(extra, list): 

150 dirs |= {str(d) for d in extra} 

151 return dirs 

152 

153 

154def _unquote(raw: str) -> str: 

155 """Strip one layer of matching quotes from a scalar, if present.""" 

156 raw = raw.strip() 

157 if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {'"', "'"}: 

158 return raw[1:-1] 

159 return raw 

160 

161 

162def _lock_files(lock: Path) -> list[str]: 

163 """Return the entries of the ``files:`` block sequence in *lock* (empty if absent). 

164 

165 Deliberately a few lines of string handling rather than a YAML dependency or an 

166 import of the plugin's own reader: this script is run standalone (and copied into 

167 repositories that vendor it), and the block it reads is a flat list of quoted or 

168 bare scalars. Any other top-level key ends the block, so a lock whose ``files`` 

169 field is missing, inline (``files: []``) or unreadable simply yields nothing — 

170 exempting no test file, which is the safe direction. 

171 """ 

172 try: 

173 text = lock.read_text(encoding="utf-8") 

174 except OSError: 

175 return [] 

176 out: list[str] = [] 

177 in_block = False 

178 for line in text.splitlines(): 

179 stripped = line.strip() 

180 if not stripped or stripped.startswith("#"): 

181 continue 

182 if in_block and stripped.startswith("- "): 

183 out.append(_unquote(stripped[2:])) 

184 continue 

185 in_block = stripped == "files:" 

186 return out 

187 

188 

189def _template_owned(lock: Path) -> set[Path]: 

190 """Return the resolved paths of the files a template sync wrote into this repo. 

191 

192 Lock entries are repo-root-relative. The root is the lock's grandparent when the 

193 lock sits in ``.rhiza/`` — the layout every sync writes — and its own directory 

194 otherwise, so an explicit ``--lock`` pointing elsewhere still resolves. 

195 """ 

196 root = lock.parent.parent if lock.parent.name == ".rhiza" else lock.parent 

197 return {(root / rel).resolve() for rel in _lock_files(lock)} 

198 

199 

200def _top_level_classes(path: Path) -> set[str]: 

201 """Return the names of top-level classes defined in *path*.""" 

202 tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) 

203 return {node.name for node in tree.body if isinstance(node, ast.ClassDef)} 

204 

205 

206def _source_modules(src: Path) -> list[Path]: 

207 """Return the source ``.py`` modules under *src* (ignoring dunder/conftest).""" 

208 return sorted(p for p in src.rglob("*.py") if p.name not in _IGNORED) 

209 

210 

211def _test_files(tests: Path, exempt: set[str] | None = None) -> list[Path]: 

212 """Return the ``test_*.py`` files under *tests* (ignoring conftest/exempt dirs).""" 

213 exempt = _DEFAULT_EXEMPT_DIRS if exempt is None else exempt 

214 return sorted( 

215 p 

216 for p in tests.rglob("test_*.py") 

217 if p.name not in _IGNORED and p.relative_to(tests).parts[0] not in exempt 

218 ) 

219 

220 

221def _forward_errors(src: Path, tests: Path) -> list[str]: 

222 """Every source module needs a mirrored test file, and a ``Test*`` per source class.""" 

223 errors: list[str] = [] 

224 for module in _source_modules(src): 

225 rel = module.relative_to(src) 

226 test_path = tests / rel.parent / f"test_{module.stem}.py" 

227 if not test_path.exists(): 

228 errors.append(f"missing test file {test_path} for source module {module}") 

229 continue 

230 test_classes = _top_level_classes(test_path) 

231 errors.extend( 

232 f"missing class Test{cls} in {test_path} for class {cls} in {module}" 

233 for cls in sorted(_top_level_classes(module)) 

234 if f"Test{cls}" not in test_classes 

235 ) 

236 return errors 

237 

238 

239def _reverse_errors(src: Path, tests: Path, exempt: set[str], owned: set[Path]) -> list[str]: 

240 """Every test file and ``Test*`` class must trace back to a source module or class. 

241 

242 The direction that catches a test left behind by a rename — which is worse than a 

243 missing test, because it keeps passing while covering nothing. *owned* paths are the 

244 exception: the repository neither wrote them nor can rename them. 

245 """ 

246 errors: list[str] = [] 

247 for test_file in _test_files(tests, exempt): 

248 if test_file.resolve() in owned: 

249 continue 

250 rel = test_file.relative_to(tests) 

251 source_name = test_file.stem[len("test_") :] 

252 source_path = src / rel.parent / f"{source_name}.py" 

253 if not source_path.exists(): 

254 errors.append(f"orphan test file {test_file} (no source module {source_path})") 

255 continue 

256 source_classes = _top_level_classes(source_path) 

257 errors.extend( 

258 f"orphan test class {cls} in {test_file} " 

259 f"(no class {cls[len('Test') :]} in {source_path})" 

260 for cls in sorted(_top_level_classes(test_file)) 

261 if cls.startswith("Test") and cls[len("Test") :] not in source_classes 

262 ) 

263 return errors 

264 

265 

266def check( 

267 src: Path, 

268 tests: Path, 

269 config: Mapping[str, object] | None = None, 

270 owned: set[Path] | None = None, 

271) -> list[str]: 

272 """Return a list of layout violations (empty when the layout is clean). 

273 

274 Both directions are checked, and both matter: a missing test leaves code uncovered, 

275 while an orphan test keeps passing after the code it named is gone. 

276 

277 *owned* holds template-written paths (see :func:`_template_owned`); test files in 

278 it are skipped, since the repository neither wrote them nor can rename them. 

279 """ 

280 exempt = _exempt_dirs(config or {}) 

281 return _forward_errors(src, tests) + _reverse_errors(src, tests, exempt, owned or set()) 

282 

283 

284def main(argv: list[str] | None = None) -> int: 

285 """Entry point: check the layout and return an exit code.""" 

286 parser = argparse.ArgumentParser(description="Check test/source layout parity.") 

287 parser.add_argument("--src", default="src", help="Source directory (default: src).") 

288 parser.add_argument("--tests", default="tests", help="Tests directory (default: tests).") 

289 parser.add_argument( 

290 "--config", 

291 default="pyproject.toml", 

292 help="pyproject.toml providing [tool.check_test_layout] (default: pyproject.toml).", 

293 ) 

294 parser.add_argument( 

295 "--lock", 

296 default=_DEFAULT_LOCK, 

297 help=f"Template lock whose files: list is exempt from parity (default: {_DEFAULT_LOCK}).", 

298 ) 

299 args = parser.parse_args(argv) 

300 

301 config = _read_config(Path(args.config)) 

302 

303 if not config.get("enforce", True): 

304 reason = str(config.get("reason", "")).strip() 

305 if not reason: 

306 print( 

307 "Test-layout check misconfigured: [tool.check_test_layout] enforce=false " 

308 "requires a non-empty 'reason' documenting the intentional layout.", 

309 file=sys.stderr, 

310 ) 

311 return 1 

312 print(f"Test layout OK: parity not enforced by request — {reason}") 

313 return 0 

314 

315 errors = check(Path(args.src), Path(args.tests), config, _template_owned(Path(args.lock))) 

316 if errors: 

317 print("Test-layout check failed:", file=sys.stderr) 

318 for err in errors: 

319 print(f" ✗ {err}", file=sys.stderr) 

320 return 1 

321 print("Test layout OK: tests mirror sources 1:1") 

322 return 0 

323 

324 

325if __name__ == "__main__": 

326 raise SystemExit(main())