Coverage for scripts/check_test_layout.py: 100%

1#!/usr/bin/env python3 

2"""Check that the test layout mirrors the source layout. 

3 

4Enforces a strict test/source parity so tests are easy to locate and no test 

5drifts loose from what it covers: 

6 

7 * every source module ``<src>/…/xyz.py`` has a test file 

8 ``<tests>/…/test_xyz.py`` (nested packages are mirrored); 

9 * every top-level ``class A`` in a source module has a matching ``TestA`` 

10 class in that test file; 

11 * no test file lacks a corresponding source module (no orphan test files); 

12 * no ``Test*`` class lacks a corresponding source class (no orphan test 

13 classes). 

14 

15``__init__.py`` and ``conftest.py`` are ignored on both sides, and the 

16``tests/benchmarks/`` and ``tests/stress/`` trees are exempt entirely — those 

17hold benchmarks and stress tests that need not mirror a source module. Test 

18*functions* are unconstrained — the rules bind files and classes only. 

19 

20Repositories that deliberately organise tests by *behaviour* rather than 1:1 

21mirroring (and guarantee per-module coverage another way, e.g. a 100% coverage 

22gate) can opt out via a ``[tool.check_test_layout]`` table in ``pyproject.toml``:: 

23 

24 [tool.check_test_layout] 

25 enforce = false 

26 reason = "Tests are grouped by behaviour; coverage is enforced by pytest." 

27 

28``enforce = false`` requires a non-empty ``reason`` so the deviation is always 

29documented. The same table accepts ``exempt_dirs = [...]`` to extend the 

30built-in benchmarks/stress exemptions when parity *is* enforced. 

31 

32Usage: 

33 uv run --python 3.12 --no-project python \ 

34 scripts/check_test_layout.py [--src DIR] [--tests DIR] [--config FILE] 

35 

36Exits 0 when the layout is clean (or parity is intentionally not enforced), 

371 (listing every violation) otherwise. 

38""" 

39 

40from __future__ import annotations 

41 

42import argparse 

43import ast 

44import sys 

45from collections.abc import Mapping 

46from pathlib import Path 

47 

48try: # py3.11+ 

49 import tomllib 

50except ModuleNotFoundError: # pragma: no cover - older interpreters 

51 try: 

52 import tomli as tomllib # type: ignore 

53 except ModuleNotFoundError: 

54 tomllib = None # type: ignore 

55 

56_IGNORED = {"__init__.py", "conftest.py"} 

57 

58# Top-level directories under the tests root that are exempt from parity by 

59# default: they hold benchmarks / stress tests that need not mirror a source 

60# module. A repo can extend this set via ``[tool.check_test_layout] exempt_dirs``. 

61_DEFAULT_EXEMPT_DIRS = {"benchmarks", "stress"} 

62 

63 

64def _coerce_scalar(raw: str) -> object: 

65 """Coerce a TOML scalar/array literal to a Python value (fallback reader). 

66 

67 Handles the narrow subset the ``[tool.check_test_layout]`` table uses: 

68 quoted strings, ``true``/``false`` booleans, and single-line arrays of 

69 quoted strings. Everything else is returned as its stripped token. 

70 """ 

71 raw = raw.strip() 

72 if raw[:1] in {'"', "'"}: 

73 end = raw.find(raw[0], 1) 

74 return raw[1:end] if end != -1 else raw[1:] 

75 if raw.startswith("["): 

76 end = raw.find("]") 

77 inner = raw[1 : end if end != -1 else len(raw)] 

78 return [v for v in (_coerce_scalar(item) for item in inner.split(",")) if v != ""] 

79 token = raw.split("#", 1)[0].strip() 

80 if token == "true": 

81 return True 

82 if token == "false": 

83 return False 

84 return token 

85 

86 

87def _parse_flat_section(text: str, header: str) -> dict[str, object]: 

88 """Extract a single flat ``[header]`` table from TOML *text*. 

89 

90 A dependency-free fallback for interpreters without ``tomllib``/``tomli`` 

91 (the plugin runs under the ambient ``python3``, which may predate 3.11). 

92 It recognises only the flat ``key = value`` table this checker reads. 

93 """ 

94 want = f"[{header}]" 

95 out: dict[str, object] = {} 

96 in_section = False 

97 for line in text.splitlines(): 

98 stripped = line.strip() 

99 if not stripped or stripped.startswith("#"): 

100 continue 

101 if stripped.startswith("["): 

102 in_section = stripped == want 

103 continue 

104 if not in_section or "=" not in stripped: 

105 continue 

106 key, _, value = stripped.partition("=") 

107 out[key.strip()] = _coerce_scalar(value) 

108 return out 

109 

110 

111def _read_config(pyproject: Path) -> dict[str, object]: 

112 """Return the ``[tool.check_test_layout]`` table from *pyproject* (empty if absent). 

113 

114 Prefers ``tomllib``/``tomli`` when importable; otherwise uses the flat-table 

115 fallback so the opt-out is honoured regardless of interpreter version. 

116 """ 

117 if not pyproject.is_file(): 

118 return {} 

119 text = pyproject.read_text(encoding="utf-8") 

120 if tomllib is not None: 

121 try: 

122 data = tomllib.loads(text) 

123 except ValueError: 

124 return {} 

125 section = data.get("tool", {}).get("check_test_layout", {}) 

126 return section if isinstance(section, dict) else {} 

127 return _parse_flat_section(text, "tool.check_test_layout") 

128 

129 

130def _exempt_dirs(config: Mapping[str, object]) -> set[str]: 

131 """Return the exempt top-level test dirs: defaults plus any from *config*.""" 

132 dirs = set(_DEFAULT_EXEMPT_DIRS) 

133 extra = config.get("exempt_dirs") 

134 if isinstance(extra, list): 

135 dirs |= {str(d) for d in extra} 

136 return dirs 

137 

138 

139def _top_level_classes(path: Path) -> set[str]: 

140 """Return the names of top-level classes defined in *path*.""" 

141 tree = ast.parse(path.read_text(), filename=str(path)) 

142 return {node.name for node in tree.body if isinstance(node, ast.ClassDef)} 

143 

144 

145def _source_modules(src: Path) -> list[Path]: 

146 """Return the source ``.py`` modules under *src* (ignoring dunder/conftest).""" 

147 return sorted(p for p in src.rglob("*.py") if p.name not in _IGNORED) 

148 

149 

150def _test_files(tests: Path, exempt: set[str] | None = None) -> list[Path]: 

151 """Return the ``test_*.py`` files under *tests* (ignoring conftest/exempt dirs).""" 

152 exempt = _DEFAULT_EXEMPT_DIRS if exempt is None else exempt 

153 return sorted( 

154 p 

155 for p in tests.rglob("test_*.py") 

156 if p.name not in _IGNORED and p.relative_to(tests).parts[0] not in exempt 

157 ) 

158 

159 

160def check(src: Path, tests: Path, config: Mapping[str, object] | None = None) -> list[str]: 

161 """Return a list of layout violations (empty when the layout is clean).""" 

162 exempt = _exempt_dirs(config or {}) 

163 errors: list[str] = [] 

164 

165 # Forward: every source module needs a mirrored test file + Test* classes. 

166 for module in _source_modules(src): 

167 rel = module.relative_to(src) 

168 test_path = tests / rel.parent / f"test_{module.stem}.py" 

169 if not test_path.exists(): 

170 errors.append(f"missing test file {test_path} for source module {module}") 

171 continue 

172 test_classes = _top_level_classes(test_path) 

173 for cls in sorted(_top_level_classes(module)): 

174 if f"Test{cls}" not in test_classes: 

175 errors.append(f"missing class Test{cls} in {test_path} for class {cls} in {module}") 

176 

177 # Reverse: every test file/class must trace back to a source module/class. 

178 for test_file in _test_files(tests, exempt): 

179 rel = test_file.relative_to(tests) 

180 source_name = test_file.stem[len("test_") :] 

181 source_path = src / rel.parent / f"{source_name}.py" 

182 if not source_path.exists(): 

183 errors.append(f"orphan test file {test_file} (no source module {source_path})") 

184 continue 

185 source_classes = _top_level_classes(source_path) 

186 for cls in sorted(_top_level_classes(test_file)): 

187 if cls.startswith("Test") and cls[len("Test") :] not in source_classes: 

188 errors.append( 

189 f"orphan test class {cls} in {test_file} " 

190 f"(no class {cls[len('Test') :]} in {source_path})" 

191 ) 

192 

193 return errors 

194 

195 

196def main(argv: list[str] | None = None) -> int: 

197 """Entry point: check the layout and return an exit code.""" 

198 parser = argparse.ArgumentParser(description="Check test/source layout parity.") 

199 parser.add_argument("--src", default="src", help="Source directory (default: src).") 

200 parser.add_argument("--tests", default="tests", help="Tests directory (default: tests).") 

201 parser.add_argument( 

202 "--config", 

203 default="pyproject.toml", 

204 help="pyproject.toml providing [tool.check_test_layout] (default: pyproject.toml).", 

205 ) 

206 args = parser.parse_args(argv) 

207 

208 config = _read_config(Path(args.config)) 

209 

210 if not config.get("enforce", True): 

211 reason = str(config.get("reason", "")).strip() 

212 if not reason: 

213 print( 

214 "Test-layout check misconfigured: [tool.check_test_layout] enforce=false " 

215 "requires a non-empty 'reason' documenting the intentional layout.", 

216 file=sys.stderr, 

217 ) 

218 return 1 

219 print(f"Test layout OK: parity not enforced by request — {reason}") 

220 return 0 

221 

222 errors = check(Path(args.src), Path(args.tests), config) 

223 if errors: 

224 print("Test-layout check failed:", file=sys.stderr) 

225 for err in errors: 

226 print(f" ✗ {err}", file=sys.stderr) 

227 return 1 

228 print("Test layout OK: tests mirror sources 1:1") 

229 return 0 

230 

231 

232if __name__ == "__main__": 

233 raise SystemExit(main())