1"""
2What is wrong with the tests a file defines, and how it is said.
3
4There are two ways to find out. A file that can be run is run, and its
5tests are looked at as the functions they are (`collection`). A file that
6can't be run is read: what its syntax tree shows about its tests is
7reported with the reason it can't be run, so that a file with three things
8to fix says all three at once. `tests_as_written` is the reading.
9
10Reading only reports what the syntax tree makes certain. A test under a
11decorator this module doesn't know may be passed values by it, so nothing
12is said about that test's parameters.
13"""
14
15import ast
16import textwrap
17from dataclasses import dataclass, field
18
19from ..definition import TestDefinitionError
20
21__all__ = []
22
23A_TEST_TAKES_ONLY_ITS_CASES = (
24 "Nothing is passed to a test by name. A test gets what it needs in its\n"
25 "body, by calling a helper or entering a `with` block, and takes values\n"
26 "only from @cases(...)."
27)
28
29_A_TEST_IS_RUN_WHERE_IT_IS_DEFINED = (
30 "A test is run by the file that defines it, so one that is imported\n"
31 "would be left out. Define it in this file. If it isn't a test, import\n"
32 "it under a name that doesn't look like one:\n"
33 "`from billing import test_connection as check_connection`. A test made\n"
34 "by a decorator belongs to the decorator's module until the decorator\n"
35 "uses `functools.wraps`."
36)
37
38_A_TEST_IS_A_FUNCTION = (
39 "A test is a function, and a file is the group. Write each of the\n"
40 "class's tests as a function of the file, and what they shared as\n"
41 "functions they call. A class that isn't a test, and has to have a\n"
42 "method named `test_*`, belongs in a helper module."
43)
44
45_A_TEST_CANT_YIELD = (
46 "A test can't yield: calling a function with a `yield` in it makes a\n"
47 "generator, and runs none of its body. Values to be checked one at a\n"
48 "time are passed in with @cases(...). Setup with cleanup after it belongs\n"
49 "in a `@contextmanager` helper, which the test enters with `with`."
50)
51
52ONE_CASES_FOR_EVERY_COMBINATION = (
53 "Each case is one flat tuple: the test's values, in the order of its\n"
54 "parameters. For every combination of two lists, build the cases from\n"
55 "both:\n"
56 "\n"
57 " @cases(*[(a, b, c) for a in FIRST for b, c in SECOND])\n"
58 "\n"
59 "Each `for` names what one entry of its list holds: `for a in FIRST`\n"
60 "when the entries are single values, `for b, c in SECOND` when they are\n"
61 "tuples."
62)
63
64# A file with more tests than this to fix says how many, not which.
65_MOST_TESTS_NAMED = 3
66
67# A run with more test files than this in the application says which
68# directories they are in, not which files.
69_MOST_FILES_NAMED = 3
70
71
72def tests_are_not_kept_in_the_application(
73 files: list[str], *, application: str, tests: str | None
74) -> str:
75 """
76 What a run says about the test files it found in the application and
77 didn't run. `tests` is the tests directory, or None when the run was
78 started in it.
79 """
80 if len(files) == 1:
81 found = f"1 test file is in {application}/, and it was not run:"
82 them = "it"
83 else:
84 found = (
85 f"{len(files)} test files are in {application}/, and none of them was run:"
86 )
87 them = "them"
88
89 if len(files) > _MOST_FILES_NAMED:
90 in_each: dict[str, int] = {}
91 for file in files:
92 directory = file.rpartition("/")[0]
93 in_each[directory] = in_each.get(directory, 0) + 1
94 widest = max(len(directory) for directory in in_each)
95 where = [
96 f"{directory.ljust(widest)} {'1 file' if count == 1 else f'{count} files'}"
97 for directory, count in in_each.items()
98 ]
99 else:
100 where = files
101
102 if tests is None:
103 where_tests_live = f"Tests live beside {application}/, in this directory."
104 move = f"Move {them} out of {application}/, into this directory."
105 imported_from = "this directory"
106 else:
107 where_tests_live = f"Tests live in {tests}/, beside {application}/."
108 move = f"Move {them} to {tests}/."
109 imported_from = f"{tests}/"
110
111 what_to_do = textwrap.fill(
112 f"{where_tests_live} {application}/ is the application: it is imported"
113 f" as `{application}`, and it is what gets deployed. {move} A test file"
114 " is the same file there, and a helper module it imports is imported"
115 f" by its path from {imported_from}.",
116 width=72,
117 )
118 return "\n\n".join([found, textwrap.indent("\n".join(where), " "), what_to_do])
119
120
121@dataclass(kw_only=True)
122class ProblemsInAFile:
123 """What is wrong with the tests one file defines."""
124
125 # A test that can't be run for a reason of its own, and what to write.
126 of_one_test: list[str] = field(default_factory=list)
127 # The tests with more than one `@cases`: the test, and how many.
128 tests_with_cases_twice: list[tuple[str, int]] = field(default_factory=list)
129 # Whether running the file would stop at a decorator that raises: a
130 # second `@cases`, a `@skip` or `@tag` that isn't called.
131 stops_at_a_decorator: bool = False
132 # The tests that have a `yield` in them.
133 tests_that_yield: list[str] = field(default_factory=list)
134 # The classes that hold tests: the line each is on, its name, and how
135 # many tests are in it.
136 classes_with_tests: list[tuple[int | None, str, int]] = field(default_factory=list)
137 # The tests that were imported, each with the module it is defined in.
138 defined_elsewhere: list[tuple[str, str]] = field(default_factory=list)
139 # The tests that take parameters nothing passes in, as they are written:
140 # `test_signup(db, client)`.
141 tests_taking_parameters: list[str] = field(default_factory=list)
142 # Each of those parameters, and how many of the tests take it.
143 parameters: dict[str, int] = field(default_factory=dict)
144
145 def __bool__(self) -> bool:
146 return bool(
147 self.of_one_test
148 or self.tests_with_cases_twice
149 or self.tests_that_yield
150 or self.classes_with_tests
151 or self.defined_elsewhere
152 or self.tests_taking_parameters
153 )
154
155 def what_is_wrong(self) -> str:
156 """
157 The tests, and the parameters nothing passes in. Without the rule
158 about parameters, which a run says once:
159 `A_TEST_TAKES_ONLY_ITS_CASES`.
160 """
161 lines = list(self.of_one_test)
162 for name, count in self.tests_with_cases_twice:
163 lines.append(f"{name} has {count} @cases. A test takes one.")
164 for line, name, count in self.classes_with_tests:
165 where = f"line {line}: " if line is not None else ""
166 tests = "1 test" if count == 1 else f"{count} tests"
167 lines.append(f"{where}{name} is a class with {tests} in it.")
168 lines.extend(
169 f"{name} is defined in {module}, not in this file."
170 for name, module in self.defined_elsewhere
171 )
172 lines.extend(f"{name}() has a `yield` in it." for name in self.tests_that_yield)
173
174 taking = self.tests_taking_parameters
175 if len(taking) > _MOST_TESTS_NAMED:
176 lines.append(
177 f"{len(taking)} tests take parameters, and nothing passes them in."
178 )
179 else:
180 lines.extend(
181 f"{written} takes parameters, and nothing passes them in."
182 for written in taking
183 )
184 sections = [
185 "These tests can't be run as written:",
186 textwrap.indent("\n".join(lines), " "),
187 ]
188
189 if self.parameters:
190 widest = max(len(name) for name in self.parameters)
191 table = []
192 for name, count in self.parameters.items():
193 tests = "1 test" if count == 1 else f"{count} tests"
194 table.append(f"{name.ljust(widest)} {tests}")
195 sections.append(textwrap.indent("\n".join(table), " "))
196 if self.tests_with_cases_twice:
197 sections.append(ONE_CASES_FOR_EVERY_COMBINATION)
198 if self.classes_with_tests:
199 sections.append(_A_TEST_IS_A_FUNCTION)
200 if self.defined_elsewhere:
201 sections.append(_A_TEST_IS_RUN_WHERE_IT_IS_DEFINED)
202 if self.tests_that_yield:
203 sections.append(_A_TEST_CANT_YIELD)
204 return "\n\n".join(sections)
205
206
207class CantBeRunAsWritten(TestDefinitionError):
208 """
209 A file's tests can't be run as they are written. `collection` says it,
210 with what the run knows: what it has said already for another file.
211 """
212
213 def __init__(self, tests: ProblemsInAFile) -> None:
214 self.tests = tests
215 super().__init__(tests.what_is_wrong())
216
217
218# ---------------------------------------------------------------------------
219# Reading a file's tests from its syntax tree
220# ---------------------------------------------------------------------------
221
222# Decorators that pass a test nothing, by where they come from.
223_PASSES_NOTHING = (
224 "plain.testing.skip",
225 "plain.testing.tag",
226 "plain.postgres.testing.isolated_db",
227)
228
229_CASES = "plain.testing.cases"
230_NEEDS_TO_BE_CALLED = {
231 "plain.testing.skip": '@skip requires a reason: @skip("why")',
232 "plain.testing.tag": '@tag requires at least one name: @tag("slow")',
233}
234
235type _Function = ast.FunctionDef | ast.AsyncFunctionDef
236
237
238def _names_imported(tree: ast.Module) -> dict[str, str]:
239 """
240 What each name a module imports stands for: `{"cases": "plain.testing.cases",
241 "pg": "plain.postgres"}`.
242 """
243 names = {}
244 for node in tree.body:
245 if isinstance(node, ast.Import):
246 for alias in node.names:
247 if alias.asname is not None:
248 names[alias.asname] = alias.name
249 else:
250 # `import plain.testing` binds `plain`.
251 top = alias.name.split(".")[0]
252 names[top] = top
253 elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
254 for alias in node.names:
255 names[alias.asname or alias.name] = f"{node.module}.{alias.name}"
256 return names
257
258
259def _dotted(node: ast.expr) -> str | None:
260 """`plain.testing.skip` for the expression that says so, or None."""
261 parts = []
262 while isinstance(node, ast.Attribute):
263 parts.append(node.attr)
264 node = node.value
265 if not isinstance(node, ast.Name):
266 return None
267 parts.append(node.id)
268 return ".".join(reversed(parts))
269
270
271def _where_it_comes_from(node: ast.expr, *, imported: dict[str, str]) -> str | None:
272 """What an expression names, by the module it was imported from."""
273 dotted = _dotted(node)
274 if dotted is None:
275 return None
276 first, _, rest = dotted.partition(".")
277 origin = imported.get(first)
278 if origin is None:
279 return None
280 return f"{origin}.{rest}" if rest else origin
281
282
283def _yields(function: _Function) -> bool:
284 """Whether a function's own body has a `yield`: not a function inside it."""
285 inside: list[ast.AST] = list(function.body)
286 while inside:
287 node = inside.pop()
288 if isinstance(node, ast.Yield | ast.YieldFrom):
289 return True
290 if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda):
291 continue
292 if isinstance(node, ast.ClassDef):
293 continue
294 inside.extend(ast.iter_child_nodes(node))
295 return False
296
297
298def _read_a_test(
299 function: _Function,
300 *,
301 name: str,
302 imported: dict[str, str],
303 problems: ProblemsInAFile,
304) -> None:
305 if _yields(function):
306 problems.tests_that_yield.append(name)
307 return
308
309 times_cases = 0
310 # Whether every decorator is one this module knows passes no values
311 # but the ones counted here.
312 parameters_are_known = True
313
314 for decorator in function.decorator_list:
315 called = decorator.func if isinstance(decorator, ast.Call) else decorator
316 origin = _where_it_comes_from(called, imported=imported)
317
318 is_called = isinstance(decorator, ast.Call)
319 if not is_called and origin in _NEEDS_TO_BE_CALLED:
320 problems.of_one_test.append(
321 f"line {decorator.lineno}: {_NEEDS_TO_BE_CALLED[origin]}"
322 )
323 problems.stops_at_a_decorator = True
324 continue
325
326 if origin == _CASES:
327 times_cases += 1
328 parameters_are_known = False # its values fill them, by position
329 elif origin is None or not origin.startswith(_PASSES_NOTHING):
330 parameters_are_known = False
331
332 if times_cases > 1:
333 problems.tests_with_cases_twice.append(
334 (f"line {function.lineno}: {name}", times_cases)
335 )
336 problems.stops_at_a_decorator = True
337
338 if not parameters_are_known:
339 return
340
341 arguments = function.args
342 positional = [*arguments.posonlyargs, *arguments.args]
343 with_no_default = positional[: len(positional) - len(arguments.defaults)]
344 keyword_only = [
345 argument
346 for argument, default in zip(
347 arguments.kwonlyargs, arguments.kw_defaults, strict=True
348 )
349 if default is None
350 ]
351 nothing_fills = [*with_no_default, *keyword_only]
352 if not nothing_fills:
353 return
354
355 every_parameter = [*positional, *arguments.kwonlyargs]
356 written = ", ".join(ast.unparse(argument) for argument in every_parameter)
357 problems.tests_taking_parameters.append(f"{name}({written})")
358 for argument in nothing_fills:
359 problems.parameters[argument.arg] = problems.parameters.get(argument.arg, 0) + 1
360
361
362def tests_as_written(tree: ast.Module) -> ProblemsInAFile:
363 """
364 What is wrong with a file's tests, as far as reading it can tell. A
365 test imported from another module is left to `collection`: whether an
366 imported name is a test depends on what it is, not what it is called.
367 """
368 imported = _names_imported(tree)
369 problems = ProblemsInAFile()
370
371 for node in tree.body:
372 if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef):
373 if node.name.startswith("test_"):
374 _read_a_test(
375 node,
376 name=node.name,
377 imported=imported,
378 problems=problems,
379 )
380
381 elif isinstance(node, ast.ClassDef):
382 tests_in_it = [
383 member
384 for member in node.body
385 if isinstance(member, ast.FunctionDef | ast.AsyncFunctionDef)
386 and member.name.startswith("test_")
387 ]
388 if tests_in_it:
389 problems.classes_with_tests.append(
390 (node.lineno, node.name, len(tests_in_it))
391 )
392
393 return problems