Compare commits

...

4 commits

Author SHA1 Message Date
d050a94272 Fix determination of fixture parameters
- Added ability to pass any parameters to `params` keyword in `pytest.fixture`, except:
  1) `params=<callable object>`, but `params=[<callable object>, ...]` is allowed
  2) `params=[*<any collection>]`

- Analysis of `pytest_generate_tests` is almost equivalent to regular fixtures, except:
  1) `metafunc.parametrize(*<any collection>, **<any dict>)`
  2) `metafunc.parametrize("any", <callable object>)`
  3) `metafunc.parametrize("any1, any2", [(<callable object>, [*<any collection>])]`

Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-11-15 16:08:24 +03:00
cd41eb5d03 Add plugin support
The following flags have been added:
- `--plugins` - patterns that plugins must match.
- `--files` - paths to files with fixtures.

Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-09-01 13:29:32 +03:00
65fab7f1c5 Fix fixture lookup in pytest_generate_tests
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-08-20 18:32:42 +03:00
156c32a0cb Add pre-commit support
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-08-20 16:51:44 +03:00
11 changed files with 403 additions and 57 deletions

7
.pre-commit-hooks.yaml Normal file
View file

@ -0,0 +1,7 @@
- id: allure-validator
name: allure-validator
entry: allure-validator
language: python
minimum_pre_commit_version: 2.20.0
require_serial: true
types_or: [python, pyi]

View file

@ -13,22 +13,19 @@ pip install git+https://git.frostfs.info/TrueCloudLab/allure-validator.git@maste
To start, simply specify the folder with the test database that needs to be checked:
```shell
allure-validator <folder>
allure-validator <folder> [OPTIONS]
```
Additionally, `allure-validator` can be added to pre-commit hooks:
```yaml
# .pre-commit-config.yaml
repos:
- repo: local
- repo: https://git.frostfs.info/TrueCloudLab/allure-validator
rev: latest
hooks:
- id: allure-validator
name: allure-validator
entry: allure-validator
language: system
args: ["pytest_tests/"] # folder with tests
args: ["pytest_tests/"]
pass_filenames: false
types: [python]
```
### Ignore mechanism
@ -52,6 +49,17 @@ class TestIngoredClass: # noqa: allure-validator
> Note: this may also miss fixtures, so use with caution.
### Options
The linter supports the following flags:
- `--plugins` - regex patterns that plugins must match (paths are compiled according to the template: `plugin_name / src / plugin_name`).
- `--files` - paths to files with fixtures.
Configuration options can also be provided using special command line arguments, for example:
```shell
allure-validator <tests folder> --plugins <plugin regex>, ... --files <path to file with fixtures>, ...
```
## Work example
Let's run the linter for some [sample tests](https://git.frostfs.info/Kiriruso/allure-validator/src/branch/master/tests):
@ -96,9 +104,12 @@ Dynamic fixtures are only supported if they are declared in `pytest_generate_tes
```python
def pytest_generate_tests(metafunc):
...
metafunc.parametrize("fixture1, fixture2", [(1, 2), (3, 4)]) # OK
...
if some_condition:
metafunc.parametrize("fixture3", *some_value_generator) # OK
```
Plugins are not supported (**temporarily**), so fixtures and tests from there cannot be found.
```

View file

@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "allure-validator"
version = "1.0.0"
version = "1.1.1"
description = "Linter for allure.title validation"
readme = "README.md"
authors = [{ name = "YADRO", email = "info@yadro.com" }]
@ -21,7 +21,7 @@ dependencies = [
"pytest>=7.1.2",
"pytest-lazy-fixture>=0.6.3"
]
requires-python = "==3.10.*"
requires-python = ">=3.10"
[project.optional-dependencies]
dev = ["black", "pylint", "isort", "pre-commit"]

View file

@ -1,10 +1,41 @@
import argparse
class _SubcommandHelpFormatter(argparse.RawDescriptionHelpFormatter):
def __init__(self, prog):
super().__init__(prog, max_help_position=80)
def _format_action(self, action):
parts = super(argparse.RawDescriptionHelpFormatter, self)._format_action(action)
if action.nargs == argparse.PARSER:
parts = "\n".join(parts.split("\n")[1:])
return parts
parser = argparse.ArgumentParser(
"pyfactoring",
"%(prog)s <folder path>",
"%(prog)s <folder path> [OPTIONS]",
"Linter for allure.title validation",
formatter_class=_SubcommandHelpFormatter,
)
parser.add_argument(
"folder",
type=str,
help="folder with tests",
)
parser.add_argument(
"--plugins",
metavar="plugin,",
nargs="*",
default=[],
help="list plugins for analysis",
)
parser.add_argument(
"--files",
metavar="file,",
nargs="*",
default=[],
help="list paths to files with fixtures",
)
parser.add_argument("folder", type=str, help="folder with tests")
args = parser.parse_args()

View file

@ -35,6 +35,53 @@ def paths(folder: str) -> list[Path]:
return paths
def files(filepaths: list[str]) -> list[Path]:
"""
Converts the given paths to `pathlib.Path` objects and checks that the file exists.
Args:
filepaths: Paths to files with fixtures.
Returns:
list[Path]: List of paths to .py files.
"""
paths = []
for path in filepaths:
path = Path(path)
if not path.exists():
raise FileNotFoundError(f"File '{path}' not found")
paths.append(path)
return paths
def plugins(plugin_pattern: str, folder: str) -> list[Path]:
"""
Collects paths to files with fixtures in plugin directories that match a given pattern.
Args:
plugin_pattern: Pattern that plugins must follow.
folder: Source folder with tests.
Returns:
list[Path]: List of paths to .py files.
"""
pytest_tests = Path(folder).absolute()
workspace = pytest_tests.parent.parent if pytest_tests.name == "pytest_tests" else pytest_tests.parent
plugins = _matching_paths(plugin_pattern, workspace)
paths = []
for plugin in plugins:
if plugin is None or not plugin.exists():
raise ModuleNotFoundError(f"Plugin with '{plugin_pattern}' pattern not found in '{workspace}'")
paths.extend(_plugin_paths(plugin))
return paths
def ast_pytest_items(paths: list[Path]) -> ASTPytestItems:
"""
Collects AST objects of pytest tests, fixtures and hooks.
@ -97,7 +144,7 @@ def ast_pytest_items(paths: list[Path]) -> ASTPytestItems:
return items
def fixtures(ast_fixtures: list[ast.FunctionDef], ast_hooks: list[ast.FunctionDef]) -> dict[tuple, Fixture]:
def fixtures(ast_fixtures: list[ast.FunctionDef], ast_hooks: list[ast.FunctionDef], plugin_paths: list[Path]) -> dict[tuple, Fixture]:
"""
From AST objects it forms fixtures and connections between these fixtures.
@ -133,21 +180,32 @@ def fixtures(ast_fixtures: list[ast.FunctionDef], ast_hooks: list[ast.FunctionDe
for ast_hook in ast_hooks:
path = getattr(ast_hook, ATTR_PATH)
for node in ast_hook.body:
if isinstance(node, ast.Expr):
if parse.decorator(node.value) == PYTEST_METAFUNC_PARAMETRIZE:
fixture_names = parse.param_names(node.value.args[0])
for name in fixture_names:
# Dynamic fake fixture
fixture = Fixture(
name,
path,
node.lineno,
node.col_offset,
args=[],
params=["parametrized"],
)
fixtures[fixture] = fixture
for node in ast.walk(ast_hook):
if not isinstance(node, ast.Expr):
continue
if parse.decorator(node.value) != PYTEST_METAFUNC_PARAMETRIZE:
continue
if len(node.value.args) < 2:
raise SyntaxError(f"{path}:{ast_hook.lineno}:{ast_hook.col_offset}: Unexpected syntax in {PYTEST_METAFUNC_PARAMETRIZE}")
ast_names, ast_params, *_ = node.value.args
fixture_names = parse.param_names(ast_names)
for name in fixture_names:
params = parse.metafunc_params(ast_params)
# Dynamic fixture
fixture = Fixture(
name,
path,
node.lineno,
node.col_offset,
args=[],
params=params,
)
fixtures[fixture] = fixture
for fixture in fixtures.values():
for arg in fixture.args:
@ -171,12 +229,19 @@ def fixtures(ast_fixtures: list[ast.FunctionDef], ast_hooks: list[ast.FunctionDe
is_conftest = current_path.name == "conftest.py"
current_path = current_path.parent.parent / "conftest.py" if is_conftest else current_path.parent / "conftest.py"
# TODO: Finding fixture from plugins
# Finding fixture from plugins
for plugin_path in plugin_paths:
possible_fixture = (arg, plugin_path, None)
if possible_fixture in fixtures.keys():
depend = fixtures[possible_fixture]
fixture.depends.add(depend)
break
return fixtures
def tests(ast_tests: list[ast.FunctionDef], fixtures: dict[tuple, Fixture]) -> list[Test]:
def tests(ast_tests: list[ast.FunctionDef], fixtures: dict[tuple, Fixture], plugin_paths: list[Path]) -> list[Test]:
"""
From AST objects it forms fixtures and connections between these fixtures.
@ -209,7 +274,7 @@ def tests(ast_tests: list[ast.FunctionDef], fixtures: dict[tuple, Fixture]) -> l
in_class=in_class,
)
depends = _only_parametirzed_depends(test, fixtures)
depends = _only_parametirzed_depends(test, fixtures, plugin_paths)
depends = {depend for depend in depends if depend.name not in params.optional}
# Test parameters take precedence over fixtures
@ -223,6 +288,50 @@ def tests(ast_tests: list[ast.FunctionDef], fixtures: dict[tuple, Fixture]) -> l
return tests
def _matching_paths(plugin_pattern: str, workspace: Path) -> list[Path]:
"""
Collects paths that match a pattern.
Paths are collected according to the following pattern: `pattern / src / pattern`.
"""
plugins = []
for outer_folder in workspace.glob(plugin_pattern):
if not outer_folder.exists():
continue
outer_folder /= "src"
for inner_folder in outer_folder.glob(plugin_pattern):
if not inner_folder.exists():
continue
plugins.append(inner_folder)
return plugins
def _plugin_paths(plugin: Path) -> list[Path]:
"""Collects paths to files with fixtures from `__init__.py` file."""
paths = []
if plugin.name.endswith(".egg-info"):
return []
init_path = plugin / "__init__.py"
if not init_path.exists():
print(f"Plugin '{plugin.name}' was skipped: '__init__.py' not found")
return []
with open(init_path, "r", encoding="utf-8") as init:
import_lines = [line.strip() for line in init.readlines() if line != "\n" and line.startswith("from")]
for line in import_lines:
if path := parse.import_path(line):
paths.append(plugin / path)
return paths
def _all_depends(fixture: Fixture) -> set[Fixture]:
"""Get all dependencies of the passed fixture."""
@ -236,7 +345,7 @@ def _all_depends(fixture: Fixture) -> set[Fixture]:
return depends
def _only_parametirzed_depends(test: Test, fixtures: dict[tuple, Fixture]) -> set[Fixture]:
def _only_parametirzed_depends(test: Test, fixtures: dict[tuple, Fixture], plugin_paths: list[Path]) -> set[Fixture]:
"""Get only parameterized dependencies for the given test."""
depends = set()
@ -262,6 +371,13 @@ def _only_parametirzed_depends(test: Test, fixtures: dict[tuple, Fixture]) -> se
is_conftest = current_path.name == "conftest.py"
current_path = current_path.parent.parent / "conftest.py" if is_conftest else current_path.parent / "conftest.py"
# TODO: Finding fixture from plugins
# Finding fixture from plugins
for plugin_path in plugin_paths:
possible_fixture = (arg, plugin_path, None)
if possible_fixture in fixtures.keys():
fixture = fixtures[possible_fixture]
depends.update(depend for depend in _all_depends(fixture) if depend.parametrized)
break
return depends

View file

@ -3,8 +3,10 @@ REPORTER_TITLE = "reporter.title"
PYTEST_FIXTURE = "pytest.fixture"
PYTEST_PARAMETRIZE = "pytest.mark.parametrize"
PYTEST_PARAM = "pytest.param"
PYTEST_GENERATE_TESTS = "pytest_generate_tests"
PYTEST_METAFUNC_PARAMETRIZE = "metafunc.parametrize"
PYTEST_FORCED_PARAMETRIZATION = ["forced", "parametrized", "fixture"]
ATTR_PATH = "__func_path"
ATTR_CLASS_NAME = "__class_name"
@ -12,5 +14,8 @@ ATTR_CLASS_NAME = "__class_name"
VALIDATE_DONE = 0
VALIDATE_ERROR = 1
DIRECTORY_ERROR = 2
PLUGIN_ERROR = 3
FILE_ERROR = 4
SYNTAX_ERROR = 5
ALLURE_VALIDATOR_IGNORE = "noqa: allure-validator"

View file

@ -1,27 +1,46 @@
import sys
from allure_validator import collect, validate
from allure_validator.arguments import args
from allure_validator.common import DIRECTORY_ERROR, VALIDATE_DONE, VALIDATE_ERROR
from allure_validator.common import DIRECTORY_ERROR, FILE_ERROR, PLUGIN_ERROR, SYNTAX_ERROR, VALIDATE_DONE, VALIDATE_ERROR
def main():
tests_folder = args.folder
try:
paths = collect.paths(tests_folder)
paths = collect.paths(args.folder)
files = collect.files(args.files)
plugins = []
for pattern in args.plugins:
plugins.extend(collect.plugins(pattern, args.folder))
paths.extend(plugins)
paths.extend(files)
except NotADirectoryError as e:
print(e)
sys.exit(DIRECTORY_ERROR)
exit(DIRECTORY_ERROR)
except ModuleNotFoundError as e:
print(e)
exit(PLUGIN_ERROR)
except FileNotFoundError as e:
print(e)
exit(FILE_ERROR)
except SyntaxError as e:
print(e)
exit(SYNTAX_ERROR)
ast_items = collect.ast_pytest_items(paths)
fixtures = collect.fixtures(ast_items.fixtures, ast_items.hooks)
tests = collect.tests(ast_items.tests, fixtures)
fixtures = collect.fixtures(ast_items.fixtures, ast_items.hooks, plugins)
tests = collect.tests(ast_items.tests, fixtures, plugins)
validate_result = validate.titles(tests)
if validate_result.errors == 0:
print(f"Done🎉\nFiles analyzed: {len(paths)}\nTests analyzed: {len(tests)}")
sys.exit(VALIDATE_DONE)
if plugins:
patterns = ", ".join(args.plugins)
print(f"Found plugins with patterns: {patterns}")
print(*(f"-> {plugin}" for plugin in plugins), sep="\n")
exit(VALIDATE_DONE)
for title, tests in validate_result.not_unique.items():
print(f"NOT UNIQUE TITLE: {title}")
@ -37,4 +56,9 @@ def main():
params_str = " OR ".join(params) if isinstance(params, list) else params
print(f"{test.link}: MISSING PARAMS: Parameters are missing from title: {params_str}")
sys.exit(VALIDATE_ERROR)
if plugins:
patterns = ", ".join(args.plugins)
print(f"Found plugins with patterns: {patterns}")
print(*(f"-> {plugin}" for plugin in plugins), sep="\n")
exit(VALIDATE_ERROR)

View file

@ -1,10 +1,29 @@
import ast
import re
from pathlib import Path
from allure_validator.common import ALLURE_TITLE, ALLURE_VALIDATOR_IGNORE, PYTEST_FIXTURE, PYTEST_PARAMETRIZE, REPORTER_TITLE
from allure_validator.common import (
ALLURE_TITLE,
ALLURE_VALIDATOR_IGNORE,
PYTEST_FIXTURE,
PYTEST_FORCED_PARAMETRIZATION,
PYTEST_PARAM,
PYTEST_PARAMETRIZE,
REPORTER_TITLE,
)
from allure_validator.pytest_items import TestParams
def import_path(import_line: str) -> Path | None:
pattern = r"(?<=from )(?:\.[A-Za-z]+)+(?= )"
if path := re.search(pattern, import_line):
path = path.group().lstrip(".").replace(".", "/")
return Path(f"{path}.py")
return None
def decorator(deco: ast.Call | ast.Attribute) -> str:
"""
Retrieves the decorator name as a string.
@ -119,25 +138,67 @@ def fixture_params(func: ast.FunctionDef) -> list[str]:
"""
ast_params = []
for deco in func.decorator_list:
if not isinstance(deco, ast.Call):
continue
for keyword in deco.keywords:
if keyword.arg == "params":
ast_params.extend(keyword.value.elts)
if isinstance(keyword.value, ast.List | ast.Tuple):
ast_params.extend(keyword.value.elts)
else:
return PYTEST_FORCED_PARAMETRIZATION
params = []
for ast_param in ast_params:
for arg in ast_param.args:
params.append(ast.unparse(arg))
if isinstance(ast_param, ast.Call) and decorator(ast_param) == PYTEST_PARAM:
params.extend(ast.unparse(arg) for arg in ast_param.args)
continue
if not isinstance(ast_param, ast.Starred):
params.append(ast.unparse(ast_param))
continue
return PYTEST_FORCED_PARAMETRIZATION
return params
def param_names(ast_names: ast.Constant | ast.List) -> list[str]:
def param_names(ast_names: ast.Constant | ast.List | ast.Tuple) -> list[str]:
"""Generates a list of parameter names from `pytest.parametrize`."""
if isinstance(ast_names, ast.Constant):
return ast_names.value.replace(",", " ").split()
return [elt.value for elt in ast_names.elts]
def metafunc_params(ast_params: ast.List | ast.Tuple) -> list:
"""
Extracts parameters for the fixture from the `params` argument of the `metafunc.parametrize` function.
Args:
ast_params: Node from which parameters should be extracted.
Returns:
list: Extracted parameters.
"""
if not isinstance(ast_params, ast.List | ast.Tuple):
return PYTEST_FORCED_PARAMETRIZATION
if any(isinstance(elt, ast.Starred) for elt in ast_params.elts):
return PYTEST_FORCED_PARAMETRIZATION
params = []
for ast_param in ast_params.elts:
if isinstance(ast_param, ast.Constant):
params.append(ast_param.value)
elif isinstance(ast_param, ast.Name | ast.Attribute):
params.append(ast.unparse(ast_param))
else:
SyntaxError(ast.unparse(ast_param))
return params

View file

@ -39,7 +39,7 @@ class Fixture(Item):
@property
def parametrized(self) -> bool:
return bool(self.params)
return len(self.params) > 1
@dataclass(eq=False)

View file

@ -2,6 +2,10 @@ import allure
import pytest
def parametrize_fixture() -> list[int]:
return [1, 2, 3]
@pytest.fixture(
scope="function",
params=[
@ -34,3 +38,12 @@ def object_fixture(file_path):
)
def parametrized_fixture(request: pytest.FixtureRequest):
return request.param
@allure.title("[Session] Optional parametrized fixture")
@pytest.fixture(
scope="session",
params=[parametrize_fixture()],
)
def optional_parametrized_fixture(request: pytest.FixtureRequest):
return request.param

View file

@ -1,18 +1,91 @@
import allure
import pytest
from frostfs_testlib.s3.aws_cli_client import AwsCliClient
from frostfs_testlib.s3.boto3_client import Boto3ClientWrapper
class Clients:
AttributeClient = ...
def parametrize_fixture() -> list[int]:
return [1, 2, 3]
# ============== DYNAMIC FIXTURE ============== #
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "runtime_fixture" not in metafunc.fixturenames:
return
# Required dynamic fixture
if "required_runtime_fixture" in metafunc.fixturenames:
metafunc.parametrize("required_runtime_fixture", [AwsCliClient, Boto3ClientWrapper, Clients.AttributeClient])
metafunc.parametrize("runtime_fixture", [1, 2, 3])
# Optional dynamic fixture 1
if "optional_runtime_fixture_1" in metafunc.fixturenames:
metafunc.parametrize("optional_runtime_fixture_1", [True])
# Optional dynamic fixture 2
if "optional_runtime_fixture_2" in metafunc.fixturenames:
metafunc.parametrize("optional_runtime_fixture_2", [parametrize_fixture()])
# Required dynamic fixtures
if "required_runtime_1" in metafunc.fixturenames and "required_runtime_2" in metafunc.fixturenames:
metafunc.parametrize("required_runtime_1,required_runtime_2", [(1, 2), (3, 4)])
# Optional dynamic fixtures
if "optional_runtime_1" in metafunc.fixturenames and "optional_runtime_2" in metafunc.fixturenames:
metafunc.parametrize(["optional_runtime_1", "optional_runtime_2"], [("optional", [1, 2, 3])])
# Required dynamic fixture with implicit parameterization 1
if "implicitly_required_runtime_fixture_1" in metafunc.fixturenames:
metafunc.parametrize("implicitly_required_runtime_fixture_1", parametrize_fixture())
# Required dynamic fixture with implicit parameterization 2
if "implicitly_required_runtime_fixture_2" in metafunc.fixturenames:
metafunc.parametrize("implicitly_required_runtime_fixture_2", [*parametrize_fixture()])
# Required dynamic fixture with implicit parameterization 3
if "implicitly_required_runtime_fixture_3" in metafunc.fixturenames:
metafunc.parametrize("implicitly_required_runtime_fixture_3", [0, parametrize_fixture(), *parametrize_fixture()])
@allure.title("Test runtime fixture")
def test_runtime_fixture(runtime_fixture):
@allure.title("Test required runtime fixture")
def test_required_runtime_fixture(required_runtime_fixture):
assert True
@allure.title("Test optional runtime fixture 1")
def test_optional_runtime_fixture_1(optional_runtime_fixture_1):
assert True
@allure.title("Test optional runtime fixture 2")
def test_optional_runtime_fixture_2(optional_runtime_fixture_2):
assert True
@allure.title("Test required runtime fixtures")
def test_required_runtime_fixtures(required_runtime_1, required_runtime_2):
assert True
@allure.title("Test optional runtime fixtures")
def test_optional_runtime_fixtures(optional_runtime_1, optional_runtime_2):
assert True
@allure.title("Test implicitly required runtime fixture 1")
def test_implicitly_required_runtime_fixture_1(implicitly_required_runtime_fixture_1):
assert True
@allure.title("Test implicitly required runtime fixture 2")
def test_implicitly_required_runtime_fixture_2(implicitly_required_runtime_fixture_2):
assert True
@allure.title("Test implicitly required runtime fixture 3")
def test_implicitly_required_runtime_fixture_3(implicitly_required_runtime_fixture_3):
assert True
@ -117,6 +190,11 @@ def test_three_multiple_indirect(object_fixture: str):
assert True
@allure.title("Test optional fixture")
def test_optional_fixture(optional_parametrized_fixture):
assert True
# ============== TESTS/FIXTURES IN CLASS ============== #