This commit is contained in:
Iliyan Angelov
2025-12-06 03:27:35 +02:00
parent 7667eb5eda
commit 5a8ca3c475
2211 changed files with 28086 additions and 37066 deletions

View File

@@ -1,25 +1,24 @@
# mypy: allow-untyped-defs
"""Implementation of the cache provider."""
# This plugin was not named "cache" to avoid conflicts with the external
# pytest-cache version.
from __future__ import annotations
from collections.abc import Generator
from collections.abc import Iterable
import dataclasses
import errno
import json
import os
from pathlib import Path
import tempfile
from typing import final
from typing import Dict
from typing import Generator
from typing import Iterable
from typing import List
from typing import Optional
from typing import Set
from typing import Union
from .pathlib import resolve_from_str
from .pathlib import rm_rf
from .reports import CollectReport
from _pytest import nodes
from _pytest._io import TerminalWriter
from _pytest.compat import final
from _pytest.config import Config
from _pytest.config import ExitCode
from _pytest.config import hookimpl
@@ -28,11 +27,10 @@ from _pytest.deprecated import check_ispytest
from _pytest.fixtures import fixture
from _pytest.fixtures import FixtureRequest
from _pytest.main import Session
from _pytest.nodes import Directory
from _pytest.nodes import File
from _pytest.python import Package
from _pytest.reports import TestReport
README_CONTENT = """\
# pytest cache directory #
@@ -74,7 +72,7 @@ class Cache:
self._config = config
@classmethod
def for_config(cls, config: Config, *, _ispytest: bool = False) -> Cache:
def for_config(cls, config: Config, *, _ispytest: bool = False) -> "Cache":
"""Create the Cache instance for a Config.
:meta private:
@@ -113,7 +111,6 @@ class Cache:
"""
check_ispytest(_ispytest)
import warnings
from _pytest.warning_types import PytestCacheWarning
warnings.warn(
@@ -122,10 +119,6 @@ class Cache:
stacklevel=3,
)
def _mkdir(self, path: Path) -> None:
self._ensure_cache_dir_and_supporting_files()
path.mkdir(exist_ok=True, parents=True)
def mkdir(self, name: str) -> Path:
"""Return a directory path object with the given name.
@@ -144,7 +137,7 @@ class Cache:
if len(path.parts) > 1:
raise ValueError("name is not allowed to contain path separators")
res = self._cachedir.joinpath(self._CACHE_PREFIX_DIRS, path)
self._mkdir(res)
res.mkdir(exist_ok=True, parents=True)
return res
def _getvaluepath(self, key: str) -> Path:
@@ -181,13 +174,19 @@ class Cache:
"""
path = self._getvaluepath(key)
try:
self._mkdir(path.parent)
if path.parent.is_dir():
cache_dir_exists_already = True
else:
cache_dir_exists_already = self._cachedir.exists()
path.parent.mkdir(exist_ok=True, parents=True)
except OSError as exc:
self.warn(
f"could not create cache path {path}: {exc}",
_ispytest=True,
)
return
if not cache_dir_exists_already:
self._ensure_supporting_files()
data = json.dumps(value, ensure_ascii=False, indent=2)
try:
f = path.open("w", encoding="UTF-8")
@@ -200,85 +199,60 @@ class Cache:
with f:
f.write(data)
def _ensure_cache_dir_and_supporting_files(self) -> None:
"""Create the cache dir and its supporting files."""
if self._cachedir.is_dir():
return
def _ensure_supporting_files(self) -> None:
"""Create supporting files in the cache dir that are not really part of the cache."""
readme_path = self._cachedir / "README.md"
readme_path.write_text(README_CONTENT, encoding="UTF-8")
self._cachedir.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix="pytest-cache-files-",
dir=self._cachedir.parent,
) as newpath:
path = Path(newpath)
gitignore_path = self._cachedir.joinpath(".gitignore")
msg = "# Created by pytest automatically.\n*\n"
gitignore_path.write_text(msg, encoding="UTF-8")
# Reset permissions to the default, see #12308.
# Note: there's no way to get the current umask atomically, eek.
umask = os.umask(0o022)
os.umask(umask)
path.chmod(0o777 - umask)
with open(path.joinpath("README.md"), "x", encoding="UTF-8") as f:
f.write(README_CONTENT)
with open(path.joinpath(".gitignore"), "x", encoding="UTF-8") as f:
f.write("# Created by pytest automatically.\n*\n")
with open(path.joinpath("CACHEDIR.TAG"), "xb") as f:
f.write(CACHEDIR_TAG_CONTENT)
try:
path.rename(self._cachedir)
except OSError as e:
# If 2 concurrent pytests both race to the rename, the loser
# gets "Directory not empty" from the rename. In this case,
# everything is handled so just continue (while letting the
# temporary directory be cleaned up).
# On Windows, the error is a FileExistsError which translates to EEXIST.
if e.errno not in (errno.ENOTEMPTY, errno.EEXIST):
raise
else:
# Create a directory in place of the one we just moved so that
# `TemporaryDirectory`'s cleanup doesn't complain.
#
# TODO: pass ignore_cleanup_errors=True when we no longer support python < 3.10.
# See https://github.com/python/cpython/issues/74168. Note that passing
# delete=False would do the wrong thing in case of errors and isn't supported
# until python 3.12.
path.mkdir()
cachedir_tag_path = self._cachedir.joinpath("CACHEDIR.TAG")
cachedir_tag_path.write_bytes(CACHEDIR_TAG_CONTENT)
class LFPluginCollWrapper:
def __init__(self, lfplugin: LFPlugin) -> None:
def __init__(self, lfplugin: "LFPlugin") -> None:
self.lfplugin = lfplugin
self._collected_at_least_one_failure = False
@hookimpl(wrapper=True)
def pytest_make_collect_report(
self, collector: nodes.Collector
) -> Generator[None, CollectReport, CollectReport]:
res = yield
if isinstance(collector, Session | Directory):
@hookimpl(hookwrapper=True)
def pytest_make_collect_report(self, collector: nodes.Collector):
if isinstance(collector, (Session, Package)):
out = yield
res: CollectReport = out.get_result()
# Sort any lf-paths to the beginning.
lf_paths = self.lfplugin._last_failed_paths
# Use stable sort to prioritize last failed.
def sort_key(node: nodes.Item | nodes.Collector) -> bool:
return node.path in lf_paths
# Use stable sort to priorize last failed.
def sort_key(node: Union[nodes.Item, nodes.Collector]) -> bool:
# Package.path is the __init__.py file, we need the directory.
if isinstance(node, Package):
path = node.path.parent
else:
path = node.path
return path in lf_paths
res.result = sorted(
res.result,
key=sort_key,
reverse=True,
)
return
elif isinstance(collector, File):
if collector.path in self.lfplugin._last_failed_paths:
out = yield
res = out.get_result()
result = res.result
lastfailed = self.lfplugin.lastfailed
# Only filter with known failures.
if not self._collected_at_least_one_failure:
if not any(x.nodeid in lastfailed for x in result):
return res
return
self.lfplugin.config.pluginmanager.register(
LFPluginCollSkipfiles(self.lfplugin), "lfplugin-collskip"
)
@@ -294,19 +268,21 @@ class LFPluginCollWrapper:
# Keep all sub-collectors.
or isinstance(x, nodes.Collector)
]
return res
return
yield
class LFPluginCollSkipfiles:
def __init__(self, lfplugin: LFPlugin) -> None:
def __init__(self, lfplugin: "LFPlugin") -> None:
self.lfplugin = lfplugin
@hookimpl
def pytest_make_collect_report(
self, collector: nodes.Collector
) -> CollectReport | None:
if isinstance(collector, File):
) -> Optional[CollectReport]:
# Packages are Files, but we only want to skip test-bearing Files,
# so don't filter Packages.
if isinstance(collector, File) and not isinstance(collector, Package):
if collector.path not in self.lfplugin._last_failed_paths:
self.lfplugin._skipped_files += 1
@@ -324,9 +300,9 @@ class LFPlugin:
active_keys = "lf", "failedfirst"
self.active = any(config.getoption(key) for key in active_keys)
assert config.cache
self.lastfailed: dict[str, bool] = config.cache.get("cache/lastfailed", {})
self._previously_failed_count: int | None = None
self._report_status: str | None = None
self.lastfailed: Dict[str, bool] = config.cache.get("cache/lastfailed", {})
self._previously_failed_count: Optional[int] = None
self._report_status: Optional[str] = None
self._skipped_files = 0 # count skipped files during collection due to --lf
if config.getoption("lf"):
@@ -335,7 +311,7 @@ class LFPlugin:
LFPluginCollWrapper(self), "lfplugin-collwrapper"
)
def get_last_failed_paths(self) -> set[Path]:
def get_last_failed_paths(self) -> Set[Path]:
"""Return a set with all Paths of the previously failed nodeids and
their parents."""
rootpath = self.config.rootpath
@@ -346,9 +322,9 @@ class LFPlugin:
result.update(path.parents)
return {x for x in result if x.exists()}
def pytest_report_collectionfinish(self) -> str | None:
if self.active and self.config.get_verbosity() >= 0:
return f"run-last-failure: {self._report_status}"
def pytest_report_collectionfinish(self) -> Optional[str]:
if self.active and self.config.getoption("verbose") >= 0:
return "run-last-failure: %s" % self._report_status
return None
def pytest_runtest_logreport(self, report: TestReport) -> None:
@@ -366,14 +342,14 @@ class LFPlugin:
else:
self.lastfailed[report.nodeid] = True
@hookimpl(wrapper=True, tryfirst=True)
@hookimpl(hookwrapper=True, tryfirst=True)
def pytest_collection_modifyitems(
self, config: Config, items: list[nodes.Item]
) -> Generator[None]:
res = yield
self, config: Config, items: List[nodes.Item]
) -> Generator[None, None, None]:
yield
if not self.active:
return res
return
if self.lastfailed:
previously_failed = []
@@ -388,8 +364,8 @@ class LFPlugin:
if not previously_failed:
# Running a subset of all tests with recorded failures
# only outside of it.
self._report_status = (
f"{len(self.lastfailed)} known failures not in selected tests"
self._report_status = "%d known failures not in selected tests" % (
len(self.lastfailed),
)
else:
if self.config.getoption("lf"):
@@ -400,13 +376,15 @@ class LFPlugin:
noun = "failure" if self._previously_failed_count == 1 else "failures"
suffix = " first" if self.config.getoption("failedfirst") else ""
self._report_status = (
f"rerun previous {self._previously_failed_count} {noun}{suffix}"
self._report_status = "rerun previous {count} {noun}{suffix}".format(
count=self._previously_failed_count, suffix=suffix, noun=noun
)
if self._skipped_files > 0:
files_noun = "file" if self._skipped_files == 1 else "files"
self._report_status += f" (skipped {self._skipped_files} {files_noun})"
self._report_status += " (skipped {files} {files_noun})".format(
files=self._skipped_files, files_noun=files_noun
)
else:
self._report_status = "no previously failed tests, "
if self.config.getoption("last_failed_no_failures") == "none":
@@ -416,8 +394,6 @@ class LFPlugin:
else:
self._report_status += "not deselecting items."
return res
def pytest_sessionfinish(self, session: Session) -> None:
config = self.config
if config.getoption("cacheshow") or hasattr(config, "workerinput"):
@@ -438,13 +414,15 @@ class NFPlugin:
assert config.cache is not None
self.cached_nodeids = set(config.cache.get("cache/nodeids", []))
@hookimpl(wrapper=True, tryfirst=True)
def pytest_collection_modifyitems(self, items: list[nodes.Item]) -> Generator[None]:
res = yield
@hookimpl(hookwrapper=True, tryfirst=True)
def pytest_collection_modifyitems(
self, items: List[nodes.Item]
) -> Generator[None, None, None]:
yield
if self.active:
new_items: dict[str, nodes.Item] = {}
other_items: dict[str, nodes.Item] = {}
new_items: Dict[str, nodes.Item] = {}
other_items: Dict[str, nodes.Item] = {}
for item in items:
if item.nodeid not in self.cached_nodeids:
new_items[item.nodeid] = item
@@ -458,10 +436,8 @@ class NFPlugin:
else:
self.cached_nodeids.update(item.nodeid for item in items)
return res
def _get_increasing_order(self, items: Iterable[nodes.Item]) -> list[nodes.Item]:
return sorted(items, key=lambda item: item.path.stat().st_mtime, reverse=True)
def _get_increasing_order(self, items: Iterable[nodes.Item]) -> List[nodes.Item]:
return sorted(items, key=lambda item: item.path.stat().st_mtime, reverse=True) # type: ignore[no-any-return]
def pytest_sessionfinish(self) -> None:
config = self.config
@@ -476,17 +452,14 @@ class NFPlugin:
def pytest_addoption(parser: Parser) -> None:
"""Add command-line options for cache functionality.
:param parser: Parser object to add command-line options to.
"""
group = parser.getgroup("general")
group.addoption(
"--lf",
"--last-failed",
action="store_true",
dest="lf",
help="Rerun only the tests that failed at the last run (or all if none failed)",
help="Rerun only the tests that failed "
"at the last run (or all if none failed)",
)
group.addoption(
"--ff",
@@ -540,7 +513,7 @@ def pytest_addoption(parser: Parser) -> None:
)
def pytest_cmdline_main(config: Config) -> int | ExitCode | None:
def pytest_cmdline_main(config: Config) -> Optional[Union[int, ExitCode]]:
if config.option.cacheshow and not config.option.help:
from _pytest.main import wrap_session
@@ -550,13 +523,6 @@ def pytest_cmdline_main(config: Config) -> int | ExitCode | None:
@hookimpl(tryfirst=True)
def pytest_configure(config: Config) -> None:
"""Configure cache system and register related plugins.
Creates the Cache instance and registers the last-failed (LFPlugin)
and new-first (NFPlugin) plugins with the plugin manager.
:param config: pytest configuration object.
"""
config.cache = Cache.for_config(config, _ispytest=True)
config.pluginmanager.register(LFPlugin(config), "lfplugin")
config.pluginmanager.register(NFPlugin(config), "nfplugin")
@@ -578,7 +544,7 @@ def cache(request: FixtureRequest) -> Cache:
return request.config.cache
def pytest_report_header(config: Config) -> str | None:
def pytest_report_header(config: Config) -> Optional[str]:
"""Display cachedir with --cache-show and if non-default."""
if config.option.verbose > 0 or config.getini("cache_dir") != ".pytest_cache":
assert config.cache is not None
@@ -595,16 +561,6 @@ def pytest_report_header(config: Config) -> str | None:
def cacheshow(config: Config, session: Session) -> int:
"""Display cache contents when --cache-show is used.
Shows cached values and directories matching the specified glob pattern
(default: '*'). Displays cache location, cached test results, and
any cached directories created by plugins.
:param config: pytest configuration object.
:param session: pytest session object.
:returns: Exit code (0 for success).
"""
from pprint import pformat
assert config.cache is not None
@@ -622,25 +578,25 @@ def cacheshow(config: Config, session: Session) -> int:
dummy = object()
basedir = config.cache._cachedir
vdir = basedir / Cache._CACHE_PREFIX_VALUES
tw.sep("-", f"cache values for {glob!r}")
tw.sep("-", "cache values for %r" % glob)
for valpath in sorted(x for x in vdir.rglob(glob) if x.is_file()):
key = str(valpath.relative_to(vdir))
val = config.cache.get(key, dummy)
if val is dummy:
tw.line(f"{key} contains unreadable content, will be ignored")
tw.line("%s contains unreadable content, will be ignored" % key)
else:
tw.line(f"{key} contains:")
tw.line("%s contains:" % key)
for line in pformat(val).splitlines():
tw.line(" " + line)
ddir = basedir / Cache._CACHE_PREFIX_DIRS
if ddir.is_dir():
contents = sorted(ddir.rglob(glob))
tw.sep("-", f"cache directories for {glob!r}")
tw.sep("-", "cache directories for %r" % glob)
for p in contents:
# if p.is_dir():
# print("%s/" % p.relative_to(basedir))
if p.is_file():
key = str(p.relative_to(basedir))
tw.line(f"{key} is a file of length {p.stat().st_size}")
tw.line(f"{key} is a file of length {p.stat().st_size:d}")
return 0