updates
This commit is contained in:
@@ -1,25 +1,24 @@
|
||||
# mypy: allow-untyped-defs
|
||||
"""Implementation of the cache provider."""
|
||||
|
||||
# This plugin was not named "cache" to avoid conflicts with the external
|
||||
# pytest-cache version.
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Generator
|
||||
from collections.abc import Iterable
|
||||
import dataclasses
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from typing import final
|
||||
from typing import Dict
|
||||
from typing import Generator
|
||||
from typing import Iterable
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Union
|
||||
|
||||
from .pathlib import resolve_from_str
|
||||
from .pathlib import rm_rf
|
||||
from .reports import CollectReport
|
||||
from _pytest import nodes
|
||||
from _pytest._io import TerminalWriter
|
||||
from _pytest.compat import final
|
||||
from _pytest.config import Config
|
||||
from _pytest.config import ExitCode
|
||||
from _pytest.config import hookimpl
|
||||
@@ -28,11 +27,10 @@ from _pytest.deprecated import check_ispytest
|
||||
from _pytest.fixtures import fixture
|
||||
from _pytest.fixtures import FixtureRequest
|
||||
from _pytest.main import Session
|
||||
from _pytest.nodes import Directory
|
||||
from _pytest.nodes import File
|
||||
from _pytest.python import Package
|
||||
from _pytest.reports import TestReport
|
||||
|
||||
|
||||
README_CONTENT = """\
|
||||
# pytest cache directory #
|
||||
|
||||
@@ -74,7 +72,7 @@ class Cache:
|
||||
self._config = config
|
||||
|
||||
@classmethod
|
||||
def for_config(cls, config: Config, *, _ispytest: bool = False) -> Cache:
|
||||
def for_config(cls, config: Config, *, _ispytest: bool = False) -> "Cache":
|
||||
"""Create the Cache instance for a Config.
|
||||
|
||||
:meta private:
|
||||
@@ -113,7 +111,6 @@ class Cache:
|
||||
"""
|
||||
check_ispytest(_ispytest)
|
||||
import warnings
|
||||
|
||||
from _pytest.warning_types import PytestCacheWarning
|
||||
|
||||
warnings.warn(
|
||||
@@ -122,10 +119,6 @@ class Cache:
|
||||
stacklevel=3,
|
||||
)
|
||||
|
||||
def _mkdir(self, path: Path) -> None:
|
||||
self._ensure_cache_dir_and_supporting_files()
|
||||
path.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
def mkdir(self, name: str) -> Path:
|
||||
"""Return a directory path object with the given name.
|
||||
|
||||
@@ -144,7 +137,7 @@ class Cache:
|
||||
if len(path.parts) > 1:
|
||||
raise ValueError("name is not allowed to contain path separators")
|
||||
res = self._cachedir.joinpath(self._CACHE_PREFIX_DIRS, path)
|
||||
self._mkdir(res)
|
||||
res.mkdir(exist_ok=True, parents=True)
|
||||
return res
|
||||
|
||||
def _getvaluepath(self, key: str) -> Path:
|
||||
@@ -181,13 +174,19 @@ class Cache:
|
||||
"""
|
||||
path = self._getvaluepath(key)
|
||||
try:
|
||||
self._mkdir(path.parent)
|
||||
if path.parent.is_dir():
|
||||
cache_dir_exists_already = True
|
||||
else:
|
||||
cache_dir_exists_already = self._cachedir.exists()
|
||||
path.parent.mkdir(exist_ok=True, parents=True)
|
||||
except OSError as exc:
|
||||
self.warn(
|
||||
f"could not create cache path {path}: {exc}",
|
||||
_ispytest=True,
|
||||
)
|
||||
return
|
||||
if not cache_dir_exists_already:
|
||||
self._ensure_supporting_files()
|
||||
data = json.dumps(value, ensure_ascii=False, indent=2)
|
||||
try:
|
||||
f = path.open("w", encoding="UTF-8")
|
||||
@@ -200,85 +199,60 @@ class Cache:
|
||||
with f:
|
||||
f.write(data)
|
||||
|
||||
def _ensure_cache_dir_and_supporting_files(self) -> None:
|
||||
"""Create the cache dir and its supporting files."""
|
||||
if self._cachedir.is_dir():
|
||||
return
|
||||
def _ensure_supporting_files(self) -> None:
|
||||
"""Create supporting files in the cache dir that are not really part of the cache."""
|
||||
readme_path = self._cachedir / "README.md"
|
||||
readme_path.write_text(README_CONTENT, encoding="UTF-8")
|
||||
|
||||
self._cachedir.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="pytest-cache-files-",
|
||||
dir=self._cachedir.parent,
|
||||
) as newpath:
|
||||
path = Path(newpath)
|
||||
gitignore_path = self._cachedir.joinpath(".gitignore")
|
||||
msg = "# Created by pytest automatically.\n*\n"
|
||||
gitignore_path.write_text(msg, encoding="UTF-8")
|
||||
|
||||
# Reset permissions to the default, see #12308.
|
||||
# Note: there's no way to get the current umask atomically, eek.
|
||||
umask = os.umask(0o022)
|
||||
os.umask(umask)
|
||||
path.chmod(0o777 - umask)
|
||||
|
||||
with open(path.joinpath("README.md"), "x", encoding="UTF-8") as f:
|
||||
f.write(README_CONTENT)
|
||||
with open(path.joinpath(".gitignore"), "x", encoding="UTF-8") as f:
|
||||
f.write("# Created by pytest automatically.\n*\n")
|
||||
with open(path.joinpath("CACHEDIR.TAG"), "xb") as f:
|
||||
f.write(CACHEDIR_TAG_CONTENT)
|
||||
|
||||
try:
|
||||
path.rename(self._cachedir)
|
||||
except OSError as e:
|
||||
# If 2 concurrent pytests both race to the rename, the loser
|
||||
# gets "Directory not empty" from the rename. In this case,
|
||||
# everything is handled so just continue (while letting the
|
||||
# temporary directory be cleaned up).
|
||||
# On Windows, the error is a FileExistsError which translates to EEXIST.
|
||||
if e.errno not in (errno.ENOTEMPTY, errno.EEXIST):
|
||||
raise
|
||||
else:
|
||||
# Create a directory in place of the one we just moved so that
|
||||
# `TemporaryDirectory`'s cleanup doesn't complain.
|
||||
#
|
||||
# TODO: pass ignore_cleanup_errors=True when we no longer support python < 3.10.
|
||||
# See https://github.com/python/cpython/issues/74168. Note that passing
|
||||
# delete=False would do the wrong thing in case of errors and isn't supported
|
||||
# until python 3.12.
|
||||
path.mkdir()
|
||||
cachedir_tag_path = self._cachedir.joinpath("CACHEDIR.TAG")
|
||||
cachedir_tag_path.write_bytes(CACHEDIR_TAG_CONTENT)
|
||||
|
||||
|
||||
class LFPluginCollWrapper:
|
||||
def __init__(self, lfplugin: LFPlugin) -> None:
|
||||
def __init__(self, lfplugin: "LFPlugin") -> None:
|
||||
self.lfplugin = lfplugin
|
||||
self._collected_at_least_one_failure = False
|
||||
|
||||
@hookimpl(wrapper=True)
|
||||
def pytest_make_collect_report(
|
||||
self, collector: nodes.Collector
|
||||
) -> Generator[None, CollectReport, CollectReport]:
|
||||
res = yield
|
||||
if isinstance(collector, Session | Directory):
|
||||
@hookimpl(hookwrapper=True)
|
||||
def pytest_make_collect_report(self, collector: nodes.Collector):
|
||||
if isinstance(collector, (Session, Package)):
|
||||
out = yield
|
||||
res: CollectReport = out.get_result()
|
||||
|
||||
# Sort any lf-paths to the beginning.
|
||||
lf_paths = self.lfplugin._last_failed_paths
|
||||
|
||||
# Use stable sort to prioritize last failed.
|
||||
def sort_key(node: nodes.Item | nodes.Collector) -> bool:
|
||||
return node.path in lf_paths
|
||||
# Use stable sort to priorize last failed.
|
||||
def sort_key(node: Union[nodes.Item, nodes.Collector]) -> bool:
|
||||
# Package.path is the __init__.py file, we need the directory.
|
||||
if isinstance(node, Package):
|
||||
path = node.path.parent
|
||||
else:
|
||||
path = node.path
|
||||
return path in lf_paths
|
||||
|
||||
res.result = sorted(
|
||||
res.result,
|
||||
key=sort_key,
|
||||
reverse=True,
|
||||
)
|
||||
return
|
||||
|
||||
elif isinstance(collector, File):
|
||||
if collector.path in self.lfplugin._last_failed_paths:
|
||||
out = yield
|
||||
res = out.get_result()
|
||||
result = res.result
|
||||
lastfailed = self.lfplugin.lastfailed
|
||||
|
||||
# Only filter with known failures.
|
||||
if not self._collected_at_least_one_failure:
|
||||
if not any(x.nodeid in lastfailed for x in result):
|
||||
return res
|
||||
return
|
||||
self.lfplugin.config.pluginmanager.register(
|
||||
LFPluginCollSkipfiles(self.lfplugin), "lfplugin-collskip"
|
||||
)
|
||||
@@ -294,19 +268,21 @@ class LFPluginCollWrapper:
|
||||
# Keep all sub-collectors.
|
||||
or isinstance(x, nodes.Collector)
|
||||
]
|
||||
|
||||
return res
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
class LFPluginCollSkipfiles:
|
||||
def __init__(self, lfplugin: LFPlugin) -> None:
|
||||
def __init__(self, lfplugin: "LFPlugin") -> None:
|
||||
self.lfplugin = lfplugin
|
||||
|
||||
@hookimpl
|
||||
def pytest_make_collect_report(
|
||||
self, collector: nodes.Collector
|
||||
) -> CollectReport | None:
|
||||
if isinstance(collector, File):
|
||||
) -> Optional[CollectReport]:
|
||||
# Packages are Files, but we only want to skip test-bearing Files,
|
||||
# so don't filter Packages.
|
||||
if isinstance(collector, File) and not isinstance(collector, Package):
|
||||
if collector.path not in self.lfplugin._last_failed_paths:
|
||||
self.lfplugin._skipped_files += 1
|
||||
|
||||
@@ -324,9 +300,9 @@ class LFPlugin:
|
||||
active_keys = "lf", "failedfirst"
|
||||
self.active = any(config.getoption(key) for key in active_keys)
|
||||
assert config.cache
|
||||
self.lastfailed: dict[str, bool] = config.cache.get("cache/lastfailed", {})
|
||||
self._previously_failed_count: int | None = None
|
||||
self._report_status: str | None = None
|
||||
self.lastfailed: Dict[str, bool] = config.cache.get("cache/lastfailed", {})
|
||||
self._previously_failed_count: Optional[int] = None
|
||||
self._report_status: Optional[str] = None
|
||||
self._skipped_files = 0 # count skipped files during collection due to --lf
|
||||
|
||||
if config.getoption("lf"):
|
||||
@@ -335,7 +311,7 @@ class LFPlugin:
|
||||
LFPluginCollWrapper(self), "lfplugin-collwrapper"
|
||||
)
|
||||
|
||||
def get_last_failed_paths(self) -> set[Path]:
|
||||
def get_last_failed_paths(self) -> Set[Path]:
|
||||
"""Return a set with all Paths of the previously failed nodeids and
|
||||
their parents."""
|
||||
rootpath = self.config.rootpath
|
||||
@@ -346,9 +322,9 @@ class LFPlugin:
|
||||
result.update(path.parents)
|
||||
return {x for x in result if x.exists()}
|
||||
|
||||
def pytest_report_collectionfinish(self) -> str | None:
|
||||
if self.active and self.config.get_verbosity() >= 0:
|
||||
return f"run-last-failure: {self._report_status}"
|
||||
def pytest_report_collectionfinish(self) -> Optional[str]:
|
||||
if self.active and self.config.getoption("verbose") >= 0:
|
||||
return "run-last-failure: %s" % self._report_status
|
||||
return None
|
||||
|
||||
def pytest_runtest_logreport(self, report: TestReport) -> None:
|
||||
@@ -366,14 +342,14 @@ class LFPlugin:
|
||||
else:
|
||||
self.lastfailed[report.nodeid] = True
|
||||
|
||||
@hookimpl(wrapper=True, tryfirst=True)
|
||||
@hookimpl(hookwrapper=True, tryfirst=True)
|
||||
def pytest_collection_modifyitems(
|
||||
self, config: Config, items: list[nodes.Item]
|
||||
) -> Generator[None]:
|
||||
res = yield
|
||||
self, config: Config, items: List[nodes.Item]
|
||||
) -> Generator[None, None, None]:
|
||||
yield
|
||||
|
||||
if not self.active:
|
||||
return res
|
||||
return
|
||||
|
||||
if self.lastfailed:
|
||||
previously_failed = []
|
||||
@@ -388,8 +364,8 @@ class LFPlugin:
|
||||
if not previously_failed:
|
||||
# Running a subset of all tests with recorded failures
|
||||
# only outside of it.
|
||||
self._report_status = (
|
||||
f"{len(self.lastfailed)} known failures not in selected tests"
|
||||
self._report_status = "%d known failures not in selected tests" % (
|
||||
len(self.lastfailed),
|
||||
)
|
||||
else:
|
||||
if self.config.getoption("lf"):
|
||||
@@ -400,13 +376,15 @@ class LFPlugin:
|
||||
|
||||
noun = "failure" if self._previously_failed_count == 1 else "failures"
|
||||
suffix = " first" if self.config.getoption("failedfirst") else ""
|
||||
self._report_status = (
|
||||
f"rerun previous {self._previously_failed_count} {noun}{suffix}"
|
||||
self._report_status = "rerun previous {count} {noun}{suffix}".format(
|
||||
count=self._previously_failed_count, suffix=suffix, noun=noun
|
||||
)
|
||||
|
||||
if self._skipped_files > 0:
|
||||
files_noun = "file" if self._skipped_files == 1 else "files"
|
||||
self._report_status += f" (skipped {self._skipped_files} {files_noun})"
|
||||
self._report_status += " (skipped {files} {files_noun})".format(
|
||||
files=self._skipped_files, files_noun=files_noun
|
||||
)
|
||||
else:
|
||||
self._report_status = "no previously failed tests, "
|
||||
if self.config.getoption("last_failed_no_failures") == "none":
|
||||
@@ -416,8 +394,6 @@ class LFPlugin:
|
||||
else:
|
||||
self._report_status += "not deselecting items."
|
||||
|
||||
return res
|
||||
|
||||
def pytest_sessionfinish(self, session: Session) -> None:
|
||||
config = self.config
|
||||
if config.getoption("cacheshow") or hasattr(config, "workerinput"):
|
||||
@@ -438,13 +414,15 @@ class NFPlugin:
|
||||
assert config.cache is not None
|
||||
self.cached_nodeids = set(config.cache.get("cache/nodeids", []))
|
||||
|
||||
@hookimpl(wrapper=True, tryfirst=True)
|
||||
def pytest_collection_modifyitems(self, items: list[nodes.Item]) -> Generator[None]:
|
||||
res = yield
|
||||
@hookimpl(hookwrapper=True, tryfirst=True)
|
||||
def pytest_collection_modifyitems(
|
||||
self, items: List[nodes.Item]
|
||||
) -> Generator[None, None, None]:
|
||||
yield
|
||||
|
||||
if self.active:
|
||||
new_items: dict[str, nodes.Item] = {}
|
||||
other_items: dict[str, nodes.Item] = {}
|
||||
new_items: Dict[str, nodes.Item] = {}
|
||||
other_items: Dict[str, nodes.Item] = {}
|
||||
for item in items:
|
||||
if item.nodeid not in self.cached_nodeids:
|
||||
new_items[item.nodeid] = item
|
||||
@@ -458,10 +436,8 @@ class NFPlugin:
|
||||
else:
|
||||
self.cached_nodeids.update(item.nodeid for item in items)
|
||||
|
||||
return res
|
||||
|
||||
def _get_increasing_order(self, items: Iterable[nodes.Item]) -> list[nodes.Item]:
|
||||
return sorted(items, key=lambda item: item.path.stat().st_mtime, reverse=True)
|
||||
def _get_increasing_order(self, items: Iterable[nodes.Item]) -> List[nodes.Item]:
|
||||
return sorted(items, key=lambda item: item.path.stat().st_mtime, reverse=True) # type: ignore[no-any-return]
|
||||
|
||||
def pytest_sessionfinish(self) -> None:
|
||||
config = self.config
|
||||
@@ -476,17 +452,14 @@ class NFPlugin:
|
||||
|
||||
|
||||
def pytest_addoption(parser: Parser) -> None:
|
||||
"""Add command-line options for cache functionality.
|
||||
|
||||
:param parser: Parser object to add command-line options to.
|
||||
"""
|
||||
group = parser.getgroup("general")
|
||||
group.addoption(
|
||||
"--lf",
|
||||
"--last-failed",
|
||||
action="store_true",
|
||||
dest="lf",
|
||||
help="Rerun only the tests that failed at the last run (or all if none failed)",
|
||||
help="Rerun only the tests that failed "
|
||||
"at the last run (or all if none failed)",
|
||||
)
|
||||
group.addoption(
|
||||
"--ff",
|
||||
@@ -540,7 +513,7 @@ def pytest_addoption(parser: Parser) -> None:
|
||||
)
|
||||
|
||||
|
||||
def pytest_cmdline_main(config: Config) -> int | ExitCode | None:
|
||||
def pytest_cmdline_main(config: Config) -> Optional[Union[int, ExitCode]]:
|
||||
if config.option.cacheshow and not config.option.help:
|
||||
from _pytest.main import wrap_session
|
||||
|
||||
@@ -550,13 +523,6 @@ def pytest_cmdline_main(config: Config) -> int | ExitCode | None:
|
||||
|
||||
@hookimpl(tryfirst=True)
|
||||
def pytest_configure(config: Config) -> None:
|
||||
"""Configure cache system and register related plugins.
|
||||
|
||||
Creates the Cache instance and registers the last-failed (LFPlugin)
|
||||
and new-first (NFPlugin) plugins with the plugin manager.
|
||||
|
||||
:param config: pytest configuration object.
|
||||
"""
|
||||
config.cache = Cache.for_config(config, _ispytest=True)
|
||||
config.pluginmanager.register(LFPlugin(config), "lfplugin")
|
||||
config.pluginmanager.register(NFPlugin(config), "nfplugin")
|
||||
@@ -578,7 +544,7 @@ def cache(request: FixtureRequest) -> Cache:
|
||||
return request.config.cache
|
||||
|
||||
|
||||
def pytest_report_header(config: Config) -> str | None:
|
||||
def pytest_report_header(config: Config) -> Optional[str]:
|
||||
"""Display cachedir with --cache-show and if non-default."""
|
||||
if config.option.verbose > 0 or config.getini("cache_dir") != ".pytest_cache":
|
||||
assert config.cache is not None
|
||||
@@ -595,16 +561,6 @@ def pytest_report_header(config: Config) -> str | None:
|
||||
|
||||
|
||||
def cacheshow(config: Config, session: Session) -> int:
|
||||
"""Display cache contents when --cache-show is used.
|
||||
|
||||
Shows cached values and directories matching the specified glob pattern
|
||||
(default: '*'). Displays cache location, cached test results, and
|
||||
any cached directories created by plugins.
|
||||
|
||||
:param config: pytest configuration object.
|
||||
:param session: pytest session object.
|
||||
:returns: Exit code (0 for success).
|
||||
"""
|
||||
from pprint import pformat
|
||||
|
||||
assert config.cache is not None
|
||||
@@ -622,25 +578,25 @@ def cacheshow(config: Config, session: Session) -> int:
|
||||
dummy = object()
|
||||
basedir = config.cache._cachedir
|
||||
vdir = basedir / Cache._CACHE_PREFIX_VALUES
|
||||
tw.sep("-", f"cache values for {glob!r}")
|
||||
tw.sep("-", "cache values for %r" % glob)
|
||||
for valpath in sorted(x for x in vdir.rglob(glob) if x.is_file()):
|
||||
key = str(valpath.relative_to(vdir))
|
||||
val = config.cache.get(key, dummy)
|
||||
if val is dummy:
|
||||
tw.line(f"{key} contains unreadable content, will be ignored")
|
||||
tw.line("%s contains unreadable content, will be ignored" % key)
|
||||
else:
|
||||
tw.line(f"{key} contains:")
|
||||
tw.line("%s contains:" % key)
|
||||
for line in pformat(val).splitlines():
|
||||
tw.line(" " + line)
|
||||
|
||||
ddir = basedir / Cache._CACHE_PREFIX_DIRS
|
||||
if ddir.is_dir():
|
||||
contents = sorted(ddir.rglob(glob))
|
||||
tw.sep("-", f"cache directories for {glob!r}")
|
||||
tw.sep("-", "cache directories for %r" % glob)
|
||||
for p in contents:
|
||||
# if p.is_dir():
|
||||
# print("%s/" % p.relative_to(basedir))
|
||||
if p.is_file():
|
||||
key = str(p.relative_to(basedir))
|
||||
tw.line(f"{key} is a file of length {p.stat().st_size}")
|
||||
tw.line(f"{key} is a file of length {p.stat().st_size:d}")
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user