import os from collections import OrderedDict import re import sys from configparser import ConfigParser, NoOptionError from pathlib import PurePath from .errors import MalformedDependencyFileError from .regex import HASH_REGEX from .dependencies import DependencyFile, Dependency from packaging.requirements import Requirement as PackagingRequirement,\ InvalidRequirement from . import filetypes from packaging.specifiers import SpecifierSet from packaging.version import Version, InvalidVersion import json if sys.version_info >= (3, 11): import tomllib else: import tomli as tomllib # this is a backport from setuptools 26.1 def setuptools_parse_requirements_backport(strs): # pragma: no cover # Copyright (C) 2016 Jason R Coombs # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files # (the "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the # following conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """Yield ``Requirement`` objects for each specification in `strs` `strs` must be a string, or a (possibly-nested) iterable thereof. """ # create a steppable iterator, so we can handle \-continuations def yield_lines(strs): """Yield non-empty/non-comment lines of a string or sequence""" if isinstance(strs, str): for s in strs.splitlines(): s = s.strip() # skip blank lines/comments if s and not s.startswith('#'): yield s else: for ss in strs: for s in yield_lines(ss): yield s lines = iter(yield_lines(strs)) for line in lines: # Drop comments -- a hash without a space may be in a URL. if ' #' in line: line = line[:line.find(' #')] # If there is a line continuation, drop it, and append the next line. if line.endswith('\\'): line = line[:-2].strip() line += next(lines) yield PackagingRequirement(line) class RequirementsTXTLineParser: """ """ @classmethod def parse(cls, line): """ :param line: :return: """ try: # setuptools requires a space before the comment. # If this isn't the case, add it. if "\t#" in line: parsed, = setuptools_parse_requirements_backport( line.replace("\t#", "\t #")) else: parsed, = setuptools_parse_requirements_backport(line) except InvalidRequirement: return None dep = Dependency( name=parsed.name, specs=parsed.specifier, line=line, extras=parsed.extras, dependency_type=filetypes.requirements_txt ) return dep class Parser: """ """ def __init__(self, obj, resolve=False): """ :param obj: """ self.obj = obj self._lines = None self.resolve = resolve def iter_lines(self, lineno=0): """ :param lineno: :return: """ yield from self.lines[lineno:] @property def lines(self): """ :return: """ if self._lines is None: self._lines = self.obj.content.splitlines() return self._lines @property def is_marked_file(self): """ :return: """ for n, line in enumerate(self.iter_lines()): for marker in self.obj.file_marker: if marker in line: return True if n >= 2: break return False def is_marked_line(self, line): """ :param line: :return: """ for marker in self.obj.line_marker: if marker in line: return True return False @classmethod def parse_hashes(cls, line): """ :param line: :return: """ hashes = [] for match in re.finditer(HASH_REGEX, line): hashes.append(line[match.start():match.end()]) return re.sub(HASH_REGEX, "", line).strip(), hashes @classmethod def parse_index_server(cls, line): """ :param line: :return: """ groups = re.split(pattern=r"[=\s]+", string=line.strip(), maxsplit=100) if len(groups) >= 2: return groups[1] if groups[1].endswith("/") else groups[1] + "/" return None @classmethod def resolve_file(cls, file_path, line): """ :param file_path: :param line: :return: """ line = line.replace("-r ", "").replace("--requirement ", "") normalized_path = PurePath(file_path) if " #" in line: line = line.split("#")[0].strip() return str(normalized_path.parent.joinpath(line)) class RequirementsTXTParser(Parser): """ """ def parse(self): """ Parses a requirements.txt-like file """ index_server = None for num, line in enumerate(self.iter_lines()): line = line.rstrip() if not line: continue if line.startswith('#'): # comments are lines that start with # only continue if line.startswith('-i') or \ line.startswith('--index-url') or \ line.startswith('--extra-index-url'): # this file is using a private index server, try to parse it index_server = self.parse_index_server(line) continue elif self.obj.path and \ (line.startswith('-r') or line.startswith('--requirement')): req_file_path = self.resolve_file(self.obj.path, line) if self.resolve and os.path.exists(req_file_path): with open(req_file_path) as f: content = f.read() dep_file = DependencyFile( content=content, path=req_file_path, resolve=True ) dep_file.parse() self.obj.resolved_files.append(dep_file) else: self.obj.resolved_files.append(req_file_path) elif line.startswith('-f') or line.startswith('--find-links') or \ line.startswith('--no-index') or \ line.startswith('--allow-external') or \ line.startswith('--allow-unverified') or \ line.startswith('-Z') or \ line.startswith('--always-unzip'): continue elif self.is_marked_line(line): continue else: try: parseable_line = line # multiline requirements are not parseable if "\\" in line: parseable_line = line.replace("\\", "") for next_line in self.iter_lines(num + 1): parseable_line += next_line.strip().replace("\\", "") line += "\n" + next_line if "\\" in next_line: continue break # ignore multiline requirements if they are marked if self.is_marked_line(parseable_line): continue hashes = [] if "--hash" in parseable_line: parseable_line, hashes = Parser.parse_hashes( parseable_line) req = RequirementsTXTLineParser.parse(parseable_line) if req: req.hashes = hashes req.index_server = index_server # replace the requirements line with the 'real' line req.line = line self.obj.dependencies.append(req) except ValueError: continue class ToxINIParser(Parser): """ """ def parse(self): """ :return: """ parser = ConfigParser() parser.read_string(self.obj.content) for section in parser.sections(): try: content = parser.get(section=section, option="deps") for n, line in enumerate(content.splitlines()): if self.is_marked_line(line): continue if line: req = RequirementsTXTLineParser.parse(line) if req: req.dependency_type = self.obj.file_type self.obj.dependencies.append(req) except NoOptionError: pass class CondaYMLParser(Parser): """ """ def parse(self): """ :return: """ import yaml try: data = yaml.safe_load(self.obj.content) if data and 'dependencies' in data and \ isinstance(data['dependencies'], list): for dep in data['dependencies']: if isinstance(dep, dict) and 'pip' in dep: for n, line in enumerate(dep['pip']): if self.is_marked_line(line): continue req = RequirementsTXTLineParser.parse(line) if req: req.dependency_type = self.obj.file_type self.obj.dependencies.append(req) except yaml.YAMLError: pass class PipfileParser(Parser): def parse(self): """ Parse a Pipfile (as seen in pipenv) :return: """ try: data = tomllib.loads(self.obj.content) if data: for package_type in ['packages', 'dev-packages']: if package_type in data: for name, specs in data[package_type].items(): # skip on VCS dependencies if not isinstance(specs, str): continue if specs == '*': specs = '' self.obj.dependencies.append( Dependency( name=name, specs=SpecifierSet(specs), dependency_type=filetypes.pipfile, line=''.join([name, specs]), sections=[package_type] ) ) except (tomllib.TOMLDecodeError, IndexError): pass class PipfileLockParser(Parser): def parse(self): """ Parse a Pipfile.lock (as seen in pipenv) :return: """ try: data = json.loads(self.obj.content, object_pairs_hook=OrderedDict) if data: for package_type in ['default', 'develop']: if package_type in data: for name, meta in data[package_type].items(): # skip VCS dependencies if 'version' not in meta: continue specs = meta['version'] hashes = meta['hashes'] self.obj.dependencies.append( Dependency( name=name, specs=SpecifierSet(specs), dependency_type=filetypes.pipfile_lock, hashes=hashes, line=''.join([name, specs]), sections=[package_type] ) ) except ValueError as e: raise MalformedDependencyFileError(info=str(e)) class SetupCfgParser(Parser): def parse(self): parser = ConfigParser() parser.read_string(self.obj.content) for section in parser.sections(): if section.name == 'options': options = 'install_requires', 'setup_requires', 'test_require' for name in options: if parser.has_option('options', name): content = section.get('options', name) self._parse_content(content) elif section == 'options.extras_require': for _, content in parser.items('options.extras_require'): self._parse_content(content) def _parse_content(self, content): for n, line in enumerate(content.splitlines()): if self.is_marked_line(line): continue if line: req = RequirementsTXTLineParser.parse(line) if req: req.dependency_type = self.obj.file_type self.obj.dependencies.append(req) class PoetryLockParser(Parser): def parse(self): """ Parse a poetry.lock """ try: from poetry.packages.locker import Locker from pathlib import Path lock_path = Path(self.obj.path) repository = Locker(lock_path, {}).locked_repository() for pkg in repository.packages: self.obj.dependencies.append( Dependency( name=pkg.name, specs=SpecifierSet(f"=={pkg.version.text}"), dependency_type=filetypes.poetry_lock, line=pkg.to_dependency().to_pep_508(), sections=list(pkg.dependency_group_names()) ) ) except Exception: try: data = tomllib.loads(self.obj.content) pkg_key = 'package' if data: dependencies = data[pkg_key] for dep in dependencies: name = dep['name'] spec = "=={version}".format( version=Version(dep['version'])) sections = [dep['category']] if "category" in dep else [] self.obj.dependencies.append( Dependency( name=name, specs=SpecifierSet(spec), dependency_type=filetypes.poetry_lock, line=''.join([name, spec]), sections=sections ) ) except Exception as e: raise MalformedDependencyFileError(info=str(e)) class PyprojectTomlParser(Parser): def parse(self) -> None: """Parse a pyproject.toml file. Refer to https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html for configuration specification. """ try: cfg = tomllib.loads(self.obj.content) except (tomllib.TOMLDecodeError, IndexError) as e: raise MalformedDependencyFileError(info=str(e)) if not cfg or "project" not in cfg: return sections = { "dependencies": cfg["project"].get("dependencies", []), **cfg["project"].get("optional-dependencies", {}), } for section, lines in sections.items(): for line in lines: req = RequirementsTXTLineParser.parse(line) if req: req.dependency_type = self.obj.file_type req.section = section self.obj.dependencies.append(req) def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None, resolve=False): """ :param content: :param file_type: :param path: :param sha: :param marker: :param parser: :return: """ dep_file = DependencyFile( content=content, path=path, sha=sha, marker=marker, file_type=file_type, parser=parser, resolve=resolve ) return dep_file.parse()