updates
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,65 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (c) the purl authors
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Visit https://github.com/package-url/packageurl-python for support and
|
||||
# download.
|
||||
|
||||
import django_filters
|
||||
|
||||
|
||||
class PackageURLFilter(django_filters.CharFilter):
|
||||
"""
|
||||
Filter by an exact Package URL string.
|
||||
|
||||
The special "EMPTY" value allows retrieval of objects with an empty Package URL.
|
||||
|
||||
This filter depends on `for_package_url` and `empty_package_url`
|
||||
methods to be available on the Model Manager,
|
||||
see for example `PackageURLQuerySetMixin`.
|
||||
|
||||
When exact_match_only is True, the filter will match only exact Package URL strings.
|
||||
"""
|
||||
|
||||
is_empty = "EMPTY"
|
||||
exact_match_only = False
|
||||
help_text = (
|
||||
'Match Package URL. Use "EMPTY" as value to retrieve objects with empty Package URL.'
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.exact_match_only = kwargs.pop("exact_match_only", False)
|
||||
kwargs.setdefault("help_text", self.help_text)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def filter(self, qs, value):
|
||||
none_values = ([], (), {}, "", None)
|
||||
if value in none_values:
|
||||
return qs
|
||||
|
||||
if self.distinct:
|
||||
qs = qs.distinct()
|
||||
|
||||
if value == self.is_empty:
|
||||
return qs.empty_package_url()
|
||||
|
||||
return qs.for_package_url(value, exact_match=self.exact_match_only)
|
||||
@@ -0,0 +1,180 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (c) the purl authors
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Visit https://github.com/package-url/packageurl-python for support and
|
||||
# download.
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import models
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from packageurl import PackageURL
|
||||
from packageurl.contrib.django.utils import purl_to_lookups
|
||||
|
||||
PACKAGE_URL_FIELDS = ("type", "namespace", "name", "version", "qualifiers", "subpath")
|
||||
|
||||
|
||||
class PackageURLQuerySetMixin:
|
||||
"""
|
||||
Add Package URL filtering methods to a django.db.models.QuerySet.
|
||||
"""
|
||||
|
||||
def for_package_url(self, purl_str, encode=True, exact_match=False):
|
||||
"""
|
||||
Filter the QuerySet based on a Package URL (purl) string with an option for
|
||||
exact match filtering.
|
||||
|
||||
When `exact_match` is False (default), the method will match any purl with the
|
||||
same base fields as `purl_str` and allow variations in other fields.
|
||||
When `exact_match` is True, only the identical purl will be returned.
|
||||
"""
|
||||
lookups = purl_to_lookups(
|
||||
purl_str=purl_str, encode=encode, include_empty_fields=exact_match
|
||||
)
|
||||
if lookups:
|
||||
return self.filter(**lookups)
|
||||
return self.none()
|
||||
|
||||
def with_package_url(self):
|
||||
"""Return objects with Package URL defined."""
|
||||
return self.filter(~models.Q(type="") & ~models.Q(name=""))
|
||||
|
||||
def without_package_url(self):
|
||||
"""Return objects with empty Package URL."""
|
||||
return self.filter(models.Q(type="") | models.Q(name=""))
|
||||
|
||||
def empty_package_url(self):
|
||||
"""Return objects with empty Package URL. Alias of without_package_url."""
|
||||
return self.without_package_url()
|
||||
|
||||
def order_by_package_url(self):
|
||||
"""Order by Package URL fields."""
|
||||
return self.order_by(*PACKAGE_URL_FIELDS)
|
||||
|
||||
|
||||
class PackageURLQuerySet(PackageURLQuerySetMixin, models.QuerySet):
|
||||
pass
|
||||
|
||||
|
||||
class PackageURLMixin(models.Model):
|
||||
"""
|
||||
Abstract Model for Package URL "purl" fields support.
|
||||
"""
|
||||
|
||||
type = models.CharField(
|
||||
max_length=16,
|
||||
blank=True,
|
||||
help_text=_(
|
||||
"A short code to identify the type of this package. "
|
||||
"For example: gem for a Rubygem, docker for a container, "
|
||||
"pypi for a Python Wheel or Egg, maven for a Maven Jar, "
|
||||
"deb for a Debian package, etc."
|
||||
),
|
||||
)
|
||||
|
||||
namespace = models.CharField(
|
||||
max_length=255,
|
||||
blank=True,
|
||||
help_text=_(
|
||||
"Package name prefix, such as Maven groupid, Docker image owner, "
|
||||
"GitHub user or organization, etc."
|
||||
),
|
||||
)
|
||||
|
||||
name = models.CharField(
|
||||
max_length=100,
|
||||
blank=True,
|
||||
help_text=_("Name of the package."),
|
||||
)
|
||||
|
||||
version = models.CharField(
|
||||
max_length=100,
|
||||
blank=True,
|
||||
help_text=_("Version of the package."),
|
||||
)
|
||||
|
||||
qualifiers = models.CharField(
|
||||
max_length=1024,
|
||||
blank=True,
|
||||
help_text=_(
|
||||
"Extra qualifying data for a package such as the name of an OS, "
|
||||
"architecture, distro, etc."
|
||||
),
|
||||
)
|
||||
|
||||
subpath = models.CharField(
|
||||
max_length=200,
|
||||
blank=True,
|
||||
help_text=_("Extra subpath within a package, relative to the package root."),
|
||||
)
|
||||
|
||||
objects = PackageURLQuerySet.as_manager()
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
@property
|
||||
def package_url(self):
|
||||
"""
|
||||
Return the Package URL "purl" string.
|
||||
"""
|
||||
try:
|
||||
package_url = self.get_package_url()
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
return str(package_url)
|
||||
|
||||
def get_package_url(self):
|
||||
"""
|
||||
Get the PackageURL instance.
|
||||
"""
|
||||
return PackageURL(
|
||||
self.type,
|
||||
self.namespace,
|
||||
self.name,
|
||||
self.version,
|
||||
self.qualifiers,
|
||||
self.subpath,
|
||||
)
|
||||
|
||||
def set_package_url(self, package_url):
|
||||
"""
|
||||
Set each field values to the values of the provided `package_url` string
|
||||
or PackageURL object.
|
||||
Existing values are always overwritten, forcing the new value or an
|
||||
empty string on all the `package_url` fields since we do not want to
|
||||
keep any previous values.
|
||||
"""
|
||||
if not isinstance(package_url, PackageURL):
|
||||
package_url = PackageURL.from_string(package_url)
|
||||
|
||||
package_url_dict = package_url.to_dict(encode=True, empty="")
|
||||
for field_name, value in package_url_dict.items():
|
||||
model_field = self._meta.get_field(field_name)
|
||||
|
||||
if value and len(value) > model_field.max_length:
|
||||
message = _(f'Value too long for field "{field_name}".')
|
||||
raise ValidationError(message)
|
||||
|
||||
setattr(self, field_name, value)
|
||||
@@ -0,0 +1,64 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (c) the purl authors
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Visit https://github.com/package-url/packageurl-python for support and
|
||||
# download.
|
||||
|
||||
|
||||
from packageurl import PackageURL
|
||||
|
||||
|
||||
def purl_to_lookups(purl_str, encode=True, include_empty_fields=False):
|
||||
"""
|
||||
Return a lookups dictionary built from the provided `purl` (Package URL) string.
|
||||
These lookups can be used as QuerySet filters.
|
||||
If include_empty_fields is provided, the resulting dictionary will include fields
|
||||
with empty values. This is useful to get exact match.
|
||||
Note that empty values are always returned as empty strings as the model fields
|
||||
are defined with `blank=True` and `null=False`.
|
||||
"""
|
||||
if not purl_str.startswith("pkg:"):
|
||||
purl_str = "pkg:" + purl_str
|
||||
|
||||
try:
|
||||
package_url = PackageURL.from_string(purl_str)
|
||||
except ValueError:
|
||||
return # Not a valid PackageURL
|
||||
|
||||
package_url_dict = package_url.to_dict(encode=encode, empty="")
|
||||
if include_empty_fields:
|
||||
return package_url_dict
|
||||
else:
|
||||
return without_empty_values(package_url_dict)
|
||||
|
||||
|
||||
def without_empty_values(input_dict):
|
||||
"""
|
||||
Return a new dict not including empty value entries from `input_dict`.
|
||||
|
||||
`None`, empty string, empty list, and empty dict/set are cleaned.
|
||||
`0` and `False` values are kept.
|
||||
"""
|
||||
empty_values = ([], (), {}, "", None)
|
||||
|
||||
return {key: value for key, value in input_dict.items() if value not in empty_values}
|
||||
@@ -0,0 +1,722 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (c) the purl authors
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Visit https://github.com/package-url/packageurl-python for support and
|
||||
# download.
|
||||
|
||||
from packageurl import PackageURL
|
||||
from packageurl.contrib.route import NoRouteAvailable
|
||||
from packageurl.contrib.route import Router
|
||||
|
||||
DEFAULT_MAVEN_REPOSITORY = "https://repo.maven.apache.org/maven2"
|
||||
|
||||
|
||||
def get_repo_download_url_by_package_type(
|
||||
type, namespace, name, version, archive_extension="tar.gz"
|
||||
):
|
||||
"""
|
||||
Return the download URL for a hosted git repository given a package type
|
||||
or None.
|
||||
"""
|
||||
if archive_extension not in ("zip", "tar.gz"):
|
||||
raise ValueError("Only zip and tar.gz extensions are supported")
|
||||
|
||||
download_url_by_type = {
|
||||
"github": f"https://github.com/{namespace}/{name}/archive/{version}.{archive_extension}",
|
||||
"bitbucket": f"https://bitbucket.org/{namespace}/{name}/get/{version}.{archive_extension}",
|
||||
"gitlab": f"https://gitlab.com/{namespace}/{name}/-/archive/{version}/{name}-{version}.{archive_extension}",
|
||||
}
|
||||
return download_url_by_type.get(type)
|
||||
|
||||
|
||||
repo_router = Router()
|
||||
download_router = Router()
|
||||
|
||||
|
||||
def _get_url_from_router(router, purl):
|
||||
if purl:
|
||||
try:
|
||||
return router.process(purl)
|
||||
except NoRouteAvailable:
|
||||
return
|
||||
|
||||
|
||||
def get_repo_url(purl):
|
||||
"""
|
||||
Return a repository URL inferred from the `purl` string.
|
||||
"""
|
||||
return _get_url_from_router(repo_router, purl)
|
||||
|
||||
|
||||
def get_download_url(purl):
|
||||
"""
|
||||
Return a download URL inferred from the `purl` string.
|
||||
"""
|
||||
download_url = _get_url_from_router(download_router, purl)
|
||||
if download_url:
|
||||
return download_url
|
||||
|
||||
# Fallback on the `download_url` qualifier when available.
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
return purl_data.qualifiers.get("download_url", None)
|
||||
|
||||
|
||||
def get_inferred_urls(purl):
|
||||
"""
|
||||
Return all inferred URLs (repo, download) from the `purl` string.
|
||||
"""
|
||||
url_functions = (
|
||||
get_repo_url,
|
||||
get_download_url,
|
||||
)
|
||||
|
||||
inferred_urls = []
|
||||
for url_func in url_functions:
|
||||
url = url_func(purl)
|
||||
if url:
|
||||
inferred_urls.append(url)
|
||||
|
||||
return inferred_urls
|
||||
|
||||
|
||||
# Backward compatibility
|
||||
purl2url = get_repo_url
|
||||
get_url = get_repo_url
|
||||
|
||||
|
||||
@repo_router.route("pkg:cargo/.*")
|
||||
def build_cargo_repo_url(purl):
|
||||
"""
|
||||
Return a cargo repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://crates.io/crates/{name}/{version}"
|
||||
elif name:
|
||||
return f"https://crates.io/crates/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:bitbucket/.*")
|
||||
def build_bitbucket_repo_url(purl):
|
||||
"""
|
||||
Return a bitbucket repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
|
||||
if name and namespace:
|
||||
return f"https://bitbucket.org/{namespace}/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:github/.*")
|
||||
def build_github_repo_url(purl):
|
||||
"""
|
||||
Return a github repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
qualifiers = purl_data.qualifiers
|
||||
|
||||
if not (name and namespace):
|
||||
return
|
||||
|
||||
repo_url = f"https://github.com/{namespace}/{name}"
|
||||
|
||||
if version:
|
||||
version_prefix = qualifiers.get("version_prefix", "")
|
||||
repo_url = f"{repo_url}/tree/{version_prefix}{version}"
|
||||
|
||||
return repo_url
|
||||
|
||||
|
||||
@repo_router.route("pkg:gitlab/.*")
|
||||
def build_gitlab_repo_url(purl):
|
||||
"""
|
||||
Return a gitlab repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
|
||||
if name and namespace:
|
||||
return f"https://gitlab.com/{namespace}/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:(gem|rubygems)/.*")
|
||||
def build_rubygems_repo_url(purl):
|
||||
"""
|
||||
Return a rubygems repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://rubygems.org/gems/{name}/versions/{version}"
|
||||
elif name:
|
||||
return f"https://rubygems.org/gems/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:cran/.*")
|
||||
def build_cran_repo_url(purl):
|
||||
"""
|
||||
Return a cran repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
return f"https://cran.r-project.org/src/contrib/{name}_{version}.tar.gz"
|
||||
|
||||
|
||||
@repo_router.route("pkg:npm/.*")
|
||||
def build_npm_repo_url(purl):
|
||||
"""
|
||||
Return a npm repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
repo_url = "https://www.npmjs.com/package/"
|
||||
if namespace:
|
||||
repo_url += f"{namespace}/"
|
||||
|
||||
repo_url += f"{name}"
|
||||
|
||||
if version:
|
||||
repo_url += f"/v/{version}"
|
||||
|
||||
return repo_url
|
||||
|
||||
|
||||
@repo_router.route("pkg:pypi/.*")
|
||||
def build_pypi_repo_url(purl):
|
||||
"""
|
||||
Return a pypi repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = (purl_data.name or "").replace("_", "-")
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://pypi.org/project/{name}/{version}/"
|
||||
elif name:
|
||||
return f"https://pypi.org/project/{name}/"
|
||||
|
||||
|
||||
@repo_router.route("pkg:composer/.*")
|
||||
def build_composer_repo_url(purl):
|
||||
"""
|
||||
Return a composer repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
namespace = purl_data.namespace
|
||||
|
||||
if name and version:
|
||||
return f"https://packagist.org/packages/{namespace}/{name}#{version}"
|
||||
elif name:
|
||||
return f"https://packagist.org/packages/{namespace}/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:nuget/.*")
|
||||
def build_nuget_repo_url(purl):
|
||||
"""
|
||||
Return a nuget repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://www.nuget.org/packages/{name}/{version}"
|
||||
elif name:
|
||||
return f"https://www.nuget.org/packages/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:hackage/.*")
|
||||
def build_hackage_repo_url(purl):
|
||||
"""
|
||||
Return a hackage repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://hackage.haskell.org/package/{name}-{version}"
|
||||
elif name:
|
||||
return f"https://hackage.haskell.org/package/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:golang/.*")
|
||||
def build_golang_repo_url(purl):
|
||||
"""
|
||||
Return a golang repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://pkg.go.dev/{namespace}/{name}@{version}"
|
||||
elif name:
|
||||
return f"https://pkg.go.dev/{namespace}/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:cocoapods/.*")
|
||||
def build_cocoapods_repo_url(purl):
|
||||
"""
|
||||
Return a CocoaPods repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
name = purl_data.name
|
||||
return name and f"https://cocoapods.org/pods/{name}"
|
||||
|
||||
|
||||
@repo_router.route("pkg:maven/.*")
|
||||
def build_maven_repo_url(purl):
|
||||
"""
|
||||
Return a Maven repo URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
qualifiers = purl_data.qualifiers
|
||||
|
||||
base_url = qualifiers.get("repository_url", DEFAULT_MAVEN_REPOSITORY)
|
||||
|
||||
if namespace and name and version:
|
||||
namespace = namespace.replace(".", "/")
|
||||
return f"{base_url}/{namespace}/{name}/{version}"
|
||||
|
||||
|
||||
# Download URLs:
|
||||
|
||||
|
||||
@download_router.route("pkg:cargo/.*")
|
||||
def build_cargo_download_url(purl):
|
||||
"""
|
||||
Return a cargo download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://crates.io/api/v1/crates/{name}/{version}/download"
|
||||
|
||||
|
||||
@download_router.route("pkg:(gem|rubygems)/.*")
|
||||
def build_rubygems_download_url(purl):
|
||||
"""
|
||||
Return a rubygems download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://rubygems.org/downloads/{name}-{version}.gem"
|
||||
|
||||
|
||||
@download_router.route("pkg:npm/.*")
|
||||
def build_npm_download_url(purl):
|
||||
"""
|
||||
Return a npm download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
base_url = "https://registry.npmjs.org"
|
||||
|
||||
if namespace:
|
||||
base_url += f"/{namespace}"
|
||||
|
||||
if name and version:
|
||||
return f"{base_url}/{name}/-/{name}-{version}.tgz"
|
||||
|
||||
|
||||
@download_router.route("pkg:maven/.*")
|
||||
def build_maven_download_url(purl):
|
||||
"""
|
||||
Return a maven download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
qualifiers = purl_data.qualifiers
|
||||
|
||||
base_url = qualifiers.get("repository_url", DEFAULT_MAVEN_REPOSITORY)
|
||||
maven_type = qualifiers.get("type", "jar") # default to "jar"
|
||||
classifier = qualifiers.get("classifier")
|
||||
|
||||
if namespace and name and version:
|
||||
namespace = namespace.replace(".", "/")
|
||||
classifier = f"-{classifier}" if classifier else ""
|
||||
return f"{base_url}/{namespace}/{name}/{version}/{name}-{version}{classifier}.{maven_type}"
|
||||
|
||||
|
||||
@download_router.route("pkg:hackage/.*")
|
||||
def build_hackage_download_url(purl):
|
||||
"""
|
||||
Return a hackage download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://hackage.haskell.org/package/{name}-{version}/{name}-{version}.tar.gz"
|
||||
|
||||
|
||||
@download_router.route("pkg:nuget/.*")
|
||||
def build_nuget_download_url(purl):
|
||||
"""
|
||||
Return a nuget download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://www.nuget.org/api/v2/package/{name}/{version}"
|
||||
|
||||
|
||||
@download_router.route("pkg:gitlab/.*", "pkg:bitbucket/.*", "pkg:github/.*")
|
||||
def build_repo_download_url(purl):
|
||||
"""
|
||||
Return a gitlab download URL from the `purl` string.
|
||||
"""
|
||||
return get_repo_download_url(purl)
|
||||
|
||||
|
||||
@download_router.route("pkg:hex/.*")
|
||||
def build_hex_download_url(purl):
|
||||
"""
|
||||
Return a hex download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://repo.hex.pm/tarballs/{name}-{version}.tar"
|
||||
|
||||
|
||||
@download_router.route("pkg:golang/.*")
|
||||
def build_golang_download_url(purl):
|
||||
"""
|
||||
Return a golang download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if not name:
|
||||
return
|
||||
|
||||
# TODO: https://github.com/package-url/packageurl-python/issues/197
|
||||
if namespace:
|
||||
name = f"{namespace}/{name}"
|
||||
|
||||
ename = escape_golang_path(name)
|
||||
eversion = escape_golang_path(version)
|
||||
|
||||
if not eversion.startswith("v"):
|
||||
eversion = "v" + eversion
|
||||
|
||||
if name and version:
|
||||
return f"https://proxy.golang.org/{ename}/@v/{eversion}.zip"
|
||||
|
||||
|
||||
@download_router.route("pkg:pub/.*")
|
||||
def build_pub_download_url(purl):
|
||||
"""
|
||||
Return a pub download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"https://pub.dev/api/archives/{name}-{version}.tar.gz"
|
||||
|
||||
|
||||
@download_router.route("pkg:swift/.*")
|
||||
def build_swift_download_url(purl):
|
||||
"""
|
||||
Return a Swift Package download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
namespace = purl_data.namespace
|
||||
|
||||
if not (namespace or name or version):
|
||||
return
|
||||
|
||||
return f"https://{namespace}/{name}/archive/{version}.zip"
|
||||
|
||||
|
||||
@download_router.route("pkg:luarocks/.*")
|
||||
def build_luarocks_download_url(purl):
|
||||
"""
|
||||
Return a LuaRocks download URL from the `purl` string.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
qualifiers = purl_data.qualifiers or {}
|
||||
|
||||
repository_url = qualifiers.get("repository_url", "https://luarocks.org")
|
||||
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
|
||||
if name and version:
|
||||
return f"{repository_url}/{name}-{version}.src.rock"
|
||||
|
||||
|
||||
@download_router.route("pkg:conda/.*")
|
||||
def build_conda_download_url(purl):
|
||||
"""
|
||||
Resolve a Conda PURL to a real downloadable URL
|
||||
|
||||
Supported qualifiers:
|
||||
- channel: e.g., main, conda-forge (required for deterministic base)
|
||||
- subdir: e.g., linux-64, osx-arm64, win-64, noarch
|
||||
- build: exact build string (optional but recommended)
|
||||
- type: 'conda' or 'tar.bz2' (preference; fallback to whichever exists)
|
||||
"""
|
||||
p = PackageURL.from_string(purl)
|
||||
if not p.name or not p.version:
|
||||
return None
|
||||
|
||||
q = p.qualifiers or {}
|
||||
name = p.name
|
||||
version = p.version
|
||||
build = q.get("build")
|
||||
channel = q.get("channel") or "main"
|
||||
subdir = q.get("subdir") or "noarch"
|
||||
req_type = q.get("type")
|
||||
|
||||
def _conda_base_for_channel(channel: str) -> str:
|
||||
"""
|
||||
Map a conda channel to its base URL.
|
||||
- 'main' / 'defaults' -> repo.anaconda.com
|
||||
- any other channel -> conda.anaconda.org/<channel>
|
||||
"""
|
||||
ch = (channel or "").lower()
|
||||
if ch in ("main", "defaults"):
|
||||
return "https://repo.anaconda.com/pkgs/main"
|
||||
return f"https://conda.anaconda.org/{ch}"
|
||||
|
||||
base = _conda_base_for_channel(channel)
|
||||
|
||||
package_identifier = (
|
||||
f"{name}-{version}-{build}.{req_type}" if build else f"{name}-{version}.{req_type}"
|
||||
)
|
||||
|
||||
download_url = f"{base}/{subdir}/{package_identifier}"
|
||||
return download_url
|
||||
|
||||
|
||||
@download_router.route("pkg:alpm/.*")
|
||||
def build_alpm_download_url(purl_str):
|
||||
purl = PackageURL.from_string(purl_str)
|
||||
name = purl.name
|
||||
version = purl.version
|
||||
arch = purl.qualifiers.get("arch", "any")
|
||||
|
||||
if not name or not version:
|
||||
return None
|
||||
|
||||
first_letter = name[0]
|
||||
url = f"https://archive.archlinux.org/packages/{first_letter}/{name}/{name}-{version}-{arch}.pkg.tar.zst"
|
||||
return url
|
||||
|
||||
|
||||
def normalize_version(version: str) -> str:
|
||||
"""
|
||||
Remove the epoch (if any) from a Debian version.
|
||||
E.g., "1:2.4.47-2" becomes "2.4.47-2"
|
||||
"""
|
||||
if ":" in version:
|
||||
_, v = version.split(":", 1)
|
||||
return v
|
||||
return version
|
||||
|
||||
|
||||
@download_router.route("pkg:deb/.*")
|
||||
def build_deb_download_url(purl_str: str) -> str:
|
||||
"""
|
||||
Construct a download URL for a Debian or Ubuntu package PURL.
|
||||
Supports optional 'repository_url' in qualifiers.
|
||||
"""
|
||||
p = PackageURL.from_string(purl_str)
|
||||
|
||||
name = p.name
|
||||
version = p.version
|
||||
namespace = p.namespace
|
||||
qualifiers = p.qualifiers or {}
|
||||
arch = qualifiers.get("arch")
|
||||
repository_url = qualifiers.get("repository_url")
|
||||
|
||||
if not name or not version:
|
||||
raise ValueError("Both name and version must be present in deb purl")
|
||||
|
||||
if not arch:
|
||||
arch = "source"
|
||||
|
||||
if repository_url:
|
||||
base_url = repository_url.rstrip("/")
|
||||
else:
|
||||
if namespace == "debian":
|
||||
base_url = "https://deb.debian.org/debian"
|
||||
elif namespace == "ubuntu":
|
||||
base_url = "http://archive.ubuntu.com/ubuntu"
|
||||
else:
|
||||
raise NotImplementedError(f"Unsupported distro namespace: {namespace}")
|
||||
|
||||
norm_version = normalize_version(version)
|
||||
|
||||
if arch == "source":
|
||||
filename = f"{name}_{norm_version}.dsc"
|
||||
else:
|
||||
filename = f"{name}_{norm_version}_{arch}.deb"
|
||||
|
||||
pool_path = f"/pool/main/{name[0].lower()}/{name}"
|
||||
|
||||
return f"{base_url}{pool_path}/{filename}"
|
||||
|
||||
|
||||
@download_router.route("pkg:apk/.*")
|
||||
def build_apk_download_url(purl):
|
||||
"""
|
||||
Return a download URL for a fully qualified Alpine Linux package PURL.
|
||||
|
||||
Example:
|
||||
pkg:apk/acct@6.6.4-r0?arch=x86&alpine_version=v3.11&repo=main
|
||||
"""
|
||||
purl = PackageURL.from_string(purl)
|
||||
name = purl.name
|
||||
version = purl.version
|
||||
arch = purl.qualifiers.get("arch")
|
||||
repo = purl.qualifiers.get("repo")
|
||||
alpine_version = purl.qualifiers.get("alpine_version")
|
||||
|
||||
if not name or not version or not arch or not repo or not alpine_version:
|
||||
raise ValueError(
|
||||
"All qualifiers (arch, repo, alpine_version) and name/version must be present in apk purl"
|
||||
)
|
||||
|
||||
return (
|
||||
f"https://dl-cdn.alpinelinux.org/alpine/{alpine_version}/{repo}/{arch}/{name}-{version}.apk"
|
||||
)
|
||||
|
||||
|
||||
def get_repo_download_url(purl):
|
||||
"""
|
||||
Return ``download_url`` if present in ``purl`` qualifiers or
|
||||
if ``namespace``, ``name`` and ``version`` are present in ``purl``
|
||||
else return None.
|
||||
"""
|
||||
purl_data = PackageURL.from_string(purl)
|
||||
|
||||
namespace = purl_data.namespace
|
||||
type = purl_data.type
|
||||
name = purl_data.name
|
||||
version = purl_data.version
|
||||
qualifiers = purl_data.qualifiers
|
||||
|
||||
download_url = qualifiers.get("download_url")
|
||||
if download_url:
|
||||
return download_url
|
||||
|
||||
if not (namespace and name and version):
|
||||
return
|
||||
|
||||
version_prefix = qualifiers.get("version_prefix", "")
|
||||
version = f"{version_prefix}{version}"
|
||||
|
||||
return get_repo_download_url_by_package_type(
|
||||
type=type, namespace=namespace, name=name, version=version
|
||||
)
|
||||
|
||||
|
||||
# TODO: https://github.com/package-url/packageurl-python/issues/196
|
||||
def escape_golang_path(path: str) -> str:
|
||||
"""
|
||||
Return an case-encoded module path or version name.
|
||||
|
||||
This is done by replacing every uppercase letter with an exclamation mark followed by the
|
||||
corresponding lower-case letter, in order to avoid ambiguity when serving from case-insensitive
|
||||
file systems.
|
||||
|
||||
See https://golang.org/ref/mod#goproxy-protocol.
|
||||
"""
|
||||
escaped_path = ""
|
||||
for c in path:
|
||||
if c >= "A" and c <= "Z":
|
||||
# replace uppercase with !lowercase
|
||||
escaped_path += "!" + chr(ord(c) + ord("a") - ord("A"))
|
||||
else:
|
||||
escaped_path += c
|
||||
return escaped_path
|
||||
@@ -0,0 +1,224 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (c) the purl authors
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Visit https://github.com/package-url/packageurl-python for support and
|
||||
# download.
|
||||
|
||||
import inspect
|
||||
import re
|
||||
from functools import wraps
|
||||
|
||||
"""
|
||||
Given a URI regex (or some string), this module can route execution to a
|
||||
callable.
|
||||
|
||||
There are several routing implementations available in Rails, Django, Flask,
|
||||
Paste, etc. However, these all assume that the routed processing is to craft a
|
||||
response to an incoming external HTTP request.
|
||||
|
||||
Here we are instead doing the opposite: given a URI (and no request yet) we are
|
||||
routing the processing to emit a request externally (HTTP or other protocol)
|
||||
and handling its response.
|
||||
|
||||
Also we crawl a lot and not only HTTP: git, svn, ftp, rsync and more.
|
||||
This simple library support this kind of arbitrary URI routing.
|
||||
|
||||
This is inspired by Guido's http://www.artima.com/weblogs/viewpost.jsp?thread=101605
|
||||
and Django, Flask, Werkzeug and other url dispatch and routing design from web
|
||||
frameworks.
|
||||
https://github.com/douban/brownant has a similar approach, using
|
||||
Werkzeug with the limitation that it does not route based on URI scheme and is
|
||||
limited to HTTP.
|
||||
"""
|
||||
|
||||
|
||||
class Rule(object):
|
||||
"""
|
||||
A rule is a mapping between a pattern (typically a URI) and a callable
|
||||
(typically a function).
|
||||
The pattern is a regex string pattern and must match entirely a string
|
||||
(typically a URI) for the rule to be considered, i.e. for the endpoint to
|
||||
be resolved and eventually invoked for a given string (typically a URI).
|
||||
"""
|
||||
|
||||
def __init__(self, pattern, endpoint):
|
||||
# To ensure the pattern will match entirely, we wrap the pattern
|
||||
# with start of line ^ and end of line $.
|
||||
self.pattern = pattern.lstrip("^").rstrip("$")
|
||||
self.pattern_match = re.compile("^" + self.pattern + "$").match
|
||||
|
||||
# ensure the endpoint is callable
|
||||
assert callable(endpoint)
|
||||
# classes are not always callable, make an extra check
|
||||
if inspect.isclass(endpoint):
|
||||
obj = endpoint()
|
||||
assert callable(obj)
|
||||
|
||||
self.endpoint = endpoint
|
||||
|
||||
def __repr__(self):
|
||||
return f'Rule(r"""{self.pattern}""", {self.endpoint.__module__}.{self.endpoint.__name__})'
|
||||
|
||||
def match(self, string):
|
||||
"""
|
||||
Match a string with the rule pattern, return True is matching.
|
||||
"""
|
||||
return self.pattern_match(string)
|
||||
|
||||
|
||||
class RouteAlreadyDefined(TypeError):
|
||||
"""
|
||||
Raised when this route Rule already exists in the route map.
|
||||
"""
|
||||
|
||||
|
||||
class NoRouteAvailable(TypeError):
|
||||
"""
|
||||
Raised when there are no route available.
|
||||
"""
|
||||
|
||||
|
||||
class MultipleRoutesDefined(TypeError):
|
||||
"""
|
||||
Raised when there are more than one route possible.
|
||||
"""
|
||||
|
||||
|
||||
class Router(object):
|
||||
"""
|
||||
A router is:
|
||||
- a container for a route map, consisting of several rules, stored in an
|
||||
ordered dictionary keyed by pattern text
|
||||
- a way to process a route, i.e. given a string (typically a URI), find the
|
||||
correct rule and invoke its callable endpoint
|
||||
- and a convenience decorator for routed callables (either a function or
|
||||
something with a __call__ method)
|
||||
|
||||
Multiple routers can co-exist as needed, such as a router to collect,
|
||||
another to fetch, etc.
|
||||
"""
|
||||
|
||||
def __init__(self, route_map=None):
|
||||
"""
|
||||
'route_map' is an ordered mapping of pattern -> Rule.
|
||||
"""
|
||||
self.route_map = route_map or dict()
|
||||
# lazy cached pre-compiled regex match() for all route patterns
|
||||
self._is_routable = None
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.route_map)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.route_map.items())
|
||||
|
||||
def keys(self):
|
||||
return self.route_map.keys()
|
||||
|
||||
def append(self, pattern, endpoint):
|
||||
"""
|
||||
Append a new pattern and endpoint Rule at the end of the map.
|
||||
Use this as an alternative to the route decorator.
|
||||
"""
|
||||
if pattern in self.route_map:
|
||||
raise RouteAlreadyDefined(pattern)
|
||||
self.route_map[pattern] = Rule(pattern, endpoint)
|
||||
|
||||
def route(self, *patterns):
|
||||
"""
|
||||
Decorator to make a callable 'endpoint' routed to one or more patterns.
|
||||
|
||||
Example:
|
||||
>>> my_router = Router()
|
||||
>>> @my_router.route('http://nexb.com', 'http://deja.com')
|
||||
... def somefunc(uri):
|
||||
... pass
|
||||
"""
|
||||
|
||||
def decorator(endpoint):
|
||||
assert patterns
|
||||
for pat in patterns:
|
||||
self.append(pat, endpoint)
|
||||
|
||||
@wraps(endpoint)
|
||||
def decorated(*args, **kwargs):
|
||||
return self.process(*args, **kwargs)
|
||||
|
||||
return decorated
|
||||
|
||||
return decorator
|
||||
|
||||
def process(self, string, *args, **kwargs):
|
||||
"""
|
||||
Given a string (typically a URI), resolve this string to an endpoint
|
||||
by searching available rules then execute the endpoint callable for
|
||||
that string passing down all arguments to the endpoint invocation.
|
||||
"""
|
||||
endpoint = self.resolve(string)
|
||||
if inspect.isclass(endpoint):
|
||||
# instantiate a class, that must define a __call__ method
|
||||
# TODO: consider passing args to the constructor?
|
||||
endpoint = endpoint()
|
||||
# call the callable
|
||||
return endpoint(string, *args, **kwargs)
|
||||
|
||||
def resolve(self, string):
|
||||
"""
|
||||
Resolve a string: given a string (typically a URI) resolve and
|
||||
return the best endpoint function for that string.
|
||||
|
||||
Ambiguous resolution is not allowed in order to keep things in
|
||||
check when there are hundreds rules: if multiple routes are
|
||||
possible for a string (typically a URI), a MultipleRoutesDefined
|
||||
TypeError is raised.
|
||||
"""
|
||||
# TODO: we could improve the performance of this by using a single
|
||||
# regex and named groups if this ever becomes a bottleneck.
|
||||
candidates = [r for r in self.route_map.values() if r.match(string)]
|
||||
|
||||
if not candidates:
|
||||
raise NoRouteAvailable(string)
|
||||
|
||||
if len(candidates) > 1:
|
||||
# this can happen when multiple patterns match the same string
|
||||
# we raise an exception with enough debugging information
|
||||
pats = repr([r.pattern for r in candidates])
|
||||
msg = "%(string)r matches multiple patterns %(pats)r" % locals()
|
||||
raise MultipleRoutesDefined(msg)
|
||||
|
||||
return candidates[0].endpoint
|
||||
|
||||
def is_routable(self, string):
|
||||
"""
|
||||
Return True if `string` is routable by this router, e.g. if it
|
||||
matches any of the route patterns.
|
||||
"""
|
||||
if not string:
|
||||
return
|
||||
|
||||
if not self._is_routable:
|
||||
# build an alternation regex
|
||||
routables = "^(" + "|".join(pat for pat in self.route_map) + ")$"
|
||||
self._is_routable = re.compile(routables, re.UNICODE).match
|
||||
|
||||
return bool(self._is_routable(string))
|
||||
Binary file not shown.
@@ -0,0 +1,123 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (c) the purl authors
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Visit https://github.com/package-url/packageurl-python for support and
|
||||
# download.
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped
|
||||
from sqlalchemy.orm import declarative_mixin
|
||||
from sqlalchemy.orm import mapped_column
|
||||
|
||||
from packageurl import PackageURL
|
||||
|
||||
|
||||
@declarative_mixin
|
||||
class PackageURLMixin:
|
||||
"""
|
||||
SQLAlchemy declarative mixin class for Package URL "purl" fields support.
|
||||
"""
|
||||
|
||||
type: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
comment=(
|
||||
"A short code to identify the type of this package. "
|
||||
"For example: gem for a Rubygem, docker for a container, "
|
||||
"pypi for a Python Wheel or Egg, maven for a Maven Jar, "
|
||||
"deb for a Debian package, etc."
|
||||
),
|
||||
)
|
||||
namespace: Mapped[str] = mapped_column(
|
||||
String(255),
|
||||
nullable=True,
|
||||
comment=(
|
||||
"Package name prefix, such as Maven groupid, Docker image owner, "
|
||||
"GitHub user or organization, etc."
|
||||
),
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(100), nullable=False, comment="Name of the package.")
|
||||
version: Mapped[str] = mapped_column(
|
||||
String(100), nullable=True, comment="Version of the package."
|
||||
)
|
||||
qualifiers: Mapped[str] = mapped_column(
|
||||
String(1024),
|
||||
nullable=True,
|
||||
comment=(
|
||||
"Extra qualifying data for a package such as the name of an OS, "
|
||||
"architecture, distro, etc."
|
||||
),
|
||||
)
|
||||
subpath: Mapped[str] = mapped_column(
|
||||
String(200),
|
||||
nullable=True,
|
||||
comment="Extra subpath within a package, relative to the package root.",
|
||||
)
|
||||
|
||||
@property
|
||||
def package_url(self) -> str:
|
||||
"""
|
||||
Return the Package URL "purl" string.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
"""
|
||||
try:
|
||||
package_url = self.get_package_url()
|
||||
except ValueError:
|
||||
return ""
|
||||
return str(package_url)
|
||||
|
||||
def get_package_url(self) -> PackageURL:
|
||||
"""
|
||||
Get the PackageURL instance.
|
||||
|
||||
Returns
|
||||
-------
|
||||
PackageURL
|
||||
"""
|
||||
return PackageURL(
|
||||
self.type,
|
||||
self.namespace,
|
||||
self.name,
|
||||
self.version,
|
||||
self.qualifiers,
|
||||
self.subpath,
|
||||
)
|
||||
|
||||
def set_package_url(self, package_url: PackageURL) -> None:
|
||||
"""
|
||||
Set or update the PackageURL object attributes.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
package_url: PackageURL
|
||||
The PackageURL object to set get attributes from.
|
||||
"""
|
||||
if not isinstance(package_url, PackageURL):
|
||||
package_url = PackageURL.from_string(package_url)
|
||||
|
||||
package_url_dict = package_url.to_dict(encode=True, empty="")
|
||||
for key, value in package_url_dict.items():
|
||||
setattr(self, key, value)
|
||||
@@ -0,0 +1,774 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright (c) the purl authors
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# Visit https://github.com/package-url/packageurl-python for support and
|
||||
# download.
|
||||
|
||||
import os
|
||||
import re
|
||||
from urllib.parse import unquote_plus
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from packageurl import PackageURL
|
||||
from packageurl.contrib.route import NoRouteAvailable
|
||||
from packageurl.contrib.route import Router
|
||||
|
||||
"""
|
||||
This module helps build a PackageURL from an arbitrary URL.
|
||||
This uses the a routing mechanism available in the route.py module.
|
||||
|
||||
In order to make it easy to use, it contains all the conversion functions
|
||||
in this single Python script.
|
||||
"""
|
||||
|
||||
|
||||
purl_router = Router()
|
||||
|
||||
|
||||
def url2purl(url):
|
||||
"""
|
||||
Return a PackageURL inferred from the `url` string or None.
|
||||
"""
|
||||
if url:
|
||||
try:
|
||||
return purl_router.process(url)
|
||||
except NoRouteAvailable:
|
||||
# If `url` does not fit in one of the existing routes,
|
||||
# we attempt to create a generic PackageURL for `url`
|
||||
return build_generic_purl(url)
|
||||
|
||||
|
||||
get_purl = url2purl
|
||||
|
||||
|
||||
def purl_from_pattern(type_, pattern, url, qualifiers=None):
|
||||
url = unquote_plus(url)
|
||||
compiled_pattern = re.compile(pattern, re.VERBOSE)
|
||||
match = compiled_pattern.match(url)
|
||||
|
||||
if not match:
|
||||
return
|
||||
|
||||
purl_data = {
|
||||
field: value for field, value in match.groupdict().items() if field in PackageURL._fields
|
||||
}
|
||||
|
||||
qualifiers = qualifiers or {}
|
||||
# Include the `version_prefix` as a qualifier to infer valid URLs in purl2url
|
||||
version_prefix = match.groupdict().get("version_prefix")
|
||||
if version_prefix:
|
||||
qualifiers.update({"version_prefix": version_prefix})
|
||||
|
||||
if qualifiers:
|
||||
if "qualifiers" in purl_data:
|
||||
purl_data["qualifiers"].update(qualifiers)
|
||||
else:
|
||||
purl_data["qualifiers"] = qualifiers
|
||||
|
||||
return PackageURL(type_, **purl_data)
|
||||
|
||||
|
||||
def register_pattern(type_, pattern, router=purl_router):
|
||||
"""
|
||||
Register a pattern with its type.
|
||||
"""
|
||||
|
||||
def endpoint(url):
|
||||
return purl_from_pattern(type_, pattern, url)
|
||||
|
||||
router.append(pattern, endpoint)
|
||||
|
||||
|
||||
def get_path_segments(url):
|
||||
"""
|
||||
Return a list of path segments from a `url` string.
|
||||
"""
|
||||
path = unquote_plus(urlparse(url).path)
|
||||
segments = [seg for seg in path.split("/") if seg]
|
||||
return segments
|
||||
|
||||
|
||||
def build_generic_purl(uri):
|
||||
"""
|
||||
Return a PackageURL from `uri`, if `uri` is a parsable URL, or None
|
||||
|
||||
`uri` is assumed to be a download URL, e.g. https://example.com/example.tar.gz
|
||||
"""
|
||||
parsed_uri = urlparse(uri)
|
||||
if parsed_uri.scheme and parsed_uri.netloc and parsed_uri.path:
|
||||
# Get file name from `uri`
|
||||
uri_path_segments = get_path_segments(uri)
|
||||
if uri_path_segments:
|
||||
file_name = uri_path_segments[-1]
|
||||
return PackageURL(type="generic", name=file_name, qualifiers={"download_url": uri})
|
||||
|
||||
|
||||
@purl_router.route(
|
||||
"https?://registry.npmjs.*/.*",
|
||||
"https?://registry.yarnpkg.com/.*",
|
||||
"https?://(www\\.)?npmjs.*/package.*",
|
||||
"https?://(www\\.)?yarnpkg.com/package.*",
|
||||
)
|
||||
def build_npm_purl(uri):
|
||||
# npm URLs are difficult to disambiguate with regex
|
||||
if "/package/" in uri:
|
||||
return build_npm_web_purl(uri)
|
||||
elif "/-/" in uri:
|
||||
return build_npm_download_purl(uri)
|
||||
else:
|
||||
return build_npm_api_purl(uri)
|
||||
|
||||
|
||||
def build_npm_api_purl(uri):
|
||||
path = unquote_plus(urlparse(uri).path)
|
||||
segments = [seg for seg in path.split("/") if seg]
|
||||
|
||||
if len(segments) < 2:
|
||||
return
|
||||
|
||||
# /@esbuild/freebsd-arm64/0.21.5
|
||||
if len(segments) == 3:
|
||||
return PackageURL("npm", namespace=segments[0], name=segments[1], version=segments[2])
|
||||
|
||||
# /@invisionag/eslint-config-ivx
|
||||
if segments[0].startswith("@"):
|
||||
return PackageURL("npm", namespace=segments[0], name=segments[1])
|
||||
|
||||
# /angular/1.6.6
|
||||
return PackageURL("npm", name=segments[0], version=segments[1])
|
||||
|
||||
|
||||
def build_npm_download_purl(uri):
|
||||
path = unquote_plus(urlparse(uri).path)
|
||||
segments = [seg for seg in path.split("/") if seg and seg != "-"]
|
||||
len_segments = len(segments)
|
||||
|
||||
# /@invisionag/eslint-config-ivx/-/eslint-config-ivx-0.0.2.tgz
|
||||
if len_segments == 3:
|
||||
namespace, name, filename = segments
|
||||
|
||||
# /automatta/-/automatta-0.0.1.tgz
|
||||
elif len_segments == 2:
|
||||
namespace = None
|
||||
name, filename = segments
|
||||
|
||||
else:
|
||||
return
|
||||
|
||||
base_filename, ext = os.path.splitext(filename)
|
||||
version = base_filename.replace(name, "")
|
||||
if version.startswith("-"):
|
||||
version = version[1:] # Removes the "-" prefix
|
||||
|
||||
return PackageURL("npm", namespace, name, version)
|
||||
|
||||
|
||||
def build_npm_web_purl(uri):
|
||||
path = unquote_plus(urlparse(uri).path)
|
||||
if path.startswith("/package/"):
|
||||
path = path[9:]
|
||||
|
||||
segments = [seg for seg in path.split("/") if seg]
|
||||
len_segments = len(segments)
|
||||
namespace = version = None
|
||||
|
||||
# @angular/cli/v/10.1.2
|
||||
if len_segments == 4:
|
||||
namespace = segments[0]
|
||||
name = segments[1]
|
||||
version = segments[3]
|
||||
|
||||
# express/v/4.17.1
|
||||
elif len_segments == 3:
|
||||
namespace = None
|
||||
name = segments[0]
|
||||
version = segments[2]
|
||||
|
||||
# @angular/cli
|
||||
elif len_segments == 2:
|
||||
namespace = segments[0]
|
||||
name = segments[1]
|
||||
|
||||
# express
|
||||
elif len_segments == 1 and len(segments) > 0 and segments[0][0] != "@":
|
||||
name = segments[0]
|
||||
|
||||
else:
|
||||
return
|
||||
|
||||
return PackageURL("npm", namespace, name, version)
|
||||
|
||||
|
||||
@purl_router.route(
|
||||
"https?://repo1.maven.org/maven2/.*",
|
||||
"https?://central.maven.org/maven2/.*",
|
||||
"maven-index://repo1.maven.org/.*",
|
||||
)
|
||||
def build_maven_purl(uri):
|
||||
path = unquote_plus(urlparse(uri).path)
|
||||
segments = [seg for seg in path.split("/") if seg and seg != "maven2"]
|
||||
|
||||
if len(segments) < 3:
|
||||
return
|
||||
|
||||
before_last_segment, last_segment = segments[-2:]
|
||||
has_filename = before_last_segment in last_segment
|
||||
|
||||
filename = None
|
||||
if has_filename:
|
||||
filename = segments.pop()
|
||||
|
||||
version = segments[-1]
|
||||
name = segments[-2]
|
||||
namespace = ".".join(segments[:-2])
|
||||
qualifiers = {}
|
||||
|
||||
if filename:
|
||||
name_version = f"{name}-{version}"
|
||||
_, _, classifier_ext = filename.rpartition(name_version)
|
||||
classifier, _, extension = classifier_ext.partition(".")
|
||||
if not extension:
|
||||
return
|
||||
|
||||
qualifiers["classifier"] = classifier.strip("-")
|
||||
|
||||
valid_types = ("aar", "ear", "mar", "pom", "rar", "rpm", "sar", "tar.gz", "war", "zip")
|
||||
if extension in valid_types:
|
||||
qualifiers["type"] = extension
|
||||
|
||||
return PackageURL("maven", namespace, name, version, qualifiers)
|
||||
|
||||
|
||||
# https://rubygems.org/gems/i18n-js-3.0.11.gem
|
||||
@purl_router.route("https?://rubygems.org/(downloads|gems)/.*")
|
||||
def build_rubygems_purl(uri):
|
||||
# We use a more general route pattern instead of using `rubygems_pattern`
|
||||
# below by itself because we want to capture all rubygems download URLs,
|
||||
# even the ones that are not completely formed. This helps prevent url2purl
|
||||
# from attempting to create a generic PackageURL from an invalid rubygems
|
||||
# download URL.
|
||||
|
||||
# https://rubygems.org/downloads/jwt-0.1.8.gem
|
||||
# https://rubygems.org/gems/i18n-js-3.0.11.gem
|
||||
rubygems_pattern = (
|
||||
r"^https?://rubygems.org/(downloads|gems)/(?P<name>.+)-(?P<version>.+)(\.gem)$"
|
||||
)
|
||||
return purl_from_pattern("gem", rubygems_pattern, uri)
|
||||
|
||||
|
||||
# https://cran.r-project.org/src/contrib/jsonlite_1.8.8.tar.gz
|
||||
# https://packagemanager.rstudio.com/cran/2022-06-23/src/contrib/curl_4.3.2.tar.gz"
|
||||
@purl_router.route(
|
||||
"https?://cran.r-project.org/.*",
|
||||
"https?://packagemanager.rstudio.com/cran/.*",
|
||||
)
|
||||
def build_cran_purl(uri):
|
||||
cran_pattern = r"^https?://(cran\.r-project\.org|packagemanager\.rstudio\.com/cran)/.*?src/contrib/(?P<name>.+)_(?P<version>.+)\.tar.gz$"
|
||||
qualifiers = {}
|
||||
if "//cran.r-project.org/" not in uri:
|
||||
qualifiers["download_url"] = uri
|
||||
return purl_from_pattern("cran", cran_pattern, uri, qualifiers)
|
||||
|
||||
|
||||
# https://pypi.org/packages/source/a/anyjson/anyjson-0.3.3.tar.gz
|
||||
# https://pypi.python.org/packages/source/a/anyjson/anyjson-0.3.3.tar.gz
|
||||
# https://pypi.python.org/packages/2.6/t/threadpool/threadpool-1.2.7-py2.6.egg
|
||||
# https://pypi.python.org/packages/any/s/setuptools/setuptools-0.6c11-1.src.rpm
|
||||
# https://files.pythonhosted.org/packages/84/d8/451842a5496844bb5c7634b231a2e4caf0d867d2e25f09b840d3b07f3d4b/multi_key_dict-2.0.win32.exe
|
||||
pypi_pattern = r"(?P<name>(\w\.?)+(-\w+)*)-(?P<version>.+)\.(zip|tar.gz|tar.bz2|tgz|egg|rpm|exe)$"
|
||||
|
||||
# This pattern can be found in the following locations:
|
||||
# - wheel.wheelfile.WHEEL_INFO_RE
|
||||
# - distlib.wheel.FILENAME_RE
|
||||
# - setuptools.wheel.WHEEL_NAME
|
||||
# - pip._internal.wheel.Wheel.wheel_file_re
|
||||
wheel_file_re = re.compile(
|
||||
r"^(?P<namever>(?P<name>.+?)-(?P<version>.*?))"
|
||||
r"((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)"
|
||||
r"\.whl)$",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
@purl_router.route(
|
||||
"https?://pypi.org/(packages|project)/.+",
|
||||
"https?://.+python.+org/(packages|project)/.*",
|
||||
)
|
||||
def build_pypi_purl(uri):
|
||||
path = unquote_plus(urlparse(uri).path)
|
||||
segments = path.split("/")
|
||||
last_segment = segments[-1]
|
||||
|
||||
# /wheel-0.29.0-py2.py3-none-any.whl
|
||||
if last_segment.endswith(".whl"):
|
||||
match = wheel_file_re.match(last_segment)
|
||||
if match:
|
||||
return PackageURL(
|
||||
"pypi",
|
||||
name=match.group("name"),
|
||||
version=match.group("version"),
|
||||
)
|
||||
|
||||
if segments[1] == "project":
|
||||
return PackageURL(
|
||||
"pypi",
|
||||
name=segments[2],
|
||||
version=segments[3] if len(segments) > 3 else None,
|
||||
)
|
||||
|
||||
return purl_from_pattern("pypi", pypi_pattern, last_segment)
|
||||
|
||||
|
||||
# https://packagist.org/packages/webmozart/assert#1.9.1
|
||||
@purl_router.route("https?://packagist.org/packages/.*")
|
||||
def build_composer_purl(uri):
|
||||
# We use a more general route pattern instead of using `composer_pattern`
|
||||
# below by itself because we want to capture all packagist download URLs,
|
||||
# even the ones that are not completely formed. This helps prevent url2purl
|
||||
# from attempting to create a generic PackageURL from an invalid packagist
|
||||
# download URL.
|
||||
|
||||
# https://packagist.org/packages/ralouphie/getallheaders
|
||||
# https://packagist.org/packages/symfony/process#v7.0.0-BETA3
|
||||
composer_pattern = r"^https?://packagist\.org/packages/(?P<namespace>[^/]+)/(?P<name>[^\#]+?)(\#(?P<version>.+))?$"
|
||||
return purl_from_pattern("composer", composer_pattern, uri)
|
||||
|
||||
|
||||
# http://nuget.org/packages/EntityFramework/4.2.0.0
|
||||
# https://www.nuget.org/api/v2/package/Newtonsoft.Json/11.0.1
|
||||
nuget_www_pattern = r"^https?://.*nuget.org/(api/v2/)?packages?/(?P<name>.+)/(?P<version>.+)$"
|
||||
|
||||
register_pattern("nuget", nuget_www_pattern)
|
||||
|
||||
|
||||
# https://api.nuget.org/v3-flatcontainer/newtonsoft.json/10.0.1/newtonsoft.json.10.0.1.nupkg
|
||||
nuget_api_pattern = (
|
||||
r"^https?://api.nuget.org/v3-flatcontainer/"
|
||||
r"(?P<name>.+)/"
|
||||
r"(?P<version>.+)/"
|
||||
r".*(nupkg)$" # ends with "nupkg"
|
||||
)
|
||||
|
||||
register_pattern("nuget", nuget_api_pattern)
|
||||
|
||||
|
||||
# https://sourceforge.net/projects/turbovnc/files/3.1/turbovnc-3.1.tar.gz/download
|
||||
# https://sourceforge.net/projects/scribus/files/scribus/1.6.0/scribus-1.6.0.tar.gz/download
|
||||
# https://sourceforge.net/projects/ventoy/files/v1.0.96/Ventoy%201.0.96%20release%20source%20code.tar.gz/download
|
||||
# https://sourceforge.net/projects/geoserver/files/GeoServer/2.23.4/geoserver-2.23.4-war.zip/download
|
||||
sourceforge_download_pattern = (
|
||||
r"^https?://.*sourceforge.net/projects/"
|
||||
r"(?P<name>.+)/"
|
||||
r"files/"
|
||||
r"(?i:(?P=name)/)?" # optional case-insensitive name segment repeated
|
||||
r"v?(?P<version>[0-9\.]+)/" # version restricted to digits and dots
|
||||
r"(?i:(?P=name)).*(?P=version).*" # case-insensitive matching for {name}-{version}
|
||||
r"(/download)$" # ending with "/download"
|
||||
)
|
||||
|
||||
register_pattern("sourceforge", sourceforge_download_pattern)
|
||||
|
||||
|
||||
# https://sourceforge.net/projects/spacesniffer/files/spacesniffer_1_3_0_2.zip/download
|
||||
sourceforge_download_pattern_bis = (
|
||||
r"^https?://.*sourceforge.net/projects/"
|
||||
r"(?P<name>.+)/"
|
||||
r"files/"
|
||||
r"(?i:(?P=name))_*(?P<version>[0-9_]+).*"
|
||||
r"(/download)$" # ending with "/download"
|
||||
)
|
||||
|
||||
register_pattern("sourceforge", sourceforge_download_pattern_bis)
|
||||
|
||||
|
||||
@purl_router.route("https?://.*sourceforge.net/project/.*")
|
||||
def build_sourceforge_purl(uri):
|
||||
# We use a more general route pattern instead of using `sourceforge_pattern`
|
||||
# below by itself because we want to capture all sourceforge download URLs,
|
||||
# even the ones that do not fit `sourceforge_pattern`. This helps prevent
|
||||
# url2purl from attempting to create a generic PackageURL from a sourceforge
|
||||
# URL that we can't handle.
|
||||
|
||||
# http://master.dl.sourceforge.net/project/libpng/zlib/1.2.3/zlib-1.2.3.tar.bz2
|
||||
sourceforge_pattern = (
|
||||
r"^https?://.*sourceforge.net/projects?/"
|
||||
r"(?P<namespace>([^/]+))/" # do not allow more "/" segments
|
||||
r"(OldFiles/)?"
|
||||
r"(?P<name>.+)/"
|
||||
r"(?P<version>[v0-9\.]+)/" # version restricted to digits and dots
|
||||
r"(?P=name).*(?P=version).*" # {name}-{version} repeated in the filename
|
||||
r"[^/]$" # not ending with "/"
|
||||
)
|
||||
|
||||
sourceforge_purl = purl_from_pattern("sourceforge", sourceforge_pattern, uri)
|
||||
|
||||
if not sourceforge_purl:
|
||||
# Get the project name from `uri` and use that as the Package name
|
||||
# http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
|
||||
split_uri = uri.split("/project/")
|
||||
|
||||
# http://master.dl.sourceforge.net, aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
|
||||
if len(split_uri) >= 2:
|
||||
# aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
|
||||
remaining_uri_path = split_uri[1]
|
||||
# aloyscore, aloyscore, 0.1a1%2520stable, 0.1a1_stable_AloysCore.zip
|
||||
remaining_uri_path_segments = remaining_uri_path.split("/")
|
||||
if remaining_uri_path_segments:
|
||||
project_name = remaining_uri_path_segments[0] # aloyscore
|
||||
sourceforge_purl = PackageURL(
|
||||
type="sourceforge", name=project_name, qualifiers={"download_url": uri}
|
||||
)
|
||||
return sourceforge_purl
|
||||
|
||||
|
||||
# https://crates.io/api/v1/crates/rand/0.7.2/download
|
||||
cargo_pattern = r"^https?://crates.io/api/v1/crates/(?P<name>.+)/(?P<version>.+)(\/download)$"
|
||||
|
||||
register_pattern("cargo", cargo_pattern)
|
||||
|
||||
|
||||
# https://raw.githubusercontent.com/volatilityfoundation/dwarf2json/master/LICENSE.txt
|
||||
github_raw_content_pattern = (
|
||||
r"https?://raw.githubusercontent.com/(?P<namespace>[^/]+)/(?P<name>[^/]+)/"
|
||||
r"(?P<version>[^/]+)/(?P<subpath>.*)$"
|
||||
)
|
||||
|
||||
register_pattern("github", github_raw_content_pattern)
|
||||
|
||||
|
||||
@purl_router.route("https?://api.github\\.com/repos/.*")
|
||||
def build_github_api_purl(url):
|
||||
"""
|
||||
Return a PackageURL object from GitHub API `url`.
|
||||
For example:
|
||||
https://api.github.com/repos/nexB/scancode-toolkit/commits/40593af0df6c8378d2b180324b97cb439fa11d66
|
||||
https://api.github.com/repos/nexB/scancode-toolkit/
|
||||
and returns a `PackageURL` object
|
||||
"""
|
||||
segments = get_path_segments(url)
|
||||
|
||||
if not (len(segments) >= 3):
|
||||
return
|
||||
namespace = segments[1]
|
||||
name = segments[2]
|
||||
version = None
|
||||
|
||||
# https://api.github.com/repos/nexB/scancode-toolkit/
|
||||
if len(segments) == 4 and segments[3] != "commits":
|
||||
version = segments[3]
|
||||
|
||||
# https://api.github.com/repos/nexB/scancode-toolkit/commits/40593af0df6c8378d2b180324b97cb439fa11d66
|
||||
if len(segments) == 5 and segments[3] == "commits":
|
||||
version = segments[4]
|
||||
|
||||
return PackageURL(type="github", namespace=namespace, name=name, version=version)
|
||||
|
||||
|
||||
# https://codeload.github.com/nexB/scancode-toolkit/tar.gz/v3.1.1
|
||||
# https://codeload.github.com/berngp/grails-rest/zip/release/0.7
|
||||
github_codeload_pattern = (
|
||||
r"https?://codeload.github.com/(?P<namespace>.+)/(?P<name>.+)/"
|
||||
r"(zip|tar.gz|tar.bz2|tgz)/(.*/)*"
|
||||
r"(?P<version>.+)$"
|
||||
)
|
||||
|
||||
register_pattern("github", github_codeload_pattern)
|
||||
|
||||
|
||||
@purl_router.route("https?://github\\.com/.*")
|
||||
def build_github_purl(url):
|
||||
"""
|
||||
Return a PackageURL object from GitHub `url`.
|
||||
"""
|
||||
|
||||
# https://github.com/apache/nifi/archive/refs/tags/rel/nifi-2.0.0-M3.tar.gz
|
||||
archive_tags_pattern = (
|
||||
r"https?://github.com/(?P<namespace>.+)/(?P<name>.+)"
|
||||
r"/archive/refs/tags/"
|
||||
r"(?P<version>.+).(zip|tar.gz|tar.bz2|.tgz)"
|
||||
)
|
||||
|
||||
# https://github.com/nexB/scancode-toolkit/archive/v3.1.1.zip
|
||||
archive_pattern = (
|
||||
r"https?://github.com/(?P<namespace>.+)/(?P<name>.+)"
|
||||
r"/archive/(.*/)*"
|
||||
r"((?P=name)(-|_|@))?"
|
||||
r"(?P<version>.+).(zip|tar.gz|tar.bz2|.tgz)"
|
||||
)
|
||||
|
||||
# https://github.com/downloads/mozilla/rhino/rhino1_7R4.zip
|
||||
download_pattern = (
|
||||
r"https?://github.com/downloads/(?P<namespace>.+)/(?P<name>.+)/"
|
||||
r"((?P=name)(-|@)?)?"
|
||||
r"(?P<version>.+).(zip|tar.gz|tar.bz2|.tgz)"
|
||||
)
|
||||
|
||||
# https://github.com/pypa/get-virtualenv/raw/20.0.31/public/virtualenv.pyz
|
||||
raw_pattern = (
|
||||
r"https?://github.com/(?P<namespace>.+)/(?P<name>.+)"
|
||||
r"/raw/(?P<version>[^/]+)/(?P<subpath>.*)$"
|
||||
)
|
||||
|
||||
# https://github.com/fanf2/unifdef/blob/master/unifdef.c
|
||||
blob_pattern = (
|
||||
r"https?://github.com/(?P<namespace>.+)/(?P<name>.+)"
|
||||
r"/blob/(?P<version>[^/]+)/(?P<subpath>.*)$"
|
||||
)
|
||||
|
||||
releases_download_pattern = (
|
||||
r"https?://github.com/(?P<namespace>.+)/(?P<name>.+)"
|
||||
r"/releases/download/(?P<version>[^/]+)/.*$"
|
||||
)
|
||||
|
||||
# https://github.com/pombredanne/schematics.git
|
||||
git_pattern = r"https?://github.com/(?P<namespace>.+)/(?P<name>.+).(git)"
|
||||
|
||||
# https://github.com/<namespace>/<name>/commit/<sha>
|
||||
commit_pattern = (
|
||||
r"https?://github.com/"
|
||||
r"(?P<namespace>[^/]+)/(?P<name>[^/]+)/commit/(?P<version>[0-9a-fA-F]{7,40})/?$"
|
||||
)
|
||||
|
||||
patterns = (
|
||||
commit_pattern,
|
||||
archive_tags_pattern,
|
||||
archive_pattern,
|
||||
raw_pattern,
|
||||
blob_pattern,
|
||||
releases_download_pattern,
|
||||
download_pattern,
|
||||
git_pattern,
|
||||
)
|
||||
|
||||
for pattern in patterns:
|
||||
matches = re.search(pattern, url)
|
||||
qualifiers = {}
|
||||
if matches:
|
||||
if pattern == releases_download_pattern:
|
||||
qualifiers["download_url"] = url
|
||||
return purl_from_pattern(
|
||||
type_="github", pattern=pattern, url=url, qualifiers=qualifiers
|
||||
)
|
||||
|
||||
segments = get_path_segments(url)
|
||||
if not len(segments) >= 2:
|
||||
return
|
||||
|
||||
namespace = segments[0]
|
||||
name = segments[1]
|
||||
version = None
|
||||
subpath = None
|
||||
|
||||
# https://github.com/TG1999/fetchcode/master
|
||||
if len(segments) >= 3 and segments[2] != "tree":
|
||||
version = segments[2]
|
||||
subpath = "/".join(segments[3:])
|
||||
|
||||
# https://github.com/TG1999/fetchcode/tree/master
|
||||
if len(segments) >= 4 and segments[2] == "tree":
|
||||
version = segments[3]
|
||||
subpath = "/".join(segments[4:])
|
||||
|
||||
return PackageURL(
|
||||
type="github",
|
||||
namespace=namespace,
|
||||
name=name,
|
||||
version=version,
|
||||
subpath=subpath,
|
||||
)
|
||||
|
||||
|
||||
# https://bitbucket.org/<namespace>/<name>/commits/<sha>
|
||||
bitbucket_commit_pattern = (
|
||||
r"https?://bitbucket.org/"
|
||||
r"(?P<namespace>[^/]+)/(?P<name>[^/]+)/commits/(?P<version>[0-9a-fA-F]{7,64})/?$"
|
||||
)
|
||||
|
||||
|
||||
@purl_router.route("https?://bitbucket\\.org/.*")
|
||||
def build_bitbucket_purl(url):
|
||||
"""
|
||||
Return a PackageURL object from BitBucket `url`.
|
||||
For example:
|
||||
https://bitbucket.org/TG1999/first_repo/src/master or
|
||||
https://bitbucket.org/TG1999/first_repo/src or
|
||||
https://bitbucket.org/TG1999/first_repo/src/master/new_folder
|
||||
https://bitbucket.org/TG1999/first_repo/commits/16a60c4a74ef477cd8c16ca82442eaab2fbe8c86
|
||||
"""
|
||||
commit_matche = re.search(bitbucket_commit_pattern, url)
|
||||
if commit_matche:
|
||||
return PackageURL(
|
||||
type="bitbucket",
|
||||
namespace=commit_matche.group("namespace"),
|
||||
name=commit_matche.group("name"),
|
||||
version=commit_matche.group("version"),
|
||||
qualifiers={},
|
||||
subpath="",
|
||||
)
|
||||
|
||||
segments = get_path_segments(url)
|
||||
|
||||
if not len(segments) >= 2:
|
||||
return
|
||||
namespace = segments[0]
|
||||
name = segments[1]
|
||||
|
||||
bitbucket_download_pattern = (
|
||||
r"https?://bitbucket.org/"
|
||||
r"(?P<namespace>.+)/(?P<name>.+)/downloads/"
|
||||
r"(?P<version>.+).(zip|tar.gz|tar.bz2|.tgz|exe|msi)"
|
||||
)
|
||||
matches = re.search(bitbucket_download_pattern, url)
|
||||
|
||||
qualifiers = {}
|
||||
if matches:
|
||||
qualifiers["download_url"] = url
|
||||
return PackageURL(type="bitbucket", namespace=namespace, name=name, qualifiers=qualifiers)
|
||||
|
||||
version = None
|
||||
subpath = None
|
||||
|
||||
# https://bitbucket.org/TG1999/first_repo/new_folder/
|
||||
if len(segments) >= 3 and segments[2] != "src":
|
||||
version = segments[2]
|
||||
subpath = "/".join(segments[3:])
|
||||
|
||||
# https://bitbucket.org/TG1999/first_repo/src/master/new_folder/
|
||||
if len(segments) >= 4 and segments[2] == "src":
|
||||
version = segments[3]
|
||||
subpath = "/".join(segments[4:])
|
||||
|
||||
return PackageURL(
|
||||
type="bitbucket",
|
||||
namespace=namespace,
|
||||
name=name,
|
||||
version=version,
|
||||
subpath=subpath,
|
||||
)
|
||||
|
||||
|
||||
@purl_router.route("https?://gitlab\\.com/(?!.*/archive/).*")
|
||||
def build_gitlab_purl(url):
|
||||
"""
|
||||
Return a PackageURL object from Gitlab `url`.
|
||||
For example:
|
||||
https://gitlab.com/TG1999/firebase/-/tree/1a122122/views
|
||||
https://gitlab.com/TG1999/firebase/-/tree
|
||||
https://gitlab.com/TG1999/firebase/-/master
|
||||
https://gitlab.com/tg1999/Firebase/-/tree/master
|
||||
https://gitlab.com/tg1999/Firebase/-/commit/bf04e5f289885cf2f20a92b387bcc6df33e30809
|
||||
"""
|
||||
# https://gitlab.com/<ns>/<name>/-/commit/<sha>
|
||||
commit_pattern = (
|
||||
r"https?://gitlab.com/"
|
||||
r"(?P<namespace>[^/]+)/(?P<name>[^/]+)/-/commit/"
|
||||
r"(?P<version>[0-9a-fA-F]{7,64})/?$"
|
||||
)
|
||||
|
||||
commit_matche = re.search(commit_pattern, url)
|
||||
if commit_matche:
|
||||
return PackageURL(
|
||||
type="gitlab",
|
||||
namespace=commit_matche.group("namespace"),
|
||||
name=commit_matche.group("name"),
|
||||
version=commit_matche.group("version"),
|
||||
qualifiers={},
|
||||
subpath="",
|
||||
)
|
||||
|
||||
segments = get_path_segments(url)
|
||||
|
||||
if not len(segments) >= 2:
|
||||
return
|
||||
namespace = segments[0]
|
||||
name = segments[1]
|
||||
version = None
|
||||
subpath = None
|
||||
|
||||
# https://gitlab.com/TG1999/firebase/master
|
||||
if (len(segments) >= 3) and segments[2] != "-" and segments[2] != "tree":
|
||||
version = segments[2]
|
||||
subpath = "/".join(segments[3:])
|
||||
|
||||
# https://gitlab.com/TG1999/firebase/-/tree/master
|
||||
if len(segments) >= 5 and (segments[2] == "-" and segments[3] == "tree"):
|
||||
version = segments[4]
|
||||
subpath = "/".join(segments[5:])
|
||||
|
||||
return PackageURL(
|
||||
type="gitlab",
|
||||
namespace=namespace,
|
||||
name=name,
|
||||
version=version,
|
||||
subpath=subpath,
|
||||
)
|
||||
|
||||
|
||||
# https://gitlab.com/hoppr/hoppr/-/archive/v1.11.1-dev.2/hoppr-v1.11.1-dev.2.tar.gz
|
||||
gitlab_archive_pattern = (
|
||||
r"^https?://gitlab.com/"
|
||||
r"(?P<namespace>.+)/(?P<name>.+)/-/archive/(?P<version>.+)/"
|
||||
r"(?P=name)-(?P=version).*"
|
||||
r"[^/]$"
|
||||
)
|
||||
|
||||
register_pattern("gitlab", gitlab_archive_pattern)
|
||||
|
||||
|
||||
# https://hackage.haskell.org/package/cli-extras-0.2.0.0/cli-extras-0.2.0.0.tar.gz
|
||||
hackage_download_pattern = (
|
||||
r"^https?://hackage.haskell.org/package/"
|
||||
r"(?P<name>.+)-(?P<version>.+)/"
|
||||
r"(?P=name)-(?P=version).*"
|
||||
r"[^/]$"
|
||||
)
|
||||
|
||||
register_pattern("hackage", hackage_download_pattern)
|
||||
|
||||
|
||||
# https://hackage.haskell.org/package/cli-extras-0.2.0.0/
|
||||
hackage_project_pattern = r"^https?://hackage.haskell.org/package/(?P<name>.+)-(?P<version>[^/]+)/"
|
||||
|
||||
register_pattern("hackage", hackage_project_pattern)
|
||||
|
||||
|
||||
@purl_router.route(
|
||||
"https?://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/.*"
|
||||
)
|
||||
def build_generic_google_code_archive_purl(uri):
|
||||
# https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com
|
||||
# /android-notifier/android-notifier-desktop-0.5.1-1.i386.rpm
|
||||
_, remaining_uri = uri.split(
|
||||
"https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/"
|
||||
)
|
||||
if remaining_uri: # android-notifier/android-notifier-desktop-0.5.1-1.i386.rpm
|
||||
split_remaining_uri = remaining_uri.split("/")
|
||||
# android-notifier, android-notifier-desktop-0.5.1-1.i386.rpm
|
||||
if split_remaining_uri:
|
||||
name = split_remaining_uri[0] # android-notifier
|
||||
return PackageURL(
|
||||
type="generic",
|
||||
namespace="code.google.com",
|
||||
name=name,
|
||||
qualifiers={"download_url": uri},
|
||||
)
|
||||
Reference in New Issue
Block a user