Merge pull request #200 from damianwasik98/vcs-urls

Parsing vcs urls in requirements and zope-event fix

core-requirements.txt

@@ -5,3 +5,4 @@ wheel-filename
 wheel-inspect>=1.6.0
 pip>=1.5.3
 junit-xml
+pip-requirements-parser>=32.0.1

devpi_builder/cli.py

@@ -114,7 +114,8 @@ def build_packages(self, packages):
 
         for package, version in packages:
             if self._should_package_be_build(package, version):
-                logger.info('Building %s %s', package, version)
+                msg = ('Building %s %s', package, version) if version else ('Building %s', package)
+                logger.info(*msg)
                 try:
                     wheel_file = self._builder(package, version)
                     self._upload_package(package, version, wheel_file)

devpi_builder/requirements.py

@@ -4,6 +4,7 @@
 Functionality for reading specifications of required packages.
 """
 
+import pip_requirements_parser
 import pkg_resources
 
 
@@ -14,23 +15,25 @@ def _extract_project_version(requirement):
     :param requirement: A pkg_config Requirement
     :return: Pair of project_name and version
     """
-    specs = requirement.specs
-    if len(specs) == 1:
-        spec = specs[0]
-        if spec[0] == '==':
-            return requirement.project_name, spec[1]
-        else:
-            raise ValueError('Versions must be specified exactly. "{}" is not an exact version specification.'.format(requirement))
-    elif len(specs) > 1:
-        raise ValueError('Multiple version specifications on a single line are not supported.')
+    if requirement.is_vcs_url:
+        return requirement.line, None
+    if not requirement.specifier:
+        raise ValueError('Version specification is missing for "{}".'.format(requirement.line))
     else:
-        raise ValueError('Version specification is missing for "{}".'.format(requirement))
+        if len(requirement.specifier) == 1 and requirement.is_pinned:
+            return requirement.name, requirement.get_pinned_version
+        elif len(requirement.specifier) == 1 and not requirement.is_pinned:
+            raise ValueError('Versions must be specified exactly. "{}" is not an exact version specification.'.format(requirement.line))
+        elif len(requirement.specifier) > 1:
+            raise ValueError('Multiple version specifications on a single line are not supported.')
 
 
 def read_raw(filename):
     if filename:
-        with open(filename) as requirements_file:
-            return list(pkg_resources.parse_requirements(requirements_file))
+        rf = pip_requirements_parser.RequirementsFile.from_file(filename)
+        if rf.invalid_lines:
+            raise ValueError("There are invalid lines in requirements file: \n", "\n".join(line.dumps() for line in rf.invalid_lines))
+        return rf.requirements
     else:
         return []
 
@@ -61,7 +64,7 @@ def matched_by_list(package, version, requirements):
     version = pkg_resources.safe_version('{}'.format(version))
     package = pkg_resources.safe_name(package)
     matches = (
-        package.lower() == requirement.key and version in requirement
+        package.lower() == pkg_resources.safe_name(requirement.name) and (requirement.specifier.contains(version) if requirement.specifier else 1)
         for requirement in requirements
     )
     return any(matches)

devpi_builder/wheeler.py

@@ -19,6 +19,7 @@
 
 class BuildError(Exception):
     def __init__(self, package, version, root_exception=None):
+        version = version or ""
         super(BuildError, self).__init__('Failed to create wheel for {} {}:\n{}\nOutput:\n{}'.format(
             package,
             version,
@@ -40,6 +41,10 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         shutil.rmtree(self.scratch_dir)
 
+    @staticmethod
+    def _standardize_package_name(name):
+        return name.replace(".", "-")
+
     def _matches_requirement(self, requirement, wheels):
         """
         List wheels matching a requirement.
@@ -52,7 +57,7 @@ def _matches_requirement(self, requirement, wheels):
         matching = []
         for wheel in wheels:
             w = wheel.parsed_filename
-            dist = Distribution(project_name=w.project, version=w.version)
+            dist = Distribution(project_name=self._standardize_package_name(w.project), version=w.version)
             if dist in req:
                 matching.append(wheel.path)
         return matching
@@ -63,7 +68,9 @@ def _find_wheel(self, name, version):
         Find a wheel with the given name and version
         """
         candidates = [WheelFile(filename) for filename in glob.iglob(path.join(self.wheelhouse, '*.whl'))]
-        matches = self._matches_requirement('{}=={}'.format(name, version), candidates)
+        name = self._standardize_package_name(name)
+        requirement = '{}=={}'.format(name, version) if version else name
+        matches = self._matches_requirement(requirement, candidates)
         if len(matches) > 0:
             return str(matches[0])
         else:
@@ -81,7 +88,7 @@ def build(self, package, version):
             subprocess.check_output([
                 'pip', 'wheel',
                 '--wheel-dir=' + self.wheelhouse,
-                '{}=={}'.format(package, version)
+                '{}=={}'.format(package, version) if version else package,
             ], stderr=subprocess.STDOUT)
             return self._find_wheel(package, version)
         except subprocess.CalledProcessError as e:

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.7
+# This file is autogenerated by pip-compile with Python 3.8
 # by the following command:
 #
 #    pip-compile --no-emit-index-url
@@ -93,6 +93,7 @@ multidict==6.0.2
 packaging==21.3
     # via
     #   build
+    #   pip-requirements-parser
     #   pytest
     #   setuptools-scm
     #   tox
@@ -103,6 +104,8 @@ pastedeploy==2.1.1
     # via plaster-pastedeploy
 pep517==0.12.0
     # via build
+pip-requirements-parser==32.0.1
+    # via -r core-requirements.txt
 pkginfo==1.8.2
     # via devpi-client
 plaster==1.0
@@ -133,7 +136,9 @@ pycparser==2.21
 pygments==2.15.0
     # via readme-renderer
 pyparsing==3.0.9
-    # via packaging
+    # via
+    #   packaging
+    #   pip-requirements-parser
 pyramid==2.0.2
     # via devpi-server
 pytest==7.1.2
@@ -157,6 +162,8 @@ requests==2.31.0
     #   devpi-plumber
 ruamel-yaml==0.17.21
     # via devpi-server
+ruamel-yaml-clib==0.2.7
+    # via ruamel-yaml
 setuptools-scm==6.4.2
     # via -r requirements.in
 six==1.16.0
@@ -173,7 +180,9 @@ toml==0.10.2
     # via tox
 tomli==2.0.1
     # via
+    #   build
     #   check-manifest
+    #   coverage
     #   pep517
     #   pytest
     #   setuptools-scm

tests/fixture/sample_vcs.txt

@@ -0,0 +1,3 @@
+requests @ git+https://github.com/kennethreitz/requests@master
+requests @ git+ssh://github.com/kennethreitz/requests@master
+requests @ git+https://github.com/kennethreitz/requests@2.16.0

tests/test_cli.py

@@ -162,7 +162,7 @@ def _assert_junit_xml_content(junit_filename, run_id=None):
     root = ET.parse(junit_filename)
     ET.dump(root)
 
-    _assert_test_case(root, 'failure', 'package-that-hopefully-not-exists 99.999' + run_id_str)
+    _assert_test_case(root, 'failure', 'package_that_hopefully_not_exists 99.999' + run_id_str)
     _assert_test_case(root, 'skipped', 'test-package 0.1.dev1' + run_id_str)
 
     pb_elems = root.findall(".//testcase[@name='progressbar 2.2{}']".format(run_id_str))

tests/test_requirements.py

@@ -8,10 +8,19 @@
 def test_read_requirements():
     expected = [
         ('progressbar', '2.2'),
-        ('six', '1.7.3')
+        ('six', '1.7.3'),
     ]
     assert expected == requirements.read_exact_versions('tests/fixture/sample_simple.txt')
 
+def test_read_vcs_requirements():
+    expected = [
+        ('requests @ git+https://github.com/kennethreitz/requests@master', None),
+        ('requests @ git+ssh://github.com/kennethreitz/requests@master', None),
+        ('requests @ git+https://github.com/kennethreitz/requests@2.16.0', None),
+    ]
+    assert expected == requirements.read_exact_versions('tests/fixture/sample_vcs.txt')
+
+
 
 def test_multiple_versions():
     expected = [

tests/test_wheeler.py

@@ -9,9 +9,19 @@
 
 
 class TestBuilder:
-    def test_build(self):
+    @pytest.mark.parametrize(
+        'args', (
+            ('progressbar', '2.2'),
+            ('progressbar', None),
+            ('requests @ git+https://github.com/kennethreitz/requests@2.16.0', None),
+            ('requests @ git+https://github.com/kennethreitz/requests@master', None),
+            ('zope-event', '5.0'),
+            ('devpi-common', '3.6.0'),
+        )
+    )
+    def test_build(self, args):
         with wheeler.Builder() as builder:
-            wheel_file = builder('progressbar', '2.2')
+            wheel_file = builder(*args)
             assert re.match(r'.*\.whl$', wheel_file)
             assert path.exists(wheel_file)