Features/dependencies on files on pyton path

lektor/builder.py

@@ -749,6 +749,7 @@ def operation(con):
 
             seen = set()
             rows = []
+
             for source in chain(self.sources, dependencies or ()):
                 source = self.build_state.to_source_filename(source)
                 if source in seen:
@@ -1014,6 +1015,11 @@ def to_source_filename(self, filename):
             filename = filename[len(folder) :].lstrip(os.path.sep)
             if os.path.altsep:
                 filename = filename.lstrip(os.path.altsep)
+        elif self._is_on_python_path(filename):
+            # can't shorten the filename, as it is not below self.env.root_path
+            # but also need to oaccept it, or else all files with dependencies on
+            # this file will be removed by the prune stage of the build
+            pass
         else:
             raise ValueError(
                 "The given value (%r) is not below the "
@@ -1023,6 +1029,12 @@ def to_source_filename(self, filename):
         self.source_filename_cache[key] = rv
         return rv
 
+    def _is_on_python_path(self, filename):
+        for path in sys.path:
+            if filename.startswith(path):
+                return True
+        return False
+
     def get_file_info(self, filename):
         """Returns the file info for a given file.  This will be cached
         on the generator for the lifetime of it.  This means that further
@@ -1031,7 +1043,7 @@ def get_file_info(self, filename):
         files have been performed on the outside.
 
         Generally this function can be used to acquire the file info for
-        any file on the file system but it should onl be used for source
+        any file on the file system but it should only be used for source
         files or carefully for other things.
 
         The filename given can be a source filename.

lektor/packages.py

@@ -75,10 +75,8 @@ def remove_package_from_project(project, name):
     return None
 
 
-def download_and_install_package(
-    package_root, package=None, version=None, requirements_file=None
-):
-    """This downloads and installs a specific version of a package."""
+def download_and_install_packages(package_root, requirement_specifiers):
+    """This downloads and installs a specific version of packages."""
     # XXX: windows
     env = dict(os.environ)
 
@@ -91,18 +89,41 @@ def download_and_install_package(
         package_root,
     ]
 
-    if package is not None:
-        args.append("%s%s%s" % (package, version and "==" or "", version or ""))
-    if requirements_file is not None:
-        args.extend(("-r", requirements_file))
-
+    args.extend(requirement_specifiers)
     rv = portable_popen(args, env=env).wait()
     if rv != 0:
         raise RuntimeError("Failed to install dependency package.")
 
 
 def install_local_package(package_root, path):
     """This installs a local dependency of a package."""
+
+    # Because of these bugs:
+    # - pip https://github.com/pypa/pip/issues/4390
+    # - setuptools https://github.com/pypa/setuptools/issues/392
+    # we cannot just call `pip install --target $folder --editable $package`.
+    # Hence the workaround of first installing only the package and then it's dependencies
+    # TODO Once https://github.com/pypa/pip/pull/9636 is released, this can all go away
+    # and the normal install will just work
+
+    # Because pip can resolve dependencies differently when it is called with them individually,
+    # it is important to call it with all of them together. Also
+
+    # requires.txt is specified here:
+    # https://setuptools.readthedocs.io/en/latest/deprecated/python_eggs.html#requires-txt
+    # requirementx.txt is specified here:
+    # https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
+    # What can be part of the dependencies is specified here:
+    # https://setuptools.readthedocs.io/en/latest/userguide/dependency_management.html#id4
+
+    editable_install_without_dependencies(package_root, path)
+
+    requirements = requirements_from_unfinished_editable_install_at_path(path)
+    if requirements is not None:
+        download_and_install_packages(package_root, requirements)
+
+
+def editable_install_without_dependencies(package_root, path):
     # XXX: windows
     env = dict(os.environ)
     env["PYTHONPATH"] = package_root
@@ -124,6 +145,8 @@ def install_local_package(package_root, path):
     if rv != 0:
         raise RuntimeError("Failed to install local package")
 
+
+def requirements_from_unfinished_editable_install_at_path(path):
     # Step 2: generate the egg info into a temp folder to find the
     # requirements.
     tmp = tempfile.mkdtemp()
@@ -143,14 +166,31 @@ def install_local_package(package_root, path):
         dirs = os.listdir(tmp)
         if rv != 0 or len(dirs) != 1:
             raise RuntimeError("Failed to create egg info for local package.")
-        requires = os.path.join(tmp, dirs[0], "requires.txt")
+        requires_path = os.path.join(tmp, dirs[0], "requires.txt")
 
-        # We have dependencies, install them!
-        if os.path.isfile(requires):
-            download_and_install_package(package_root, requirements_file=requires)
+        if os.path.isfile(requires_path):
+            # We have dependencies, install them!
+            return requirements_from_requires_file(requires_path)
     finally:
         shutil.rmtree(tmp)
 
+    # no dependencies to install
+    return None
+
+
+def requirements_from_requires_file(requires_path):
+    """Create a sanitized copy of `requires.txt`."""
+    # requires.txt can contain [extras_require] sections wich pip doesn't understand
+    with open(requires_path, "r") as requires:
+        section_name, extracted_requirements = next(
+            pkg_resources.split_sections(requires)
+        )
+        if section_name is None:
+            return extracted_requirements
+
+    # no dependencies to install
+    return None
+
 
 def get_package_info(path):
     """Returns the name of a package at a path."""
@@ -292,7 +332,9 @@ def update_cache(package_root, remote_packages, local_package_path, refresh=Fals
                     package_root, os.path.join(local_package_path, package[1:])
                 )
             else:
-                download_and_install_package(package_root, package, version)
+                requirement_spec = f"{package}=={version}" if version else package
+                download_and_install_packages(package_root, [requirement_spec])
+
         write_manifest(manifest_file, all_packages)
 
 

tests/test_builder.py

@@ -268,3 +268,9 @@ def test_slug_contains_slash(pad, builder):
     prog, _ = builder.build(record)
     (artifact,) = prog.artifacts
     assert artifact.artifact_name == "extra/long/path/index.html"
+
+
+def test_path_cache_allows_paths_on_pythonpath(env):
+    # cache = PathCache(env)
+    # FIXME no idea yet how to test this
+    pass

tests/test_packages.py

@@ -0,0 +1,151 @@
+import textwrap
+
+import pytest
+
+from lektor import packages
+
+
+def create_plugin(package_dir, plugin_name, setup):
+    plugin_dir = package_dir / plugin_name
+    plugin_dir.mkdir(parents=True)
+    setup_py = plugin_dir / "setup.py"
+    setup_py.write_text(textwrap.dedent(setup))
+    return plugin_dir
+
+
+def test_install_local_package_without_dependency(tmp_path):
+    packages_dir = tmp_path / "packages"
+    plugin_dir = create_plugin(
+        packages_dir,
+        "dependency",
+        setup="""
+        from setuptools import setup
+
+        setup(
+            name='dependency',
+        )
+        """,
+    )
+
+    install_dir = tmp_path / "target"
+    install_dir.mkdir()
+
+    packages.install_local_package(install_dir.as_posix(), plugin_dir.as_posix())
+
+    assert (install_dir / "dependency.egg-link").is_file()
+
+
+def test_install_local_package_with_dependency(tmp_path):
+    packages_dir = tmp_path / "packages"
+    plugin_dir = create_plugin(
+        packages_dir,
+        "dependency",
+        setup="""
+        from setuptools import setup
+
+        setup(
+            name='dependency',
+            install_requires=['watching_testrunner']
+        )
+        """,
+    )
+
+    install_dir = tmp_path / "target"
+    install_dir.mkdir()
+
+    packages.install_local_package(install_dir.as_posix(), plugin_dir.as_posix())
+
+    assert (install_dir / "dependency.egg-link").is_file()
+    assert (install_dir / "watching_testrunner.py").is_file()
+
+
+def test_install_local_package_with_dependency_and_extras_require(tmp_path):
+    packages_dir = tmp_path / "packages"
+    plugin_dir = create_plugin(
+        packages_dir,
+        "dependency",
+        setup="""
+        from setuptools import setup
+
+        setup(
+            name='dependency',
+            install_requires=['watching_testrunner'],
+            extras_require={
+                'test': ['pyexpect']
+            }
+        )
+        """,
+    )
+
+    install_dir = tmp_path / "target"
+    install_dir.mkdir()
+
+    packages.install_local_package(install_dir.as_posix(), plugin_dir.as_posix())
+
+    assert (install_dir / "dependency.egg-link").is_file()
+    assert (install_dir / "watching_testrunner.py").is_file()
+    assert not (install_dir / "pyexpect").is_dir()
+
+
+def test_install_local_package_with_only_extras_require(tmp_path):
+    packages_dir = tmp_path / "packages"
+    plugin_dir = create_plugin(
+        packages_dir,
+        "extras_require",
+        setup="""
+        from setuptools import setup
+
+        setup(
+            name='extras_require',
+            extras_require={
+                'test': ['pyexpect']
+            }
+        )
+        """,
+    )
+
+    install_dir = tmp_path / "target"
+    install_dir.mkdir()
+
+    packages.install_local_package(install_dir.as_posix(), plugin_dir.as_posix())
+
+    assert (install_dir / "extras-require.egg-link").is_file()
+    assert not (install_dir / "pyexpect").is_dir()
+
+
+@pytest.mark.xfail
+def test_install_local_package_with_environment_markers_in_requires(tmp_path):
+    # See https://www.python.org/dev/peps/pep-0508/#environment-markers
+    # This test is bad news, as it seems that setuptools internally transforms the more exotic specifiers to
+    # [:python_version >= "3.6"]
+    # watching_testrunner
+    # Which is then of course incompatible with a pip requirements.txt
+    # This seems contrary to what
+    # https://setuptools.readthedocs.io/en/latest/pkg_resources.html?highlight=parse_requirements#requirements-parsing
+    # specifies - not sure what is going on here
+    # This will start to work as soon as we can remove the workarounds that where neccessary until
+    # https://github.com/pypa/pip/pull/9636 is released
+
+    packages_dir = tmp_path / "packages"
+    plugin_dir = create_plugin(
+        packages_dir,
+        "environment_markers",
+        setup="""
+        from setuptools import setup
+
+        setup(
+            name='environment_markers',
+            install_requires=[
+                'watching_testrunner;python_version>="3.6"',
+            ]
+        )
+        """,
+    )
+
+    install_dir = tmp_path / "target"
+    install_dir.mkdir()
+
+    packages.install_local_package(install_dir.as_posix(), plugin_dir.as_posix())
+
+    assert (install_dir / "extras-require.egg-link").is_file()
+    assert (install_dir / "watching_testrunner.py").is_file()