support URLs as packages

Author: jcushman

piptools/_compat/__init__.py

@@ -10,6 +10,7 @@
     Command,
     FormatControl,
     InstallRequirement,
+    Link,
     PackageFinder,
     PyPI,
     RequirementSet,
@@ -20,6 +21,7 @@
     install_req_from_line,
     is_file_url,
     parse_requirements,
+    path_to_url,
     stdlib_pkgs,
     url_to_path,
     user_cache_dir,

piptools/exceptions.py

@@ -48,16 +48,6 @@ def __str__(self):
         return "\n".join(lines)
 
 
-class UnsupportedConstraint(PipToolsError):
-    def __init__(self, message, constraint):
-        super(UnsupportedConstraint, self).__init__(message)
-        self.constraint = constraint
-
-    def __str__(self):
-        message = super(UnsupportedConstraint, self).__str__()
-        return "{} (constraint was: {})".format(message, str(self.constraint))
-
-
 class IncompatibleRequirements(PipToolsError):
     def __init__(self, ireq_a, ireq_b):
         self.ireq_a = ireq_a

piptools/repositories/base.py

@@ -25,7 +25,7 @@ def find_best_match(self, ireq):
     @abstractmethod
     def get_dependencies(self, ireq):
         """
-        Given a pinned or an editable InstallRequirement, returns a set of
+        Given a pinned, URL, or editable InstallRequirement, returns a set of
         dependencies (also InstallRequirements, but not necessarily pinned).
         They indicate the secondary dependencies for the given requirement.
         """

piptools/repositories/pypi.py

@@ -12,13 +12,15 @@
 
 from .._compat import (
     FAVORITE_HASH,
+    Link,
     PackageFinder,
     PyPI,
     RequirementSet,
     TemporaryDirectory,
     Wheel,
     contextlib,
     is_file_url,
+    path_to_url,
     url_to_path,
 )
 from ..cache import CACHE_DIR
@@ -137,7 +139,7 @@ def find_best_match(self, ireq, prereleases=None):
         Returns a Version object that indicates the best match for the given
         InstallRequirement according to the external repository.
         """
-        if ireq.editable or is_url_requirement(ireq, self):
+        if ireq.editable or is_url_requirement(ireq):
             return ireq  # return itself as the best match
 
         all_candidates = self.find_all_candidates(ireq.name)
@@ -229,13 +231,17 @@ def resolve_reqs(self, download_dir, ireq, wheel_cache):
 
     def get_dependencies(self, ireq):
         """
-        Given a pinned, a url, or an editable InstallRequirement, returns a set of
+        Given a pinned, URL, or editable InstallRequirement, returns a set of
         dependencies (also InstallRequirements, but not necessarily pinned).
         They indicate the secondary dependencies for the given requirement.
         """
-        if not (ireq.editable or is_url_requirement(ireq, self) or is_pinned_requirement(ireq)):
+        if not (
+            ireq.editable or is_url_requirement(ireq) or is_pinned_requirement(ireq)
+        ):
             raise TypeError(
-                "Expected url, pinned or editable InstallRequirement, got {}".format(ireq)
+                "Expected url, pinned or editable InstallRequirement, got {}".format(
+                    ireq
+                )
             )
 
         if ireq not in self._dependencies_cache:
@@ -282,6 +288,16 @@ def get_hashes(self, ireq):
         if ireq.editable:
             return set()
 
+        if is_url_requirement(ireq):
+            # url requirements may have been previously downloaded and cached
+            # locally by self.resolve_reqs()
+            cached_path = os.path.join(self._download_dir, ireq.link.filename)
+            if os.path.exists(cached_path):
+                cached_link = Link(path_to_url(cached_path))
+            else:
+                cached_link = ireq.link
+            return {self._get_file_hash(cached_link)}
+
         if not is_pinned_requirement(ireq):
             raise TypeError("Expected pinned requirement, got {}".format(ireq))
 

piptools/resolver.py

@@ -9,7 +9,6 @@
 from . import click
 from ._compat import install_req_from_line
 from .cache import DependencyCache
-from .exceptions import UnsupportedConstraint
 from .logging import log
 from .utils import (
     UNSAFE_PACKAGES,
@@ -102,8 +101,6 @@ def resolve(self, max_rounds=10):
             self.dependency_cache.clear()
             self.repository.clear_caches()
 
-        self.check_constraints(chain(self.our_constraints, self.their_constraints))
-
         # Ignore existing packages
         os.environ[str("PIP_EXISTS_ACTION")] = str(
             "i"
@@ -140,16 +137,6 @@ def resolve(self, max_rounds=10):
         # Only include hard requirements and not pip constraints
         return {req for req in best_matches if not req.constraint}
 
-    @staticmethod
-    def check_constraints(constraints):
-        for constraint in constraints:
-            if constraint.link is not None and constraint.link.url.startswith('file:') and not constraint.editable:
-                msg = (
-                    "pip-compile does not support file URLs as packages, unless "
-                    "they are editable. Perhaps add -e option?"
-                )
-                raise UnsupportedConstraint(msg, constraint)
-
     def _group_constraints(self, constraints):
         """
         Groups constraints (remember, InstallRequirements!) by their key name,
@@ -281,7 +268,7 @@ def get_best_match(self, ireq):
             Flask==0.10.1 => Flask==0.10.1
 
         """
-        if ireq.editable or is_url_requirement(ireq, self.repository):
+        if ireq.editable or is_url_requirement(ireq):
             # NOTE: it's much quicker to immediately return instead of
             # hitting the index server
             best_match = ireq
@@ -311,7 +298,7 @@ def _iter_dependencies(self, ireq):
         Editable requirements will never be looked up, as they may have
         changed at any time.
         """
-        if ireq.editable or is_url_requirement(ireq, self.repository):
+        if ireq.editable or is_url_requirement(ireq):
             for dependency in self.repository.get_dependencies(ireq):
                 yield dependency
             return
@@ -346,5 +333,7 @@ def _iter_dependencies(self, ireq):
             yield install_req_from_line(dependency_string, constraint=ireq.constraint)
 
     def reverse_dependencies(self, ireqs):
-        non_editable = [ireq for ireq in ireqs if not (ireq.editable or is_url_requirement(ireq, self.repository))]
+        non_editable = [
+            ireq for ireq in ireqs if not (ireq.editable or is_url_requirement(ireq))
+        ]
         return self.dependency_cache.reverse_dependencies(non_editable)

piptools/scripts/compile.py

@@ -14,7 +14,13 @@
 from ..pip import get_pip_command, pip_defaults
 from ..repositories import LocalRequirementsRepository, PyPIRepository
 from ..resolver import Resolver
-from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_req
+from ..utils import (
+    UNSAFE_PACKAGES,
+    dedup,
+    is_pinned_requirement,
+    key_from_ireq,
+    key_from_req,
+)
 from ..writer import OutputWriter
 
 DEFAULT_REQUIREMENTS_FILE = "requirements.in"
@@ -338,9 +344,6 @@ def cli(
         for find_link in repository.finder.find_links:
             log.debug("  -f {}".format(find_link))
 
-    # Check the given base set of constraints first
-    Resolver.check_constraints(constraints)
-
     try:
         resolver = Resolver(
             constraints,
@@ -413,10 +416,10 @@ def cli(
         unsafe_requirements=resolver.unsafe_constraints,
         reverse_dependencies=reverse_dependencies,
         primary_packages={
-            key_from_req(ireq.req) for ireq in constraints if not ireq.constraint
+            key_from_ireq(ireq) for ireq in constraints if not ireq.constraint
         },
         markers={
-            key_from_req(ireq.req): ireq.markers for ireq in constraints if ireq.markers
+            key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers
         },
         hashes=hashes,
     )

piptools/sync.py

@@ -5,17 +5,17 @@
 from subprocess import check_call  # nosec
 
 from . import click
-from .exceptions import IncompatibleRequirements, UnsupportedConstraint
+from ._compat import DEV_PKGS, stdlib_pkgs
+from .exceptions import IncompatibleRequirements
 from .utils import (
     flat_map,
     format_requirement,
     get_hashes_from_ireq,
+    is_url_requirement,
     key_from_ireq,
     key_from_req,
 )
 
-from piptools._compat import DEV_PKGS, stdlib_pkgs
-
 PACKAGES_TO_IGNORE = (
     ["-markerlib", "pip", "pip-tools", "pip-review", "pkg-resources"]
     + list(stdlib_pkgs)
@@ -77,14 +77,10 @@ def merge(requirements, ignore_conflicts):
     by_key = {}
 
     for ireq in requirements:
-        if ireq.link is not None and ireq.link.url.startswith('file:') and not ireq.editable:
-            msg = (
-                "pip-compile does not support file URLs as packages, unless they are "
-                "editable. Perhaps add -e option?"
-            )
-            raise UnsupportedConstraint(msg, ireq)
-
-        key = ireq.link or key_from_req(ireq.req)
+        # Limitation: URL requirements are merged by precise string match, so
+        # "file:///example.zip#egg=example", "file:///example.zip", and
+        # "example==1.0" will not merge with each other
+        key = key_from_ireq(ireq)
 
         if not ignore_conflicts:
             existing_ireq = by_key.get(key)
@@ -96,16 +92,35 @@ def merge(requirements, ignore_conflicts):
 
         # TODO: Always pick the largest specifier in case of a conflict
         by_key[key] = ireq
-
     return by_key.values()
 
 
+def diff_key_from_ireq(ireq):
+    """
+    Calculate a key for comparing a compiled requirement with installed modules.
+    For URL requirements, only provide a useful key if the url includes
+    #egg=name==version, which will set ireq.req.name and ireq.specifier.
+    Otherwise return ireq.link so the key will not match and the package will
+    reinstall. Reinstall is necessary to ensure that packages will reinstall
+    if the URL is changed but the version is not.
+    """
+    if is_url_requirement(ireq):
+        if (
+            ireq.req
+            and (getattr(ireq.req, "key", None) or getattr(ireq.req, "name", None))
+            and ireq.specifier
+        ):
+            return key_from_ireq(ireq)
+        return str(ireq.link)
+    return key_from_ireq(ireq)
+
+
 def diff(compiled_requirements, installed_dists):
     """
     Calculate which packages should be installed or uninstalled, given a set
     of compiled requirements and a list of currently installed modules.
     """
-    requirements_lut = {r.link or key_from_req(r.req): r for r in compiled_requirements}
+    requirements_lut = {diff_key_from_ireq(r): r for r in compiled_requirements}
 
     satisfied = set()  # holds keys
     to_install = set()  # holds InstallRequirement objects

piptools/utils.py

@@ -59,16 +59,12 @@ def make_install_requirement(name, version, extras, constraint=False):
     )
 
 
-def is_url_requirement(ireq, repository=None):
+def is_url_requirement(ireq):
     """
-    Finds if a requirement is a URL
+    Return True if requirement was specified as a path or URL.
+    ireq.original_link will have been set by InstallRequirement.__init__
     """
-    if not ireq.link or 'pypi.python.org' in ireq.link.url or ireq.link.url.startswith('file'):
-        return False
-    if repository is not None and hasattr(repository, 'finder'):
-        if any(index_url in ireq.link.url for index_url in repository.finder.index_urls):
-            return False
-    return True
+    return bool(ireq.original_link)
 
 
 def format_requirement(ireq, marker=None, hashes=None):
@@ -124,10 +120,7 @@ def is_pinned_requirement(ireq):
     if ireq.editable:
         return False
 
-    try:
-        if len(ireq.specifier._specs) != 1:
-            return False
-    except Exception:
+    if ireq.req is None or len(ireq.specifier._specs) != 1:
         return False
 
     op, version = next(iter(ireq.specifier._specs))._spec

piptools/writer.py

@@ -11,7 +11,7 @@
     dedup,
     format_requirement,
     get_compile_command,
-    key_from_req,
+    key_from_ireq,
 )
 
 
@@ -129,7 +129,7 @@ def _iter_lines(
                 ireq,
                 reverse_dependencies,
                 primary_packages,
-                markers.get(key_from_req(ireq.req)),
+                markers.get(key_from_ireq(ireq)),
                 hashes=hashes,
             )
             yield line
@@ -147,7 +147,7 @@ def _iter_lines(
                     ireq,
                     reverse_dependencies,
                     primary_packages,
-                    marker=markers.get(key_from_req(ireq.req)),
+                    marker=markers.get(key_from_ireq(ireq)),
                     hashes=hashes,
                 )
                 if not self.allow_unsafe:
@@ -185,7 +185,7 @@ def _format_requirement(
 
         line = format_requirement(ireq, marker=marker, hashes=ireq_hashes)
 
-        if not self.annotate or key_from_req(ireq.req) in primary_packages:
+        if not self.annotate or key_from_ireq(ireq) in primary_packages:
             return line
 
         # Annotate what packages this package is required by