Dev (#63)

* fix abi tag for python 3.8

* handle circular deps; print info ResolutionImpossible; add tests

* fix readme

* version 2.2.1

Author: DavHau

Changelog.md

@@ -1,3 +1,17 @@
+# 2.2.1 (11 Aug 2020)
+Handle circular dependencies, fix python 3.8 wheels, improve error message
+
+### Features
+ - Print more detailed info when the resolver raises a ResolutionImpossible error.
+ - Warm on circular dependencies and fix them automatically.
+
+### Fixes
+ - Fix crash on circular dependencies.
+ - Python 3.8 wheels have abi tag cp38, not cp38m. This was not considered before which prevented finding suitable manylinux wheels for python 3.8
+ 
+### Development
+ - Added integration tests under [./tests/](/tests/)
+
 # 2.2.0 (09 Aug 2020)
 Improved success rate, MacOS support, bugfixes, optimizations
 

Readme.md

@@ -55,11 +55,11 @@ Table of Contents
 You can either install mach-nix via pip or by using nix in case you already have the nix package manager installed.
 #### Installing via pip
 ```shell
-pip install git+git://github.com/DavHau/mach-nix@2.2.0
+pip install git+git://github.com/DavHau/mach-nix@2.2.1
 ```
 #### Installing via nix
 ```shell
-nix-env -if https://github.com/DavHau/mach-nix/tarball/2.2.0 -A mach-nix
+nix-env -if https://github.com/DavHau/mach-nix/tarball/2.2.1 -A mach-nix
 ```
 
 ---
@@ -91,7 +91,7 @@ You can call mach-nix directly from a nix expression
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.2.0";
+    ref = "2.2.1";
   });
 in
 mach-nix.mkPython {
@@ -158,7 +158,7 @@ Providers can be disabled/enabled/preferred like in the following examples:
 
 Mach-nix will always satisfy your **requirements.txt** fully with the configured providers or fail with a **ResolutionImpossible** error.
 
-If a mach-nix build fails, Most of the times it can be resolved by just switching the provider of a package, which is simple and doesn't require writing a lot of nix code. For some more complex scenarios, checkout the following examples.
+If a mach-nix build fails, Most of the times it can be resolved by just switching the provider of a package, which is simple and doesn't require writing a lot of nix code. For some more complex scenarios, checkout the [./examples.md](/examples.md).
 
 ## Why nix?
  Usually people rely on multiple layers of different package management tools for building their software environments. These tools are often not well integrated with each other and don't offer strong reproducibility. Example: You are on debian/ubuntu and use APT (layer 1) to install python. Then you use venv (layer 2) to overcome some of your layer 1 limitations (not being able to have multiple versions of the same package installed) and afterwards you are using pip (layer 3) to install python packages. You notice that even after pinning all your requirements, your environment behaves differently on your server or your colleagues machine because their underlying system differs from yours. You start using docker (layer 4) to overcome this problem which adds extra complexity to the whole process and gives you some nasty limitations during development. You need to configure your IDE's docker integration and so on. Despite all the effort you put in, still the problem is not fully solved and from time to time your build pipeline just breaks and you need to fix it manually. 

examples.md

@@ -24,7 +24,7 @@ build a python environment from a list of requirements
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.2.0";
+    ref = "2.2.1";
   });
 in mach-nix.mkPython {
   requirements = builtins.readFile ./requirements.txt;
@@ -37,7 +37,7 @@ Build a python package from its source code and a list of requirements
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.2.0";
+    ref = "2.2.1";
   });
 in mach-nix.buildPythonPackage {
   pname = "my-package";
@@ -53,7 +53,7 @@ Build a python package from its source code and a list of requirements
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.2.0";
+    ref = "2.2.1";
   });
 in mach-nix.buildPythonPackage rec {
   pname = "projectname";
@@ -77,7 +77,7 @@ I have a complex set of requirements including tensorflow. I'd like to have tens
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.2.0";
+    ref = "2.2.1";
   });
 in mach-nix.mkPython {
 
@@ -100,7 +100,7 @@ I'd like to install a more recent version of tensorflow which is not available f
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.2.0";
+    ref = "2.2.1";
   });
 in mach-nix.mkPython {
 
@@ -129,7 +129,7 @@ I'd like to use a recent version of Pytorch from wheel, but I'd like to build th
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.2.0";
+    ref = "2.2.1";
   });
 overlays = []; # some very useful overlays
 in mach-nix.mkPython rec {
@@ -164,7 +164,7 @@ I have a complex requirements.txt which includes `imagecodecs`. It is available
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "2.2.0";
+    ref = "2.2.1";
   });
 in mach-nix.mkPython rec {
 

mach_nix/VERSION

@@ -1 +1 @@
-2.2.0
+2.2.1

mach_nix/data/providers.py

@@ -260,7 +260,7 @@ def __init__(self, data_dir: str, *args, **kwargs):
         maj = self.py_ver_digits[0]  # major version
         min = self.py_ver_digits[1]  # minor version
         if self.system == "linux":
-            cp_abi = f"cp{maj}{min}mu" if int(maj) == 2 else f"cp{maj}{min}m"
+            cp_abi = f"cp{maj}{min}mu" if int(maj) == 2 else f"cp{maj}{min}m?"
             self.preferred_wheels = (
                 re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-manylinux2014_{self.platform}"),
                 re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-manylinux2010_{self.platform}"),

mach_nix/visualize.py mach_nix/deptree.pymach_nix/visualize.py renamed to mach_nix/deptree.py

@@ -1,5 +1,6 @@
 from operator import itemgetter
-from typing import Iterable
+from typing import Iterable, List
+
 from tree_format import format_tree
 
 from mach_nix.data.nixpkgs import NixpkgsIndex
@@ -15,23 +16,45 @@ def __init__(self, pkg: ResolvedPkg, name, parent: 'Node' = None):
         if parent:
             self.parent.children.append(self)
 
+    def all_parents(self) -> List['Node']:
+        if self.parent is None:
+            return []
+        return [self.parent] + self.parent.all_parents()
+
 
 def make_name(pkg: ResolvedPkg, nixpkgs: NixpkgsIndex):
     pi = pkg.provider_info
-    name = f"{pkg.name} - {pkg.ver} - {pi.provider}"
+    extras = f"[{' '.join(pkg.extras_selected)}]" if pkg.extras_selected else ''
+    name = f"{pkg.name}{extras} - {pkg.ver} - {pi.provider}"
     if pi.provider == 'wheel':
         name += f" - {'-'.join(pi.wheel_fname.split('-')[-3:])[:-4]}"
     if pi.provider == 'nixpkgs':
         name += f" (attrs: {' '.join(c.nix_key for c in nixpkgs.get_all_candidates(pkg.name))})"
     return name
 
 
-def build_tree(pkgs: dict, root: Node, nixpkgs: NixpkgsIndex):
+def build_tree(pkgs: dict, root: Node, nixpkgs: NixpkgsIndex) -> List[str]:
+    """
+    Recursively adds children to given root node.
+    Removes cycles from original graph while processing.
+    Returns list of warnings.
+    """
     root_pkg = pkgs[root.pkg.name]
+    warnings = []
     for name in sorted(root_pkg.build_inputs + root_pkg.prop_build_inputs):
         child_pkg: ResolvedPkg = pkgs[name]
+        # detect circles
+        if child_pkg in [node.pkg for node in root.all_parents()]:
+            warnings.append(
+                f"WARNING: Circular dependency detected and removed:"
+                f" {root.pkg.name}:{root.pkg.ver} -> {child_pkg.name}:{child_pkg.ver}")
+            root.pkg.build_inputs = [bi for bi in root.pkg.build_inputs if bi != child_pkg.name]
+            root.pkg.prop_build_inputs = [bi for bi in root.pkg.prop_build_inputs if bi != child_pkg.name]
+            root.pkg.removed_circular_deps.append(child_pkg.name)
+            continue
         child_node = Node(child_pkg, make_name(child_pkg, nixpkgs), root)
-        build_tree(pkgs, child_node, nixpkgs)
+        warnings += build_tree(pkgs, child_node, nixpkgs)
+    return warnings
 
 
 def tree_to_dict(root_node: Node):
@@ -44,11 +67,13 @@ def print_tree(root_node):
         d, format_node=itemgetter(0), get_children=itemgetter(1)))
 
 
-def print_deps(pkgs: Iterable[ResolvedPkg], nixpkgs: NixpkgsIndex):
+def remove_circles_and_print(pkgs: Iterable[ResolvedPkg], nixpkgs: NixpkgsIndex):
     print("\n### Resolved Dependencies ###\n")
     indexed_pkgs = {p.name: p for p in sorted(pkgs, key=lambda p: p.name)}
     roots: Iterable[ResolvedPkg] = (p for p in pkgs if p.is_root)
     for root in roots:
         root_node = Node(root, make_name(root, nixpkgs))
-        build_tree(indexed_pkgs, root_node, nixpkgs)
+        warnings = build_tree(indexed_pkgs, root_node, nixpkgs)
         print_tree(root_node)
+        if warnings:
+            print(''.join(warnings) + '\n')

mach_nix/generate.py

@@ -1,6 +1,14 @@
+import json
 import os
 import sys
+from os.path import dirname
+from pprint import pformat
+from typing import List
 
+from resolvelib import ResolutionImpossible
+from resolvelib.resolvers import RequirementInformation
+
+import mach_nix
 from mach_nix.data.nixpkgs import NixpkgsIndex
 from mach_nix.data.providers import CombinedDependencyProvider, ProviderSettings
 from mach_nix.generators.overides_generator import OverridesGenerator
@@ -18,10 +26,12 @@ def load_env(name, *args, **kwargs):
 
 
 def main():
+    providers_json = load_env('providers')
+
     disable_checks = load_env('disable_checks')
     nixpkgs_json = load_env('nixpkgs_json')
     out_file = load_env('out_file')
-    provider_settings = ProviderSettings(load_env('providers'))
+    provider_settings = ProviderSettings(providers_json)
     py_ver_str = load_env('py_ver_str')
     pypi_deps_db_src = load_env('pypi_deps_db_src')
     pypi_fetcher_commit = load_env('pypi_fetcher_commit')
@@ -49,9 +59,35 @@ def main():
         ResolvelibResolver(nixpkgs, deps_provider),
     )
     reqs = filter_reqs_by_eval_marker(parse_reqs(requirements), context(py_ver, platform, system))
-    expr = generator.generate(reqs)
-    with open(out_file, 'w') as f:
-        f.write(expr)
+    try:
+        expr = generator.generate(reqs)
+    except ResolutionImpossible as e:
+        handle_resolution_impossible(e, requirements, providers_json, py_ver_str)
+        exit(1)
+    else:
+        with open(out_file, 'w') as f:
+            f.write(expr)
+
+
+def handle_resolution_impossible(exc: ResolutionImpossible, reqs_str, providers_json, py_ver_str):
+    causes: List[RequirementInformation] = exc.causes
+    causes_str = ''
+    for ri in causes:
+        causes_str += f"\n  {ri.requirement}"
+        if ri.parent:
+            causes_str += \
+                f" - parent: {ri.parent.name}{ri.parent.extras if ri.parent.extras else None}:{ri.parent.version}"
+    nl = '\n'
+    print(
+        f"\nSome requirements could not be resolved.\n"
+        f"Top level requirements: \n  {'  '.join(l for l in reqs_str.splitlines())}\n"
+        f"Providers:\n  {f'{nl}  '.join(pformat(json.loads(providers_json)).splitlines())}\n"
+        f"Mach-nix version: {open(dirname(mach_nix.__file__) + '/VERSION').read().strip()}\n"
+        f"Python: {py_ver_str}\n"
+        f"Cause: {exc.__context__}\n"
+        f"The requirements which caused the error:"
+        f"{causes_str}\n",
+        file=sys.stderr)
 
 
 if __name__ == "__main__":

mach_nix/generators/overides_generator.py

@@ -68,14 +68,17 @@ def _gen_prop_build_inputs(self, prop_build_inputs_local, prop_build_inputs_nixp
             f"{b}" for b in sorted(prop_build_inputs_local | prop_build_inputs_nixpkgs))
         return prop_build_inputs_str
 
-    def _gen_overrideAttrs(self, name, ver, nix_name, build_inputs_str, prop_build_inputs_str, keep_src=False):
+    def _gen_overrideAttrs(self, name, ver, circular_deps, nix_name, build_inputs_str, prop_build_inputs_str, keep_src=False):
         out = f"""
             {nix_name} = python-super.{nix_name}.overridePythonAttrs ( oldAttrs: {{
               pname = "{name}";
               version = "{ver}";"""
         if not keep_src:
             out += f"""
               src = fetchPypi "{name}" "{ver}";"""
+        if circular_deps:
+            out += f"""
+              pipInstallFlags = "--no-dependencies";"""
         if build_inputs_str:
             out += f"""
               buildInputs = with python-self; (filter_deps oldAttrs "buildInputs") ++ [ {build_inputs_str} ];"""
@@ -90,12 +93,15 @@ def _gen_overrideAttrs(self, name, ver, nix_name, build_inputs_str, prop_build_i
             });\n"""
         return unindent(out, 8)
 
-    def _gen_builPythonPackage(self, name, ver, nix_name, build_inputs_str, prop_build_inputs_str):
+    def _gen_builPythonPackage(self, name, ver, circular_deps, nix_name, build_inputs_str, prop_build_inputs_str):
         out = f"""
             {nix_name} = python-self.buildPythonPackage {{
               pname = "{name}";
               version = "{ver}";
               src = fetchPypi "{name}" "{ver}";"""
+        if circular_deps:
+            out += f"""
+              pipInstallFlags = "--no-dependencies";"""
         if build_inputs_str.strip():
             out += f"""
               buildInputs = with python-self; [ {build_inputs_str} ];"""
@@ -110,7 +116,7 @@ def _gen_builPythonPackage(self, name, ver, nix_name, build_inputs_str, prop_bui
             };\n"""
         return unindent(out, 8)
 
-    def _gen_wheel_buildPythonPackage(self, name, ver, prop_build_inputs_str, fname):
+    def _gen_wheel_buildPythonPackage(self, name, ver, circular_deps, prop_build_inputs_str, fname):
         manylinux = "manylinux1 ++ " if 'manylinux' in fname else ''
 
         # dontStrip added due to this bug - https://github.com/pypa/manylinux/issues/119
@@ -123,6 +129,9 @@ def _gen_wheel_buildPythonPackage(self, name, ver, prop_build_inputs_str, fname)
               doCheck = false;
               doInstallCheck = false;
               dontStrip = true;"""
+        if circular_deps:
+            out += f"""
+              pipInstallFlags = "--no-dependencies";"""
         if manylinux:
             out += f"""
               nativeBuildInputs = [ autoPatchelfHook ];
@@ -174,19 +183,19 @@ def _gen_overrides(self, pkgs: Dict[str, ResolvedPkg], overlay_keys, pkgs_names:
                 # or by creating it from scratch using `buildPythonPackage`
                 nix_name = self._get_ref_name(pkg.name, pkg.ver)
                 if self.nixpkgs.exists(pkg.name):
-                    out += self._gen_overrideAttrs(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str)
+                    out += self._gen_overrideAttrs(pkg.name, pkg.ver, pkg.removed_circular_deps, nix_name, build_inputs_str, prop_build_inputs_str)
                     out += self._unify_nixpkgs_keys(pkg.name, main_key=nix_name)
                 else:
-                    out += self._gen_builPythonPackage(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str)
+                    out += self._gen_builPythonPackage(pkg.name, pkg.ver, pkg.removed_circular_deps, nix_name, build_inputs_str, prop_build_inputs_str)
             elif pkg.provider_info.provider == WheelDependencyProvider.name:
-                out += self._gen_wheel_buildPythonPackage(pkg.name, pkg.ver, prop_build_inputs_str,
+                out += self._gen_wheel_buildPythonPackage(pkg.name, pkg.ver, pkg.removed_circular_deps, prop_build_inputs_str,
                                                           pkg.provider_info.wheel_fname)
                 if self.nixpkgs.exists(pkg.name):
                     out += self._unify_nixpkgs_keys(pkg.name)
             elif pkg.provider_info.provider == NixpkgsDependencyProvider.name:
                 nix_name = self.nixpkgs.find_best_nixpkgs_candidate(pkg.name, pkg.ver)
                 out += self._gen_overrideAttrs(
-                    pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str,
+                    pkg.name, pkg.ver, pkg.removed_circular_deps, nix_name, build_inputs_str, prop_build_inputs_str,
                     keep_src=True)
                 out += self._unify_nixpkgs_keys(pkg.name, main_key=nix_name)
         end_overlay_section = f"""

mach_nix/resolver/__init__.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import List, Iterable, Optional
 
 from packaging.version import Version
@@ -17,6 +17,7 @@ class ResolvedPkg:
     is_root: bool
     provider_info: ProviderInfo
     extras_selected: List[str]
+    removed_circular_deps: List[str] = field(default_factory=list)
 
 
 class Resolver(ABC):

mach_nix/resolver/resolvelib_resolver.py

@@ -9,7 +9,7 @@
 from mach_nix.requirements import Requirement
 from mach_nix.resolver import Resolver, ResolvedPkg
 from mach_nix.versions import filter_versions
-from mach_nix.visualize import print_deps
+from mach_nix.deptree import remove_circles_and_print
 
 
 @dataclass
@@ -81,5 +81,5 @@ def resolve(self, reqs: Iterable[Requirement]) -> List[ResolvedPkg]:
                 provider_info=provider_info,
                 extras_selected=list(result.mapping[name].extras)
             ))
-        print_deps(nix_py_pkgs, self.nixpkgs)
+        remove_circles_and_print(nix_py_pkgs, self.nixpkgs)
         return nix_py_pkgs