Add basic setup.py extraction

Author: thatch

metadata_please/source_checkout.py

@@ -7,9 +7,11 @@
 - PEP 621 metadata (pyproject.toml)
 - Poetry metadata (pyproject.toml)
 - Setuptools static metadata (setup.cfg)
+- Setuptools, low effort reading (setup.py)
 
-Notably, does not read setup.py or attempt to emulate anything that can't be read staticly.
+Notably, does not read nontrivial setup.py or attempt to emulate anything that can't be read staticly.
 """
+import ast
 import re
 from pathlib import Path
 
@@ -22,6 +24,8 @@
 
 from packaging.utils import canonicalize_name
 
+from .source_checkout_ast import SetupFindingVisitor, UNKNOWN
+
 from .types import BasicMetadata
 
 OPERATOR_RE = re.compile(r"([<>=~]+)(\d.*)")
@@ -54,6 +58,7 @@ def from_source_checkout(path: Path) -> bytes:
         from_pep621_checkout(path)
         or from_poetry_checkout(path)
         or from_setup_cfg_checkout(path)
+        or from_setup_py_checkout(path)
     )
 
 
@@ -227,6 +232,36 @@ def from_setup_cfg_checkout(path: Path) -> bytes:
     return "".join(buf).encode("utf-8")
 
 
+def from_setup_py_checkout(path: Path) -> bytes:
+    try:
+        data = (path / "setup.py").read_bytes()
+    except FileNotFoundError:
+        return b""
+
+    v = SetupFindingVisitor()
+    v.visit(ast.parse(data))
+
+    if not v.setup_call_args:
+        return b""
+
+    buf = []
+    if r := v.setup_call_args.get("install_requires"):
+        if r is UNKNOWN:
+            raise ValueError("Complex setup call can't extract reqs")
+        for dep in r:
+            buf.append(f"Requires-Dist: {dep}\n")
+    if er := v.setup_call_args.get("extras_require"):
+        if er is UNKNOWN:
+            raise ValueError("Complex setup call can't extract extras")
+        for k, deps in er.items():
+            extra_name = canonicalize_name(k)
+            buf.append(f"Provides-Extra: {extra_name}\n")
+            for i in deps:
+                buf.append("Requires-Dist: " + merge_extra_marker(extra_name, i) + "\n")
+
+    return "".join(buf).encode("utf-8")
+
+
 def basic_metadata_from_source_checkout(path: Path) -> BasicMetadata:
     return BasicMetadata.from_metadata(from_source_checkout(path))
 

metadata_please/source_checkout_ast.py

@@ -0,0 +1,121 @@
+"""
+Reads static values from setup.py when they are simple enough.
+
+With the goal of just getting dependencies, and returning a clear error if we don't understand, this has a simpler
+
+This only reads ~50% of current setup.py, vs dowsing which is more like 80%.
+
+I experimented with a more complex version of this in
+[dowsing](https://github.com/python-packaging/dowsing/) with a goal of 100%
+coverage of open source
+"""
+
+import ast
+
+
+# Copied from orig-index
+class ShortCircuitingVisitor(ast.NodeVisitor):
+    """
+    This visitor behaves more like libcst.CSTVisitor in that a visit_ method
+    can return true or false to specify whether children get visited, and the
+    visiting of children is not the responsibility of the visit_ method.
+    """
+
+    def visit(self, node):
+        method = "visit_" + node.__class__.__name__
+        visitor = getattr(self, method, self.generic_visit)
+        rv = visitor(node)
+        if rv:
+            self.visit_children(node)
+
+    def visit_children(self, node):
+        for field, value in ast.iter_fields(node):
+            if isinstance(value, list):
+                for item in value:
+                    if isinstance(item, ast.AST):
+                        self.visit(item)
+            elif isinstance(value, ast.AST):
+                self.visit(value)
+
+    def generic_visit(self, node) -> bool:
+        return True
+
+
+class QualifiedNameSaver(ShortCircuitingVisitor):
+    """Similar to LibCST's QualifiedNameProvider except simpler and wronger"""
+
+    def __init__(self):
+        super().__init__()
+        self.qualified_name_prefixes = {}
+
+    def qualified_name(self, node: ast.AST) -> str:
+        if isinstance(node, ast.Attribute):
+            return self.qualified_name(node.value) + "." + node.attr
+        elif isinstance(node, ast.Expr):
+            return self.qualified_name(node.value)
+        elif isinstance(node, ast.Name):
+            if new := self.qualified_name_prefixes.get(node.id):
+                return new
+            return f"<locals>.{node.id}"
+        else:
+            raise ValueError(f"Complex expression: {type(node)}")
+
+    def visit_Import(self, node: ast.Import):
+        # .names
+        #     alias = (identifier name, identifier? asname)
+        for a in node.names:
+            self.qualified_name_prefixes[a.asname or a.name] = a.name
+
+    def visit_ImportFrom(self, node: ast.ImportFrom):
+        # .identifier / .level
+        # .names
+        #     alias = (identifier name, identifier? asname)
+        if node.module:
+            prefix = f"{node.module}."
+        else:
+            prefix = "." * node.level
+
+        for a in node.names:
+            self.qualified_name_prefixes[a.asname or a.name] = prefix + a.name
+
+
+class Unknown:
+    pass
+
+
+UNKNOWN = Unknown()
+
+
+class SetupFindingVisitor(QualifiedNameSaver):
+    def __init__(self):
+        super().__init__()
+        self.setup_call_args = None
+        self.setup_call_kwargs = None
+
+    def visit_Call(self, node):
+        # .func (expr, can just be name)
+        # .args
+        # .keywords
+        qn = self.qualified_name(node.func)
+        if qn in ("setuptools.setup", "distutils.setup"):
+            self.setup_call_args = d = {}
+            self.setup_call_kwargs = False
+            # Positional args are rarely used
+            for k in node.keywords:
+                if not k.arg:
+                    self.setup_call_kwargs = True
+                else:
+                    try:
+                        d[k.arg] = ast.literal_eval(k.value)
+                    except ValueError:  # malformed node or string...
+                        d[k.arg] = UNKNOWN
+
+
+if __name__ == "__main__":
+    import sys
+    from pathlib import Path
+
+    mod = ast.parse(Path(sys.argv[1]).read_bytes())
+    v = SetupFindingVisitor()
+    v.visit(mod)
+    print(v.setup_call_args)

metadata_please/tests/source_checkout.py

@@ -95,6 +95,40 @@ def test_setuptools_empty(self) -> None:
                 basic_metadata_from_source_checkout(Path(d)),
             )
 
+    def test_setuppy_empty(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            Path(d, "setup.py").write_text("")
+            self.assertEqual(
+                BasicMetadata((), frozenset()),
+                basic_metadata_from_source_checkout(Path(d)),
+            )
+
+    def test_setuppy_trivial(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            Path(d, "setup.py").write_text("from setuptools import setup; setup()")
+            self.assertEqual(
+                BasicMetadata((), frozenset()),
+                basic_metadata_from_source_checkout(Path(d)),
+            )
+
+    def test_setuppy(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            Path(d, "setup.py").write_text(
+                "import setuptools; setuptools.setup(install_requires=['a'], extras_require={'b': ['c']})"
+            )
+            self.assertEqual(
+                BasicMetadata(["a", 'c ; extra == "b"'], frozenset("b")),
+                basic_metadata_from_source_checkout(Path(d)),
+            )
+
+    def test_setuppy_toocomplex(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            Path(d, "setup.py").write_text(
+                "from setuptools import setup; setup(install_requires=blarg)"
+            )
+            with self.assertRaises(ValueError):
+                basic_metadata_from_source_checkout(Path(d))
+
     def test_setuptools_extras(self) -> None:
         with tempfile.TemporaryDirectory() as d:
             Path(d, "setup.cfg").write_text(