Add python specific logic (#200)

Author: simontoens

BUILD

@@ -15,7 +15,8 @@ py_library(
                  "src/common/*.py",
                  "src/crawl/*.py",
                  "src/generate/*.py",
-                 "src/generate/impl/*.py"]),
+                 "src/generate/impl/*.py",
+                 "src/generate/impl/py/*.py"]),
     data = ["src/config/pom_template.xml"],
     visibility = ["//misc:__pkg__",],
 )
@@ -230,6 +231,15 @@ py_test(
     python_version = python_version,
 )
 
+py_test(
+    name = "requirementsparsertest",
+    srcs = ["tests/generate/impl/py/requirementsparsertest.py"],
+    deps = [":pomgen_lib"],
+    imports = ["src"],
+    size = "small",
+    python_version = python_version,
+)
+
 py_test(
     name = "workspacetest",
     srcs = ["tests/workspacetest.py"],

src/generate/impl/py/dependency.py

@@ -0,0 +1,39 @@
+class Dependency:
+    """
+    Represents a Python package dependency.
+    """
+
+    def __init__(self, name, version, extras=None):
+        self.name = name
+        self.version = version
+        self.extras = tuple(extras) if extras else ()
+
+        self.child_dependencies = []
+
+    def to_pyproject_format(self):
+        """Convert the dependency to pyproject.toml format."""
+        if self.extras:
+            # Format as TOML table with extras
+            extras_str = ', '.join(f'"{extra}"' for extra in self.extras)
+            return f'{{ version = "{self.version}", extras = [{extras_str}] }}'
+        else:
+            # Simple version string for packages without extras
+            return f'"{self.version}"'
+
+    def __eq__(self, other):
+        if not isinstance(other, Dependency):
+            return False
+        return (self.name == other.name and 
+                self.version == other.version and
+                self.extras == other.extras)
+
+    def __hash__(self):
+        return hash((self.name, self.version, self.extras))
+
+    def __str__(self):
+        extras_str = f"[{','.join(self.extras)}]" if self.extras else ""
+        return f"{self.name}{extras_str}{self.version}"
+
+    def __repr__(self):
+        extras_str = f"[{','.join(self.extras)}]" if self.extras else ""
+        return f"Dependency(name='{self.name}{extras_str}', version='{self.version}')"

src/generate/impl/py/requirementsparser.py

@@ -0,0 +1,82 @@
+from .dependency import Dependency
+
+
+class RequirementsParser:
+    """
+    Requirements lock file parser.
+    """
+    def parse_requirements_lock_file(self, content):
+        """
+        Parse requirements lock file content into a list of Dependency
+        instances. Ignores hash information and comments.
+        
+        Args:
+            content: Content of a requirements lock file
+            
+        Returns:
+            List of Dependency instances representing top-level dependencies
+        """
+        dependencies, name_to_dependency, name_to_vias = self._parse_dependencies_and_vias(content)
+        self._attach_dependencies(name_to_dependency, name_to_vias)
+        return dependencies
+
+    def _parse_dependencies_and_vias(self, content):
+        # return values:
+        dependencies = []  # list of Dependeny instances to preserve order
+        name_to_dependency = {}  # dict of dependency name -> Dependency inst
+        name_to_vias = {}  # dict of dependency name -> list of via values
+
+        current_vias = None
+        for line in content.splitlines():
+            line = line.strip()
+            if len(line) == 0:
+                pass
+            elif line.startswith("--"):
+                pass
+            elif line.startswith("#"):
+                line = line[1:].strip()
+                if line.startswith("via"):
+                    assert current_vias is None
+                    current_vias = []
+                    line = line[3:].strip()
+                    if len(line) > 0:
+                        if line.startswith("-r tools"):
+                            # via -r tools/pip/requirements.in
+                            # directly referenced in requirements, no via
+                            pass
+                        else:
+                            current_vias.append(line)
+                else:
+                    if current_vias is None:
+                        pass
+                    else:
+                        current_vias.append(line)
+            else:
+                version_sep = "==" # we should support other comparison binops?
+                version_sep_index = line.index(version_sep)
+                name = line[0:version_sep_index]
+                extras = ()
+                if name.endswith("]"):
+                    extras_start_index = name.index("[")
+                    extras = name[extras_start_index+1:-1].split(",")
+                    name = name[:extras_start_index]
+                space_index = line.find(" ", version_sep_index)
+                version = line[version_sep_index + len(version_sep):space_index]
+                dependency = Dependency(name, version, extras)
+                dependencies.append(dependency)
+                assert name not in name_to_dependency
+                name_to_dependency[name] = dependency
+                assert name not in name_to_vias
+                # current_vias is None if there is no # via comment at all
+                name_to_vias[name] = () if current_vias is None else current_vias
+                current_vias = None
+        return dependencies, name_to_dependency, name_to_vias
+
+    def _attach_dependencies(self, name_to_dependency, name_to_vias):
+        """Attach dependencies based on via relationships."""
+        for name, vias in name_to_vias.items():
+            child_dependency = name_to_dependency[name]
+            for via in vias:
+                parent_dependency = name_to_dependency[via]
+                parent_dependency.child_dependencies.append(child_dependency)
+

src/generate/impl/pygenerationstrategy.py

@@ -0,0 +1,59 @@
+import generate
+
+
+class PyGenerationStrategy(generate.AbstractGenerationStrategy):
+    """
+    Strategy for generating Python package metadat as pyproject.toml.
+
+    [build-system]
+    requires = ["setuptools>=42", "wheel"]
+    build-backend = "setuptools.build_meta"
+
+    [project]
+    name = "hello-world"
+    version = "0.1.0"
+    description = "A sample Python project"
+    readme = "README.md"
+    requires-python = ">=3.8"
+    license = {text = "Apache-2.0"}
+    authors = [
+        {name = "Your Name", email = "[email protected]"}
+    ]
+    classifiers = [
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Developers",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+    ]
+
+    dependencies = [
+        "requests>=2.28.0",
+        "pytest>=7.0.0",
+        "black>=22.0.0",
+    ]
+    """
+
+    def __init__(self, workspace, template):
+        assert workspace is not None
+        assert template is not None
+        self.workspace = workspace
+        self.template = template
+
+    def load_dependency(self, label, artifact_def):
+        """
+        Load a dependency for a Python package.
+        
+        Args:
+            label: The label for the dependency
+            artifact_def: The artifact definition for source dependencies
+            
+        Returns:
+            A dependency instance appropriate for Python packages
+        """
+        pass
+
+    def load_transitive_closure(self, dependency):
+        pass

tests/generate/impl/py/requirementsparsertest.py

@@ -0,0 +1,97 @@
+import unittest
+from generate.impl.py.requirementsparser import RequirementsParser
+
+CONTENT = """
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    bazel run //:requirements.update
+#
+--extra-index-url https://proxy.net/nexus/repository/pypi/
+--trusted-host proxy.net
+
+    # via pydantic
+    # httpx
+ansi==0.3.7 \
+    --hash=sha256:7e59108922259e03c54e4d93fc611bba0e756513086849708b86b6c80f8d4cd4 \
+    --hash=sha256:bdd9e3c2dc3e4c8df8c2b745ca6f07f715aa4edee5ed4a5bcb29065da06a3f71
+    # via cffi
+pydantic==2.10.5 \
+    --hash=sha256:278b38dbbaec562011d659ee05f63346951b3a248a6f3642e1bc68894ea2b4ff \
+    --hash=sha256:4dd4e322dbe55472cb7ca7e73f4b63574eecccf2835ffa2af9021ce113c83c53
+    # via
+    #   httpx
+    #   llama-cloud
+cffi==1.17.1 \
+    --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8
+    --hash=sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b
+    # via uvicorn
+httpx==0.27.2 \
+    --hash=sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0 \
+    --hash=sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2
+uvicorn[standard]==0.34.0 \
+    --hash=sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4 \
+    --hash=sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9
+    # via -r tools/pip/requirements.in
+uvloop==0.21.0 \
+    --hash=sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0
+llama-cloud==0.1.15 \
+    --hash=sha256:3ec98422072ee4290c1e7597bc33889af3f03edb2a50ab09c15d5586d9f8635c \
+    --hash=sha256:8e7376346d580b244a87d023eb7493506f06b53c898fb2a2e712b9fe7eb5b310
+"""
+
+class RequirementsParserTest(unittest.TestCase):
+
+    def setUp(self):
+        self.parser = RequirementsParser()
+
+    def test_parse_requirements_lock_file(self):
+        dependencies = self.parser.parse_requirements_lock_file(CONTENT)
+
+        self.assertEqual(7, len(dependencies))
+        ansi = dependencies[0]
+        self.assertEqual("ansi", ansi.name)
+        self.assertEqual("0.3.7", ansi.version)
+        self.assertEqual(0, len(ansi.extras))
+        pydantic = dependencies[1]
+        self.assertEqual("pydantic", pydantic.name)
+        self.assertEqual("2.10.5", pydantic.version)
+        self.assertEqual(0, len(pydantic.extras))
+        cffi = dependencies[2]
+        self.assertEqual("cffi", cffi.name)
+        self.assertEqual("1.17.1", cffi.version)
+        self.assertEqual(0, len(cffi.extras))
+        httpx = dependencies[3]
+        self.assertEqual("httpx", httpx.name)
+        self.assertEqual("0.27.2", httpx.version)
+        self.assertEqual(0, len(httpx.extras))
+        uvicorn = dependencies[4]
+        self.assertEqual("uvicorn", uvicorn.name)
+        self.assertEqual("0.34.0", uvicorn.version)
+        self.assertEqual("standard", uvicorn.extras[0])
+        uvloop = dependencies[5]
+        self.assertEqual("uvloop", uvloop.name)
+        self.assertEqual("0.21.0", uvloop.version)
+        self.assertEqual(0, len(uvloop.extras))
+        llama_cloud = dependencies[6]
+        self.assertEqual("llama-cloud", llama_cloud.name)
+        self.assertEqual("0.1.15", llama_cloud.version)
+        self.assertEqual(0, len(llama_cloud.extras))
+
+        self.assertEqual(1, len(pydantic.child_dependencies))
+        self.assertIs(pydantic.child_dependencies[0], ansi)
+        self.assertEqual(1, len(cffi.child_dependencies))
+        self.assertIs(cffi.child_dependencies[0], pydantic)
+        self.assertEqual(2, len(httpx.child_dependencies))
+        self.assertIs(httpx.child_dependencies[0], ansi)
+        self.assertIs(httpx.child_dependencies[1], cffi)
+        self.assertEqual(1, len(llama_cloud.child_dependencies))
+        self.assertIs(llama_cloud.child_dependencies[0], cffi)
+        self.assertEqual(1, len(uvicorn.child_dependencies))
+        self.assertIs(uvicorn.child_dependencies[0], httpx)
+        self.assertEqual(0, len(uvloop.child_dependencies))
+
+
+if __name__ == '__main__':
+    unittest.main()