Moving to new rules_jvm_external lock file format (#149)

kriscfoster · web-flow · commit 52075cd96c2a · 2023-10-04T06:35:30.000+01:00
* pomgen changes

* tests passing

* re-work exclusions

* some general clean up

* pomgen

* update comment

* full transitive closure

* test for bazel_buildable
diff --git a/crawl/bazel.py b/crawl/bazel.py
@@ -92,6 +92,8 @@ def query_all_libraries(repository_root_path, packages, verbose=False):
                 package = os.path.dirname(package)
     return sorted(lib_roots)
 
+def _normalize_dependency_string_to_group_id_artifact_id(dependency):
+    return ":".join(dependency.split(":")[:2]) # remove classifier/package, so we can grab version from artifacts list
 
 def parse_maven_install(mvn_install_name, json_file_path):
     """
@@ -108,34 +110,55 @@ def parse_maven_install(mvn_install_name, json_file_path):
     with open(json_file_path, "r") as f:
         content = f.read()
         install_json = json.loads(content)
-        json_dep_tree = install_json["dependency_tree"]
-        conflict_resolution = _parse_conflict_resolution(json_dep_tree, mvn_install_name)
-        json_deps = json_dep_tree["dependencies"]
-        for json_dep in json_deps:
-            coord = json_dep["coord"]
+
+        artifacts = install_json["artifacts"]
+        deps = install_json["dependencies"]
+        repository_key = list(install_json["repositories"].keys())[0]
+        all_artifacts = install_json["repositories"][repository_key]
+
+        conflict_resolution = _parse_conflict_resolution(install_json, mvn_install_name)
+
+        for json_dep in all_artifacts:
+            group_id_artifact_id = _normalize_dependency_string_to_group_id_artifact_id(json_dep)
+            version = artifacts[group_id_artifact_id]["version"]
+            coord = json_dep + ":" + version
             dep = dependency.new_dep_from_maven_art_str(coord, mvn_install_name)
             if dep in conflict_resolution:
                 dep = conflict_resolution[dep]
             if dep.classifier != "sources":
                 transitives = []
-                for transitive_gav in json_dep["dependencies"]:
-                    transitive_dep = dependency.new_dep_from_maven_art_str(transitive_gav, mvn_install_name)
-                    if transitive_dep in conflict_resolution:
-                        transitive_dep = conflict_resolution[transitive_dep]
-                    transitives.append(transitive_dep)
-                # exclusions only specify group_id:artifact_id - we use
-                # dependency.Dependency instances instead of raw strings for
-                # consistency, but then we need to add a dummy version
-                if "exclusions" in json_dep:
-                    exclusions = [dependency.new_dep_from_maven_art_str(
-                        "%s:%s" % (d, dependency.GA_DUMMY_DEP_VERSION),
-                        mvn_install_name) for d in json_dep["exclusions"]]
-                else:
-                    exclusions = ()
+                transitives_to_process = []
+                if group_id_artifact_id in deps:
+                    for transitive_gav in deps[group_id_artifact_id]:
+                        transitive_group_id_artifact_id = _normalize_dependency_string_to_group_id_artifact_id(transitive_gav)
+                        version = artifacts[transitive_group_id_artifact_id]["version"]
+                        transitive_gav = transitive_gav + ":" + version
+                        transitive_dep = dependency.new_dep_from_maven_art_str(transitive_gav, mvn_install_name)
+                        if transitive_dep in conflict_resolution:
+                            transitive_dep = conflict_resolution[transitive_dep]
+                        transitives.append(transitive_dep)
+                        # We need to process the transitives dependencies too to get the full transitive closure.
+                        transitives_to_process.append(transitive_gav)
+                    # Process dependencies of transitives in the same way as we do above.
+                    # This is done recursively so if we encounter a transitive that we haven't processed yet, append it to be processed.
+                    for transitive_to_process in transitives_to_process:
+                        transitive_group_id_artifact_id = _normalize_dependency_string_to_group_id_artifact_id(transitive_to_process)
+                        if transitive_group_id_artifact_id in deps:
+                            for transitive_gav in deps[transitive_group_id_artifact_id]:
+                                transitive_group_id_artifact_id = _normalize_dependency_string_to_group_id_artifact_id(transitive_gav)
+                                version = artifacts[transitive_group_id_artifact_id]["version"]
+                                transitive_gav = transitive_gav + ":" + version
+                                transitive_dep = dependency.new_dep_from_maven_art_str(transitive_gav, mvn_install_name)
+                                if transitive_dep in conflict_resolution:
+                                    transitive_dep = conflict_resolution[transitive_dep]
+                                if transitive_dep not in transitives:
+                                    transitives.append(transitive_dep)
+                                if transitive_gav not in transitives_to_process:
+                                    transitives_to_process.append(transitive_gav)
+                exclusions = ()
                 result.append((dep, transitives, exclusions))
     return result
 
-
 def target_pattern_to_path(target_pattern):
     """
     Converts a bazel target pattern to a directory path.
diff --git a/crawl/pom.py b/crawl/pom.py
@@ -578,11 +578,11 @@ def _gen_dependencies_xml(self, pomcontenttype, dependencies, indent):
         for dep in dependencies:
             content, indent = self._gen_dependency_element(pomcontenttype, dep, content, indent, close_element=False)
             # handle <exclusions>
-            excluded_group_and_artifact_ids = [(d.group_id, d.artifact_id) for d in self._workspace.dependency_metadata.get_transitive_exclusions(dep)]
-            excluded_group_and_artifact_ids += self._get_hardcoded_exclusions_for_dep(dep)
-            if len(excluded_group_and_artifact_ids) > 0:
+            # if a dep is built in the shared-repo, do not add any exclusions, they will do that themselves.
+            if not dep.bazel_buildable:
+                # exclude all transitives from <dependencies> as all transitives are already root level anyway
+                excluded_group_and_artifact_ids = [("*", "*")]
                 content, indent = self._gen_exclusions(content, indent, excluded_group_and_artifact_ids)
-
             content, indent = self._xml(content, "dependency", indent, close_element=True)
         return content
 
@@ -609,21 +609,6 @@ def _get_transitive_deps(self, dependencies):
 
         return transitives
 
-    def _get_hardcoded_exclusions_for_dep(self, dep):
-        """
-        A few jar artifacts reference dependencies that do not exist; these 
-        need to be excluded explicitly.
-
-        Returns tuples (group_id, artifact_id) to exclude.
-        """
-        if dep.group_id == "com.twitter.common.zookeeper" and \
-           dep.artifact_id in ("client", "group", "server-set"):
-            return (("com.twitter", "finagle-core-java"),
-                    ("com.twitter", "util-core-java"),
-                    ("org.apache.zookeeper", "zookeeper-client"))
-
-        return ()
-
 
 class DependencyManagementPomGen(AbstractPomGen):
     """
diff --git a/crawl/pomparser.py b/crawl/pomparser.py
@@ -80,9 +80,8 @@ def __init__(self, dependencies=set(), dependency_to_exclusions=defaultdict(list
 
     def get_parsed_exclusions_for(self, dependency):
         """
-        Returns the exclusions for the specified dependency that were parsed 
+        Returns the exclusions for the specified dependency that were parsed
         out of a pom template.
-
         The exclusions are returned as a list of Dependency instances.
         """
         parsed_dep = self.get_parsed_dependency_for(dependency)
diff --git a/misc/tests/extdeps_pomgentest.py b/misc/tests/extdeps_pomgentest.py
@@ -46,55 +46,35 @@ def _add_WORKSPACE_file(self):
     def _add_maven_install_json_file(self):
         content = """
 {
-    "dependency_tree": {
-        "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": -1063907353,
-        "conflict_resolution": {},
-        "dependencies": [
-            {
-                "coord": "com.google.guava:guava:23.0",
-                "dependencies": [],
-                "directDependencies": [],
-                "exclusions": [
-                    "*:*"
-                ],
-                "file": "v1/https/repo1.maven.org/maven2/com/google/guava/guava/23.0/guava-23.0.jar",
-                "mirror_urls": [
-                    "https://repo1.maven.org/maven2/com/google/guava/guava/23.0/guava-23.0.jar"
-                ],
-                "sha256": "7baa80df284117e5b945b19b98d367a85ea7b7801bd358ff657946c3bd1b6596",
-                "url": "https://repo1.maven.org/maven2/com/google/guava/guava/23.0/guava-23.0.jar"
+    "artifacts": {
+        "com.google.guava:guava": {
+            "shasums": {
+                "jar": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac"
             },
-            {
-                "coord": "org.apache.commons:commons-lang3:3.9",
-                "dependencies": [],
-                "directDependencies": [],
-                "exclusions": [
-                    "*:*"
-                ],
-                "file": "v1/https/repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar",
-                "mirror_urls": [
-                    "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar"
-                ],
-                "sha256": "de2e1dcdcf3ef917a8ce858661a06726a9a944f28e33ad7f9e08bea44dc3c230",
-                "url": "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar"
+            "version": "23.0"
+        },
+        "org.apache.commons:commons-lang3": {
+            "shasums": {
+                "jar": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac"
             },
-            {
-                "coord": "org.apache.commons:commons-math3:3.6.1",
-                "dependencies": [],
-                "directDependencies": [],
-                "exclusions": [
-                    "*:*"
-                ],
-                "file": "v1/https/repo1.maven.org/maven2/org/apache/commons/commons-math3/3.6.1/commons-math3-3.6.1.jar",
-                "mirror_urls": [
-                    "https://repo1.maven.org/maven2/org/apache/commons/commons-math3/3.6.1/commons-math3-3.6.1.jar"
-                ],
-                "sha256": "1e56d7b058d28b65abd256b8458e3885b674c1d588fa43cd7d1cbb9c7ef2b308",
-                "url": "https://repo1.maven.org/maven2/org/apache/commons/commons-math3/3.6.1/commons-math3-3.6.1.jar"
-            }
-        ],
-        "version": "0.1.0"
-    }
+            "version": "3.9"
+        },
+        "org.apache.commons:commons-math3": {
+            "shasums": {
+                "jar": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac"
+            },
+            "version": "3.6.1"
+        }
+    },
+    "dependencies": {},
+    "repositories": {
+        "https://maven.google.com/": [
+            "com.google.guava:guava",
+            "org.apache.commons:commons-lang3",
+            "org.apache.commons:commons-math3"
+        ]
+    },
+    "version": "2"
 }
         """
         self._write_file("", "", "maven_install.json", content)
diff --git a/tests/bazeltest.py b/tests/bazeltest.py
@@ -20,18 +20,12 @@ def test_parse_maven_install(self):
             f.write(MVN_INSTALL_JSON_CONTENT)
 
         result = bazel.parse_maven_install("maven", path)
-
-        self.assertEqual(1, len(result))
+        self.assertEqual(3, len(result))
         dep, transitives, exclusions = result[0]
         self.assertEqual(dependency.new_dep_from_maven_art_str("ch.qos.logback:logback-classic:1.2.3", "maven"), dep)
         self.assertEqual(2, len(transitives))
         self.assertEqual(dependency.new_dep_from_maven_art_str("ch.qos.logback:logback-core:1.2.3", "maven"), transitives[0])
         self.assertEqual(dependency.new_dep_from_maven_art_str("org.slf4j:slf4j-api:jar:1.7.30", "maven"), transitives[1])
-        self.assertEqual(2, len(exclusions))
-        self.assertEqual("jakarta.el", exclusions[0].group_id)
-        self.assertEqual("jakarta.el-api", exclusions[0].artifact_id)
-        self.assertEqual("org.glassfish", exclusions[1].group_id)
-        self.assertEqual("jakarta.el", exclusions[1].artifact_id)
 
     def test_conflict_resolution_is_honored(self):
         """
@@ -76,72 +70,76 @@ def test_ensure_unique_deps(self):
 
 MVN_INSTALL_JSON_CONTENT = """
 {
-    "dependency_tree": {
-        "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": -163681522,
-        "conflict_resolution": {},
-        "dependencies": [
-            {
-                "coord": "ch.qos.logback:logback-classic:1.2.3",
-                "dependencies": [
-                    "ch.qos.logback:logback-core:1.2.3",
-                    "org.slf4j:slf4j-api:jar:1.7.30"
-                ],
-                "directDependencies": [
-                    "org.slf4j:slf4j-api:1.7.30"
-                ],
-                "exclusions": [
-                    "jakarta.el:jakarta.el-api",
-                    "org.glassfish:jakarta.el"
-                ]
+    "artifacts": {
+        "ch.qos.logback:logback-classic": {
+            "shasums": {
+                "jar": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac",
+                "sources": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac"
             },
-            {
-                "coord": "ch.qos.logback:logback-classic:jar:sources:1.2.3",
-                "dependencies": [
-                    "org.slf4j:slf4j-api:jar:sources:1.7.30",
-                    "ch.qos.logback:logback-core:jar:sources:1.2.3"
-                ],
-                "directDependencies": [
-                    "ch.qos.logback:logback-core:jar:sources:1.2.3",
-                    "org.slf4j:slf4j-api:jar:sources:1.7.30"
-                ],
-                "exclusions": [
-                ]
-
-            }
-        ],
-        "version": "0.1.0"
-    }
+            "version": "1.2.3"
+        },
+        "ch.qos.logback:logback-core": {
+            "shasums": {
+                "jar": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac"
+            },
+            "version": "1.2.3"
+        },
+        "org.slf4j:slf4j-api": {
+            "shasums": {
+                "jar": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac"
+            },
+            "version": "1.7.30"
+        }
+    },
+    "dependencies": {
+        "ch.qos.logback:logback-classic": [
+            "ch.qos.logback:logback-core",
+            "org.slf4j:slf4j-api"
+        ]
+    },
+    "repositories": {
+        "https://maven.google.com/": [
+            "ch.qos.logback:logback-classic",
+            "ch.qos.logback:logback-classic:jar:sources",
+            "ch.qos.logback:logback-core",
+            "org.slf4j:slf4j-api"
+        ]
+    },
+    "version": "2"
 }
 """
 
 MVN_INSTALL_JSON_CONTENT_CONFLICT_RESOLUTION = """
 {
-    "dependency_tree": {
-        "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": -163681522,
-        "conflict_resolution": {
-            "com.google.guava:guava:31.0.1-jre": "com.google.guava:guava:31.0.1-jre-SNAPSHOT"
+    "conflict_resolution": {
+        "com.google.guava:guava:31.0.1-jre": "com.google.guava:guava:31.0.1-jre-SNAPSHOT"
+    },
+    "artifacts": {
+        "ch.qos.logback:logback-classic": {
+            "shasums": {
+                "jar": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac"
+            },
+            "version": "1.2.3"
         },
-        "dependencies": [
-            {
-                "coord": "ch.qos.logback:logback-classic:1.2.3",
-                "dependencies": [
-                    "com.google.guava:guava:31.0.1-jre-SNAPSHOT"
-                ],
-                "directDependencies": [
-                    "com.google.guava:guava:31.0.1-jre-SNAPSHOT"
-                ]
+        "com.google.guava:guava": {
+            "shasums": {
+                "jar": "ef95ae468097f378880be69a8c6756f8d15180e0f07547fb0a99617ff421b2ac"
             },
-            {
-                "coord": "com.google.guava:guava:31.0.1-jre-SNAPSHOT",
-                "dependencies": [],
-                "directDependencies": [],
-                "exclusions": [
-                    "*:*"
-                ]
-            }
-        ],
-        "version": "0.1.0"
-    }
+            "version": "31.0.1-jre-SNAPSHOT"
+        }
+    },
+    "dependencies": {
+        "ch.qos.logback:logback-classic": [
+            "com.google.guava:guava"
+        ]
+    },
+    "repositories": {
+        "https://maven.google.com/": [
+            "ch.qos.logback:logback-classic",
+            "com.google.guava:guava"
+        ]
+    },
+    "version": "2"
 }
 """
 
diff --git a/tests/pomgentest.py b/tests/pomgentest.py
@@ -43,8 +43,10 @@ def _setup_workspace(self):
         self._add_pom_template()
         self._write_file("","","maven_install.json", """
 {
-    "dependency_tree": {
-        "dependencies": []
+    "artifacts": {},
+    "dependencies": {},
+    "repositories": {
+        "https://maven.google.com/": []
     }
 }
 """)
diff --git a/tests/pomtest.py b/tests/pomtest.py

Original file line number	Diff line number	Diff line change
`@@ -43,8 +43,10 @@ def _setup_workspace(self):`
`43`	`43`	`self._add_pom_template()`
`44`	`44`	`self._write_file("","","maven_install.json", """`
`45`	`45`	`{`
`46`		`- "dependency_tree": {`
`47`		`- "dependencies": []`
	`46`	`+ "artifacts": {},`
	`47`	`+ "dependencies": {},`
	`48`	`+ "repositories": {`
	`49`	`+ "https://maven.google.com/": []`
`48`	`50`	`}`
`49`	`51`	`}`
`50`	`52`	`""")`