From 5aff1b7af2f882ee1adaa8a57c9571e626bd9bd2 Mon Sep 17 00:00:00 2001 From: danimtb Date: Thu, 27 Nov 2025 10:42:40 +0100 Subject: [PATCH 1/5] Include metadata files into build info --- extensions/commands/art/cmd_build_info.py | 12 +++++-- tests/test_build_info.py | 39 +++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index cbe52bf..717b810 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -203,8 +203,10 @@ def _get_local_artifacts(): artifacts_folder = Path(artifacts_folder) dl_folder = artifacts_folder.parents[0] / "d" dl_folder_files = [file for file in dl_folder.glob("*") if file.name in artifacts_names] + metadata_folder = dl_folder / "metadata" + metadata_folder_files = [file for file in metadata_folder.rglob("*") if file.is_file()] artifacts_folder_files = [file for file in artifacts_folder.glob("*") if file.name in artifacts_names] - all_files = dl_folder_files + artifacts_folder_files + all_files = dl_folder_files + metadata_folder_files + artifacts_folder_files processed_files = set() @@ -223,7 +225,13 @@ def _get_local_artifacts(): node.get('ref')) if artifact_type == "recipe" else _get_remote_path(node.get('ref'), node.get("package_id"), node.get("prev")) - artifact_info.update({"name": file_name, "path": f'{origin_repo}/{remote_path}/{file_name}'}) + if "metadata" in file_path.parts: + idx = file_path.parts.index("metadata") + ending_path = Path(*file_path.parts[idx + 1:]) + artifactory_path = f'{origin_repo}/{remote_path}/metadata/{ending_path.as_posix()}' + else: + artifactory_path = f'{origin_repo}/{remote_path}/{file_name}' + artifact_info.update({"name": file_name, "path": artifactory_path}) else: ref = node.get("ref") pkg = f":{node.get('package_id')}#{node.get('prev')}" if artifact_type == "package" else "" diff --git a/tests/test_build_info.py b/tests/test_build_info.py index 84d7603..15f86d9 100644 --- a/tests/test_build_info.py +++ b/tests/test_build_info.py @@ -157,3 +157,42 @@ def package_info(self): # Check libb package now has the meson dependency build_info["modules"][3]["dependencies"] assert len(build_info["modules"][3]["dependencies"]) == 2 + + +def test_build_info_with_metadata_files(): + """ + Test that metadata files are added to the build info + """ + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import save + + class Meson(ConanFile): + name = "meson" + version = 1.0 + package_type = "application" + + def export(self): + save(self, os.path.join(self.recipe_metadata_folder, "logs", "extra_info.txt"), "some info") + + def build(self): + save(self, os.path.join(self.package_metadata_folder, "logs", "build.log"), "srclog!!") + """) + save(os.path.join(os.curdir, "conanfile.py"), conanfile) + run("conan create . -f json > create.json") + + graph = json.loads(load("create.json"))["graph"] + _fake_conan_sources(graph) + + run("conan art:build-info create create.json build_name 1 repo > bi.json") + build_info = json.loads(load("bi.json")) + # recipe module + assert build_info['modules'][0]['artifacts'][1]['name'] == "extra_info.txt" + assert build_info['modules'][0]['artifacts'][1]['path'] == \ + "repo/_/meson/1.0/_/4fd8594b139818338a93342fdf2bf404/export/metadata/logs/extra_info.txt" + # package module + assert build_info['modules'][1]['artifacts'][0]['name'] == "build.log" + assert build_info['modules'][1]['artifacts'][0]['path'] == \ + "repo/_/meson/1.0/_/4fd8594b139818338a93342fdf2bf404/package/da39a3ee5e6b4b0d3255bfef95601890afd80709/" \ + "0ba8627bd47edc3a501e8f0eb9a79e5e/metadata/logs/build.log" From 14e94c6b62547dd3c30a7f6d4cc1202bc6164a14 Mon Sep 17 00:00:00 2001 From: danimtb Date: Thu, 27 Nov 2025 11:06:48 +0100 Subject: [PATCH 2/5] test --- tests/test_build_info.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_build_info.py b/tests/test_build_info.py index 15f86d9..a7325cb 100644 --- a/tests/test_build_info.py +++ b/tests/test_build_info.py @@ -168,10 +168,9 @@ def test_build_info_with_metadata_files(): from conan import ConanFile from conan.tools.files import save - class Meson(ConanFile): - name = "meson" + class Recipe(ConanFile): + name = "pkg-w-metadata" version = 1.0 - package_type = "application" def export(self): save(self, os.path.join(self.recipe_metadata_folder, "logs", "extra_info.txt"), "some info") @@ -185,14 +184,14 @@ def build(self): graph = json.loads(load("create.json"))["graph"] _fake_conan_sources(graph) - run("conan art:build-info create create.json build_name 1 repo > bi.json") + run("conan art:build-info create create.json build_name 1 danimtb-local > bi.json") build_info = json.loads(load("bi.json")) # recipe module assert build_info['modules'][0]['artifacts'][1]['name'] == "extra_info.txt" assert build_info['modules'][0]['artifacts'][1]['path'] == \ - "repo/_/meson/1.0/_/4fd8594b139818338a93342fdf2bf404/export/metadata/logs/extra_info.txt" + "danimtb-local/_/pkg-w-metadata/1.0/_/35152d357d762ebd841b9f857a8aeb25/export/metadata/logs/extra_info.txt" # package module assert build_info['modules'][1]['artifacts'][0]['name'] == "build.log" assert build_info['modules'][1]['artifacts'][0]['path'] == \ - "repo/_/meson/1.0/_/4fd8594b139818338a93342fdf2bf404/package/da39a3ee5e6b4b0d3255bfef95601890afd80709/" \ - "0ba8627bd47edc3a501e8f0eb9a79e5e/metadata/logs/build.log" + "danimtb-local/_/pkg-w-metadata/1.0/_/35152d357d762ebd841b9f857a8aeb25/package/" \ + "da39a3ee5e6b4b0d3255bfef95601890afd80709/0ba8627bd47edc3a501e8f0eb9a79e5e/metadata/logs/build.log" From 8fcc8d81c071e0471272943e3a95ae9f90c1585c Mon Sep 17 00:00:00 2001 From: danimtb Date: Thu, 27 Nov 2025 11:34:41 +0100 Subject: [PATCH 3/5] fix test --- tests/test_build_info.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_build_info.py b/tests/test_build_info.py index a7325cb..8bcd10d 100644 --- a/tests/test_build_info.py +++ b/tests/test_build_info.py @@ -188,10 +188,7 @@ def build(self): build_info = json.loads(load("bi.json")) # recipe module assert build_info['modules'][0]['artifacts'][1]['name'] == "extra_info.txt" - assert build_info['modules'][0]['artifacts'][1]['path'] == \ - "danimtb-local/_/pkg-w-metadata/1.0/_/35152d357d762ebd841b9f857a8aeb25/export/metadata/logs/extra_info.txt" + assert "/export/metadata/logs/extra_info.txt" in build_info['modules'][0]['artifacts'][1]['path'] # package module assert build_info['modules'][1]['artifacts'][0]['name'] == "build.log" - assert build_info['modules'][1]['artifacts'][0]['path'] == \ - "danimtb-local/_/pkg-w-metadata/1.0/_/35152d357d762ebd841b9f857a8aeb25/package/" \ - "da39a3ee5e6b4b0d3255bfef95601890afd80709/0ba8627bd47edc3a501e8f0eb9a79e5e/metadata/logs/build.log" + assert "/metadata/logs/build.log" in build_info['modules'][1]['artifacts'][0]['path'] From 2962576bc04e4b3ca027748ae58f738554264f3c Mon Sep 17 00:00:00 2001 From: danimtb Date: Wed, 31 Dec 2025 17:00:17 +0100 Subject: [PATCH 4/5] fix --- tests/test_build_info.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_build_info.py b/tests/test_build_info.py index 7bb0bd7..08d1c50 100644 --- a/tests/test_build_info.py +++ b/tests/test_build_info.py @@ -181,9 +181,6 @@ def build(self): save(self, os.path.join(self.package_metadata_folder, "logs", "build.log"), "srclog!!") """) save(os.path.join(os.curdir, "conanfile.py"), conanfile) -def test_formatted_time(): - """Compare local timestamp hours from build-info JSON with current timestamp in UTC""" - run("conan new cmake_lib -d name=lib1 -d version=1.0") run("conan create . -f json > create.json") graph = json.loads(load("create.json"))["graph"] @@ -199,6 +196,16 @@ def test_formatted_time(): assert "/metadata/logs/build.log" in build_info['modules'][1]['artifacts'][0]['path'] run("conan art:build-info create create.json build_name 1 repo --with-dependencies > bi.json") + +def test_formatted_time(): + """Compare local timestamp hours from build-info JSON with current timestamp in UTC""" + run("conan new cmake_lib -d name=lib1 -d version=1.0") + run("conan create . -f json > create.json") + + graph = json.loads(load("create.json"))["graph"] + _fake_conan_sources(graph) + + run("conan art:build-info create create.json build_name 1 danimtb-local > bi.json") build_info = json.loads(load("bi.json")) timestamp = build_info["started"] From 673a1721e84739746cdd20e8ccf1c16c5c0bf21d Mon Sep 17 00:00:00 2001 From: danimtb Date: Wed, 31 Dec 2025 17:54:33 +0100 Subject: [PATCH 5/5] separate logic --- extensions/commands/art/cmd_build_info.py | 143 +++++++++++++++++----- tests/test_build_info.py | 8 +- 2 files changed, 118 insertions(+), 33 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index 5bba5ad..303ddd0 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -173,6 +173,97 @@ def get_artifacts_folder(self, node, artifact_type): reference = RecipeReference.loads(node.get("ref")) return self._conan_api.cache.export_path(reference) + def _get_metadata_files(self, dl_folder): + """ + Collect all metadata files from the metadata folder. + + Args: + dl_folder: Path to the download folder containing the metadata subfolder + + Returns: + List of Path objects for all metadata files found + """ + metadata_folder = dl_folder / "metadata" + if not metadata_folder.exists(): + return [] + return [file for file in metadata_folder.rglob("*") if file.is_file()] + + def _process_metadata_file(self, file_path, node, artifact_type, origin_repo, is_dependency): + """ + Process a metadata file and create artifact info for it. + + Args: + file_path: Path to the metadata file + node: Node information from the graph + artifact_type: Type of artifact ("recipe" or "package") + origin_repo: Repository where the artifact originates + is_dependency: Whether this is a dependency artifact + + Returns: + Dictionary with artifact information + """ + file_name = file_path.name + md5, sha1, sha256 = _get_hashes(file_path) + artifact_info = { + "type": os.path.splitext(file_name)[1].lstrip('.'), + "sha256": sha256, + "sha1": sha1, + "md5": md5 + } + + if not is_dependency: + remote_path = _get_remote_path( + node.get('ref')) if artifact_type == "recipe" else _get_remote_path( + node.get('ref'), node.get("package_id"), node.get("prev")) + + # Extract the path relative to the metadata folder + idx = file_path.parts.index("metadata") + ending_path = Path(*file_path.parts[idx + 1:]) + artifactory_path = f'{origin_repo}/{remote_path}/metadata/{ending_path.as_posix()}' + artifact_info.update({"name": file_name, "path": artifactory_path}) + else: + ref = node.get("ref") + pkg = f":{node.get('package_id')}#{node.get('prev')}" if artifact_type == "package" else "" + artifact_info.update({"id": f"{ref}{pkg} :: {file_name}"}) + + return artifact_info + + def _process_regular_artifact(self, file_path, node, artifact_type, origin_repo, is_dependency): + """ + Process a regular (non-metadata) artifact file and create artifact info for it. + + Args: + file_path: Path to the artifact file + node: Node information from the graph + artifact_type: Type of artifact ("recipe" or "package") + origin_repo: Repository where the artifact originates + is_dependency: Whether this is a dependency artifact + + Returns: + Dictionary with artifact information + """ + file_name = file_path.name + md5, sha1, sha256 = _get_hashes(file_path) + artifact_info = { + "type": os.path.splitext(file_name)[1].lstrip('.'), + "sha256": sha256, + "sha1": sha1, + "md5": md5 + } + + if not is_dependency: + remote_path = _get_remote_path( + node.get('ref')) if artifact_type == "recipe" else _get_remote_path( + node.get('ref'), node.get("package_id"), node.get("prev")) + artifactory_path = f'{origin_repo}/{remote_path}/{file_name}' + artifact_info.update({"name": file_name, "path": artifactory_path}) + else: + ref = node.get("ref") + pkg = f":{node.get('package_id')}#{node.get('prev')}" if artifact_type == "package" else "" + artifact_info.update({"id": f"{ref}{pkg} :: {file_name}"}) + + return artifact_info + def get_artifacts(self, node, artifact_type, is_dependency=False): """ Function to get artifact information, those artifacts can be added as artifacts of a @@ -202,41 +293,35 @@ def _get_local_artifacts(): artifacts_folder = Path(artifacts_folder) dl_folder = artifacts_folder.parents[0] / "d" + + # Collect regular artifacts dl_folder_files = [file for file in dl_folder.glob("*") if file.name in artifacts_names] - metadata_folder = dl_folder / "metadata" - metadata_folder_files = [file for file in metadata_folder.rglob("*") if file.is_file()] artifacts_folder_files = [file for file in artifacts_folder.glob("*") if file.name in artifacts_names] - all_files = dl_folder_files + metadata_folder_files + artifacts_folder_files - + regular_files = dl_folder_files + artifacts_folder_files + + # Collect metadata files separately + metadata_files = self._get_metadata_files(dl_folder) + processed_files = set() - - for file_path in all_files: + + # Process regular artifacts + for file_path in regular_files: file_name = file_path.name if file_path.is_file() and file_name not in processed_files: - processed_files.add(file_path.name) - md5, sha1, sha256 = _get_hashes(file_path) - artifact_info = {"type": os.path.splitext(file_name)[1].lstrip('.'), - "sha256": sha256, - "sha1": sha1, - "md5": md5} - - if not is_dependency: - remote_path = _get_remote_path( - node.get('ref')) if artifact_type == "recipe" else _get_remote_path(node.get('ref'), - node.get("package_id"), - node.get("prev")) - if "metadata" in file_path.parts: - idx = file_path.parts.index("metadata") - ending_path = Path(*file_path.parts[idx + 1:]) - artifactory_path = f'{origin_repo}/{remote_path}/metadata/{ending_path.as_posix()}' - else: - artifactory_path = f'{origin_repo}/{remote_path}/{file_name}' - artifact_info.update({"name": file_name, "path": artifactory_path}) - else: - ref = node.get("ref") - pkg = f":{node.get('package_id')}#{node.get('prev')}" if artifact_type == "package" else "" - artifact_info.update({"id": f"{ref}{pkg} :: {file_name}"}) + processed_files.add(file_name) + artifact_info = self._process_regular_artifact( + file_path, node, artifact_type, origin_repo, is_dependency + ) + local_artifacts.append(artifact_info) + # Process metadata files + for file_path in metadata_files: + file_name = file_path.name + if file_name not in processed_files: + processed_files.add(file_name) + artifact_info = self._process_metadata_file( + file_path, node, artifact_type, origin_repo, is_dependency + ) local_artifacts.append(artifact_info) missing_files = set(artifacts_names) - processed_files diff --git a/tests/test_build_info.py b/tests/test_build_info.py index 08d1c50..3c77a49 100644 --- a/tests/test_build_info.py +++ b/tests/test_build_info.py @@ -189,11 +189,11 @@ def build(self): run("conan art:build-info create create.json build_name 1 danimtb-local > bi.json") build_info = json.loads(load("bi.json")) # recipe module - assert build_info['modules'][0]['artifacts'][1]['name'] == "extra_info.txt" - assert "/export/metadata/logs/extra_info.txt" in build_info['modules'][0]['artifacts'][1]['path'] + assert build_info['modules'][0]['artifacts'][3]['name'] == "extra_info.txt" + assert "/export/metadata/logs/extra_info.txt" in build_info['modules'][0]['artifacts'][3]['path'] # package module - assert build_info['modules'][1]['artifacts'][0]['name'] == "build.log" - assert "/metadata/logs/build.log" in build_info['modules'][1]['artifacts'][0]['path'] + assert build_info['modules'][1]['artifacts'][2]['name'] == "build.log" + assert "/metadata/logs/build.log" in build_info['modules'][1]['artifacts'][2]['path'] run("conan art:build-info create create.json build_name 1 repo --with-dependencies > bi.json")