Merge pull request #84 from MITLibraries/TIMX-401-expect-v2-datasets

TIMX 401 - expect and support v2 parquet dataset Transmogrifier output

Author: ghukill

.pre-commit-config.yaml

@@ -24,6 +24,6 @@ repos:
         types: ["python"]
       - id: safety
         name: safety
-        entry: pipenv check --ignore 70612
+        entry: pipenv check
         language: system
         pass_filenames: false

Makefile

@@ -45,7 +45,7 @@ ruff: # Run 'ruff' linter and print a preview of errors
 	pipenv run ruff check .
 
 safety: # Check for security vulnerabilities and verify Pipfile.lock is up-to-date
-	pipenv check --ignore 70612
+	pipenv check
 	pipenv verify
 
 lint-apply: black-apply ruff-apply # Apply changes with 'black' and resolve 'fixable errors' with 'ruff'

Pipfile.lock

@@ -103,13 +103,9 @@ output/mvp
         ├── run.json
         └── transformed
             ├── a
-            │   ├── alma-2023-02-19-daily-transformed-records-to-index.json
-            │   ├── dspace-2024-10-11-daily-transformed-records-to-index.json
-            │   └── libguides-2024-04-03-full-transformed-records-to-index.json
+            │   └── dataset <parquet dataset from Transmogrifier>
             ├── b
-            │   ├── alma-2023-02-19-daily-transformed-records-to-index.json
-            │   ├── dspace-2024-10-11-daily-transformed-records-to-index.json
-            │   └── libguides-2024-04-03-full-transformed-records-to-index.json
+            │   └── dataset <parquet dataset from Transmogrifier>
             └── logs.txt
 ```
 
@@ -175,8 +171,10 @@ Usage: -c init-job [OPTIONS]
 Options:
   -d, --job-directory TEXT  Job directory to create.  [required]
   -m, --message TEXT        Message to describe Job.
+  -la, --location-a TEXT    Location to clone Transmogrifier version 'A'
   -a, --commit-sha-a TEXT   Transmogrifier commit SHA for version 'A'
                             [required]
+  -lb, --location-b TEXT    Location to clone Transmogrifier version 'B'
   -b, --commit-sha-b TEXT   Transmogrifier commit SHA for version 'B'
                             [required]
   -h, --help                Show this message and exit.

README.md

@@ -88,13 +88,29 @@ def ping() -> None:
     help="Message to describe Job.",
     default="Not provided.",
 )
+@click.option(
+    "-la",
+    "--location-a",
+    type=str,
+    required=False,
+    default="https://github.com/MITLibraries/transmogrifier.git",
+    help="Location to clone Transmogrifier version 'A'",
+)
 @click.option(
     "-a",
     "--commit-sha-a",
     type=str,
     required=True,
     help="Transmogrifier commit SHA for version 'A'",
 )
+@click.option(
+    "-lb",
+    "--location-b",
+    type=str,
+    required=False,
+    default="https://github.com/MITLibraries/transmogrifier.git",
+    help="Location to clone Transmogrifier version 'B'",
+)
 @click.option(
     "-b",
     "--commit-sha-b",
@@ -106,7 +122,9 @@ def init_job(
     job_directory: str,
     message: str,
     commit_sha_a: str,
+    location_a: str,
     commit_sha_b: str,
+    location_b: str,
 ) -> None:
     """Initialize a new Job."""
     try:
@@ -119,7 +137,9 @@ def init_job(
 
     build_ab_images(
         job_directory,
+        location_a,
         commit_sha_a,
+        location_b,
         commit_sha_b,
     )
 
@@ -179,7 +199,7 @@ def run_diff(
     if download_files:
         download_input_files(input_files_list)
 
-    ab_transformed_file_lists = run_ab_transforms(
+    ab_transformed_datasets = run_ab_transforms(
         run_directory=run_directory,
         image_tag_a=job_data["image_tag_a"],
         image_tag_b=job_data["image_tag_b"],
@@ -189,7 +209,7 @@ def run_diff(
 
     collated_dataset_path = collate_ab_transforms(
         run_directory=run_directory,
-        ab_transformed_file_lists=ab_transformed_file_lists,
+        ab_transformed_datasets=ab_transformed_datasets,
     )
 
     diffs_dataset_path = calc_ab_diffs(

abdiff/cli.py

@@ -14,13 +14,13 @@
 from abdiff.extras.minio.download_input_files import download_input_files
 
 __all__ = [
-    "init_job",
-    "init_run",
     "build_ab_images",
-    "download_input_files",
-    "run_ab_transforms",
-    "collate_ab_transforms",
     "calc_ab_diffs",
     "calc_ab_metrics",
+    "collate_ab_transforms",
     "create_final_records",
+    "download_input_files",
+    "init_job",
+    "init_run",
+    "run_ab_transforms",
 ]

abdiff/core/__init__.py

@@ -15,30 +15,34 @@
 
 def build_ab_images(
     job_directory: str,
+    location_a: str,
     commit_sha_a: str,
+    location_b: str,
     commit_sha_b: str,
     docker_client: docker.client.DockerClient | None = None,
 ) -> tuple[str, str]:
     """Build Docker images based on 2 commit SHAs.
 
     Args:
         job_directory: The directory containing all files related to a job.
+        location_a: Location of Transmogrifier version 'A' to clone from.
         commit_sha_a: The SHA of the first commit for comparison.
+        location_b: Location of Transmogrifier version 'B' to clone from.
         commit_sha_b: The SHA of the second commit for comparison.
         docker_client: A configured Docker client.
     """
     if not docker_client:
         docker_client = docker.from_env()
 
     image_tags = []
-    for commit_sha in [commit_sha_a, commit_sha_b]:
+    for location, commit_sha in [(location_a, commit_sha_a), (location_b, commit_sha_b)]:
         logger.debug(f"Processing commit: {commit_sha}")
         image_tag = generate_image_name(commit_sha)
         if docker_image_exists(docker_client, image_tag):
             logger.debug(f"Docker image already exists with tag: {image_tag}")
             image_tags.append(image_tag)
         else:
-            image = build_image(commit_sha, docker_client)
+            image = build_image(location, commit_sha, docker_client)
             image_tags.append(image.tags[0])
         logger.debug(f"Finished processing commit: {commit_sha}")
 
@@ -67,43 +71,44 @@ def docker_image_exists(
 
 
 def build_image(
+    location: str,
     commit_sha: str,
     docker_client: docker.client.DockerClient,
 ) -> docker.models.images.Image:
     """Clone repo and build Docker image.
 
     Args:
-        job_directory: The directory containing all files related to a job.
+        location: Location of Transmogrifier to clone from.
         commit_sha: The SHA of the commit.
         docker_client: A configured Docker client.
     """
     with tempfile.TemporaryDirectory() as clone_directory:
         image_tag = generate_image_name(commit_sha)
-        clone_repo_and_reset_to_commit(clone_directory, commit_sha)
+        clone_repo_and_reset_to_commit(location, clone_directory, commit_sha)
         image, _ = docker_client.images.build(path=clone_directory, tag=image_tag)
         logger.debug(f"Docker image created with tag: {image}")
         return image
 
 
-def clone_repo_and_reset_to_commit(clone_directory: str, commit_sha: str) -> None:
+def clone_repo_and_reset_to_commit(
+    location: str,
+    clone_directory: str,
+    commit_sha: str,
+) -> None:
     """Clone GitHub repo and reset to a specified commit.
 
     Args:
+        location: Location of Transmogrifier to clone from.
         clone_directory: The directory for the cloned repo.
         commit_sha: The SHA of a repo commit.
     """
-    logger.debug(f"Cloning repo to: {clone_directory}")
-    transmogrifier_url = "https://github.com/MITLibraries/transmogrifier.git"
+    logger.debug(f"Cloning repo from: {location}, to: {clone_directory}")
     repository = clone_repository(
-        transmogrifier_url,
+        location,
         clone_directory,
     )
-    logger.debug(f"Cloned repo to: {clone_directory}")
-
     try:
         repository.reset(commit_sha, ResetMode.HARD)
         logger.debug(f"Cloned repo reset to commit: {commit_sha}")
     except KeyError as exception:
-        raise InvalidRepositoryCommitSHAError(
-            transmogrifier_url, commit_sha
-        ) from exception
+        raise InvalidRepositoryCommitSHAError(location, commit_sha) from exception