Support local version of Transmogrifier

Why these changes are being introduced:

Formerly, only git commit SHAs from the remote Transmogrifier github repository
could be used when initializing a job.  Sometimes during development on Transmogrifier,
it would be helpful to be able to run this application on local commits (potentially ones
that will never see the Github light of day).

How this addresses that need:
* Adds new CLI arguments --location-a and --location-b that define where Transmogrifier
should be cloned from, supporting local paths

Side effects of this change:
* None

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/TIMX-401

Author: ghukill

README.md

@@ -171,8 +171,10 @@ Usage: -c init-job [OPTIONS]
 Options:
   -d, --job-directory TEXT  Job directory to create.  [required]
   -m, --message TEXT        Message to describe Job.
+  -la, --location-a TEXT    Location to clone Transmogrifier version 'A'
   -a, --commit-sha-a TEXT   Transmogrifier commit SHA for version 'A'
                             [required]
+  -lb, --location-b TEXT    Location to clone Transmogrifier version 'B'
   -b, --commit-sha-b TEXT   Transmogrifier commit SHA for version 'B'
                             [required]
   -h, --help                Show this message and exit.

abdiff/cli.py

@@ -88,13 +88,29 @@ def ping() -> None:
     help="Message to describe Job.",
     default="Not provided.",
 )
+@click.option(
+    "-la",
+    "--location-a",
+    type=str,
+    required=False,
+    default="https://github.com/MITLibraries/transmogrifier.git",
+    help="Location to clone Transmogrifier version 'A'",
+)
 @click.option(
     "-a",
     "--commit-sha-a",
     type=str,
     required=True,
     help="Transmogrifier commit SHA for version 'A'",
 )
+@click.option(
+    "-lb",
+    "--location-b",
+    type=str,
+    required=False,
+    default="https://github.com/MITLibraries/transmogrifier.git",
+    help="Location to clone Transmogrifier version 'B'",
+)
 @click.option(
     "-b",
     "--commit-sha-b",
@@ -106,7 +122,9 @@ def init_job(
     job_directory: str,
     message: str,
     commit_sha_a: str,
+    location_a: str,
     commit_sha_b: str,
+    location_b: str,
 ) -> None:
     """Initialize a new Job."""
     try:
@@ -119,7 +137,9 @@ def init_job(
 
     build_ab_images(
         job_directory,
+        location_a,
         commit_sha_a,
+        location_b,
         commit_sha_b,
     )
 

abdiff/core/build_ab_images.py

@@ -15,30 +15,34 @@
 
 def build_ab_images(
     job_directory: str,
+    location_a: str,
     commit_sha_a: str,
+    location_b: str,
     commit_sha_b: str,
     docker_client: docker.client.DockerClient | None = None,
 ) -> tuple[str, str]:
     """Build Docker images based on 2 commit SHAs.
 
     Args:
         job_directory: The directory containing all files related to a job.
+        location_a: Location of Transmogrifier version 'A' to clone from.
         commit_sha_a: The SHA of the first commit for comparison.
+        location_b: Location of Transmogrifier version 'B' to clone from.
         commit_sha_b: The SHA of the second commit for comparison.
         docker_client: A configured Docker client.
     """
     if not docker_client:
         docker_client = docker.from_env()
 
     image_tags = []
-    for commit_sha in [commit_sha_a, commit_sha_b]:
+    for location, commit_sha in [(location_a, commit_sha_a), (location_b, commit_sha_b)]:
         logger.debug(f"Processing commit: {commit_sha}")
         image_tag = generate_image_name(commit_sha)
         if docker_image_exists(docker_client, image_tag):
             logger.debug(f"Docker image already exists with tag: {image_tag}")
             image_tags.append(image_tag)
         else:
-            image = build_image(commit_sha, docker_client)
+            image = build_image(location, commit_sha, docker_client)
             image_tags.append(image.tags[0])
         logger.debug(f"Finished processing commit: {commit_sha}")
 
@@ -67,43 +71,44 @@ def docker_image_exists(
 
 
 def build_image(
+    location: str,
     commit_sha: str,
     docker_client: docker.client.DockerClient,
 ) -> docker.models.images.Image:
     """Clone repo and build Docker image.
 
     Args:
-        job_directory: The directory containing all files related to a job.
+        location: Location of Transmogrifier to clone from.
         commit_sha: The SHA of the commit.
         docker_client: A configured Docker client.
     """
     with tempfile.TemporaryDirectory() as clone_directory:
         image_tag = generate_image_name(commit_sha)
-        clone_repo_and_reset_to_commit(clone_directory, commit_sha)
+        clone_repo_and_reset_to_commit(location, clone_directory, commit_sha)
         image, _ = docker_client.images.build(path=clone_directory, tag=image_tag)
         logger.debug(f"Docker image created with tag: {image}")
         return image
 
 
-def clone_repo_and_reset_to_commit(clone_directory: str, commit_sha: str) -> None:
+def clone_repo_and_reset_to_commit(
+    location: str,
+    clone_directory: str,
+    commit_sha: str,
+) -> None:
     """Clone GitHub repo and reset to a specified commit.
 
     Args:
+        location: Location of Transmogrifier to clone from.
         clone_directory: The directory for the cloned repo.
         commit_sha: The SHA of a repo commit.
     """
-    logger.debug(f"Cloning repo to: {clone_directory}")
-    transmogrifier_url = "https://github.com/MITLibraries/transmogrifier.git"
+    logger.debug(f"Cloning repo from: {location}, to: {clone_directory}")
     repository = clone_repository(
-        transmogrifier_url,
+        location,
         clone_directory,
     )
-    logger.debug(f"Cloned repo to: {clone_directory}")
-
     try:
         repository.reset(commit_sha, ResetMode.HARD)
         logger.debug(f"Cloned repo reset to commit: {commit_sha}")
     except KeyError as exception:
-        raise InvalidRepositoryCommitSHAError(
-            transmogrifier_url, commit_sha
-        ) from exception
+        raise InvalidRepositoryCommitSHAError(location, commit_sha) from exception

tests/conftest.py

@@ -152,6 +152,11 @@ def job_directory(tmp_path):
     return str(tmp_path / "example-job-1")
 
 
+@pytest.fixture
+def default_transmogrifier_location():
+    return "https://github.com/MITLibraries/transmogrifier.git"
+
+
 @pytest.fixture
 def example_job_directory(tmp_path):
     """Copy example job from fixtures to tmp path where it will be modified during test"""

tests/test_build_ab_images.py

@@ -14,13 +14,17 @@
 
 
 @patch("abdiff.core.build_ab_images.clone_repository")
-def test_build_ab_images_success(mocked_clone, job_directory, mocked_docker_client):
+def test_build_ab_images_success(
+    mocked_clone, job_directory, mocked_docker_client, default_transmogrifier_location
+):
     side_effect = os.makedirs(job_directory + "/clone")
     mocked_clone.side_effect = side_effect
 
     images = build_ab_images(
         job_directory,
+        default_transmogrifier_location,
         "abc123",
+        default_transmogrifier_location,
         "def456",
         mocked_docker_client,
     )
@@ -34,7 +38,11 @@ def test_build_ab_images_success(mocked_clone, job_directory, mocked_docker_clie
 
 @patch("abdiff.core.build_ab_images.clone_repository")
 def test_build_ab_images_invalid_commit_sha_raise_error(
-    mocked_clone, job_directory, mocked_docker_client, caplog
+    mocked_clone,
+    job_directory,
+    mocked_docker_client,
+    caplog,
+    default_transmogrifier_location,
 ):
     caplog.set_level("DEBUG")
     side_effect = os.makedirs(job_directory + "/clone")
@@ -44,7 +52,9 @@ def test_build_ab_images_invalid_commit_sha_raise_error(
     with pytest.raises(InvalidRepositoryCommitSHAError):
         build_ab_images(
             job_directory,
+            default_transmogrifier_location,
             "invalid",
+            default_transmogrifier_location,
             "def456",
             mocked_docker_client,
         )
@@ -69,20 +79,27 @@ def test_docker_image_exists_returns_false(mocked_docker_client):
 
 
 @patch("abdiff.core.build_ab_images.clone_repository")
-def test_build_image_success(mocked_clone, mocked_docker_client):
+def test_build_image_success(
+    mocked_clone, mocked_docker_client, default_transmogrifier_location
+):
     image = build_image(
+        default_transmogrifier_location,
         "abc123",
         mocked_docker_client,
     )
     assert image.tags[0] == "transmogrifier-example-job-1-abc123:latest"
 
 
 @patch("abdiff.core.build_ab_images.clone_repository")
-def test_clone_repo_and_reset_to_commit_success(mocked_clone, job_directory):
+def test_clone_repo_and_reset_to_commit_success(
+    mocked_clone, job_directory, default_transmogrifier_location
+):
     clone_directory = job_directory + "/clone"
     assert not os.path.exists(clone_directory)
     side_effect = os.makedirs(clone_directory)
     mocked_clone.side_effect = side_effect
 
-    clone_repo_and_reset_to_commit(clone_directory, "abc123")
+    clone_repo_and_reset_to_commit(
+        default_transmogrifier_location, clone_directory, "abc123"
+    )
     assert os.path.exists(clone_directory)