Merge pull request #80 from dbt-labs/feature/non-zero-return-code-on-sync

Author: b-per

README.md

@@ -31,7 +31,7 @@ With this package's approach, people don't need to learn another tool and can co
 ### Installation
 
 - Create a Python virtual environment and activate it
-- Run `pip install dbt-jobs-as-code` (or `pip install dbt-jobs-as-code~=0.6.0` to install a specific release and its patches)
+- Run `pip install dbt-jobs-as-code` (or `pip install dbt-jobs-as-code~=0.7` to install a specific release and its patches)
 
 The CLI is now available as `dbt-jobs-as-code`
 

example_cicd/prod_plan_on_pr.yml

@@ -21,7 +21,7 @@ jobs:
           python-version: "3.12.x"
 
       - name: Install dbt-jobs-as-code
-        run: pip install dbt-jobs-as-code~=0.6.0
+        run: pip install dbt-jobs-as-code~=0.7.0
 
       - name: Run dbt-jobs-as-code
         run: dbt-jobs-as-code plan jobs/jobs.yml --vars-yml jobs/vars_prod.yml --limit-projects-envs-to-yml

example_cicd/prod_sync_on_merge.yml

@@ -24,7 +24,7 @@ jobs:
           python-version: "3.12.x"
 
       - name: Install dbt-jobs-as-code
-        run: pip install dbt-jobs-as-code~=0.6.0
+        run: pip install dbt-jobs-as-code~=0.7.0
 
       - name: Run dbt-jobs-as-code
         run: dbt-jobs-as-code sync jobs/jobs.yml --vars-yml jobs/vars_prod.yml --limit-projects-envs-to-yml

example_cicd/qa_plan_on_pr.yml

@@ -21,7 +21,7 @@ jobs:
           python-version: "3.12.x"
 
       - name: Install dbt-jobs-as-code
-        run: pip install dbt-jobs-as-code~=0.6.0
+        run: pip install dbt-jobs-as-code~=0.7.0
 
       - name: Run dbt-jobs-as-code
         run: dbt-jobs-as-code plan jobs/jobs.yml --vars-yml jobs/vars_qa.yml --limit-projects-envs-to-yml

example_cicd/qa_sync_on_merge.yml

@@ -24,7 +24,7 @@ jobs:
           python-version: "3.12.x"
 
       - name: Install dbt-jobs-as-code
-        run: pip install dbt-jobs-as-code~=0.6.0
+        run: pip install dbt-jobs-as-code~=0.7.0
 
       - name: Run dbt-jobs-as-code
         run: dbt-jobs-as-code sync jobs/jobs.yml --vars-yml jobs/vars_qa.yml --limit-projects-envs-to-yml

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dbt-jobs-as-code"
-version = "0.6.1"
+version = "0.7.0"
 description = "A CLI to allow defining dbt Cloud jobs as code"
 authors = ["dbt Labs <info@dbtlabs.com>"]
 license = "Apache License 2.0"

src/changeset/change_set.py

@@ -8,7 +8,7 @@
 from pydantic import BaseModel, RootModel
 from rich.table import Table
 
-from src.client import DBTCloud
+from src.client import DBTCloud, DBTCloudException
 from src.loader.load import load_job_configuration
 from src.schemas import check_env_var_same, check_job_mapping_same
 from src.schemas.job import JobDefinition
@@ -35,10 +35,11 @@ def apply(self):
         self.sync_function(**self.parameters)
 
 
-class ChangeSet(RootModel):
+class ChangeSet(BaseModel):
     """Store the set of changes to be displayed or applied."""
 
     root: List[Change] = []
+    apply_success: bool = True
 
     def __iter__(self):
         return iter(self.root)
@@ -77,7 +78,10 @@ def __len__(self):
 
     def apply(self):
         for change in self.root:
-            change.apply()
+            try:
+                change.apply()
+            except DBTCloudException:
+                self.apply_success = False
 
 
 # Don't bear type this function as we do some odd things in tests

src/client/__init__.py

@@ -18,6 +18,14 @@
     VERSION = "dev"
 
 
+class DBTCloudException(Exception):
+    pass
+
+
+class DBTCloudParamsException(Exception):
+    pass
+
+
 class DBTCloud:
     """A minimalistic API client for fetching dbt Cloud data."""
 
@@ -56,10 +64,10 @@ def _clear_env_var_cache(self, job_definition_id: Optional[int]) -> None:
     def _check_for_creds(self):
         """Confirm the presence of credentials"""
         if not self._api_key:
-            raise Exception("An API key is required to get dbt Cloud jobs.")
+            raise DBTCloudParamsException("An API key is required to get dbt Cloud jobs.")
 
         if not self.account_id:
-            raise Exception("An account_id is required to get dbt Cloud jobs.")
+            raise DBTCloudParamsException("An account_id is required to get dbt Cloud jobs.")
 
     def build_mapping_job_identifier_job_id(
         self, cloud_jobs: Optional[List[JobDefinition]] = None
@@ -89,6 +97,7 @@ def update_job(self, job: JobDefinition) -> JobDefinition:
 
         if response.status_code >= 400:
             logger.error(response.json())
+            raise DBTCloudException(f"Error updating job {job.name}")
         else:
             logger.success("Job updated successfully.")
 
@@ -108,6 +117,7 @@ def create_job(self, job: JobDefinition) -> Optional[JobDefinition]:
 
         if response.status_code >= 400:
             logger.error(response.json())
+            raise DBTCloudException(f"Error creating job {job.name}")
             return None
         else:
             logger.success("Job created successfully.")
@@ -127,6 +137,7 @@ def delete_job(self, job: JobDefinition) -> None:
 
         if response.status_code >= 400:
             logger.error(response.json())
+            raise DBTCloudException(f"Error deleting job {job.name}")
         else:
             logger.success("Job deleted successfully.")
 
@@ -142,7 +153,7 @@ def get_job(self, job_id: int) -> Optional[JobDefinition]:
         )
         if response.status_code > 200:
             logger.error(f"Issue getting the job {job_id}")
-            return None
+            raise DBTCloudException(f"Error getting the job {job_id}")
         return JobDefinition(**response.json()["data"])
 
     def get_job_missing_fields(self, job_id: int) -> Optional[JobMissingFields]:
@@ -157,7 +168,7 @@ def get_job_missing_fields(self, job_id: int) -> Optional[JobMissingFields]:
         )
         if response.status_code > 200:
             logger.error(f"Issue getting the job {job_id}")
-            return None
+            raise DBTCloudException(f"Error getting the job {job_id}")
         return JobMissingFields(**response.json()["data"])
 
     def get_jobs(
@@ -285,6 +296,7 @@ def create_env_var(
 
         if response.status_code >= 400:
             logger.error(response.json())
+            raise DBTCloudException(f"Error creating the env var {env_var.name}")
 
         # If the new environment variables has a job_definition_id, then clear the EnvVar cache.
         self._clear_env_var_cache(job_definition_id=env_var.job_definition_id)
@@ -329,6 +341,7 @@ def update_env_var(
 
         if response.status_code >= 400:
             logger.error(response.json())
+            raise DBTCloudException(f"Error updating the env var {custom_env_var.name}")
 
         self._clear_env_var_cache(job_definition_id=payload.job_definition_id)
 
@@ -348,6 +361,7 @@ def delete_env_var(self, project_id: int, env_var_id: int) -> None:
 
         if response.status_code >= 400:
             logger.error(response.json())
+            raise DBTCloudException(f"Error deleting the env var {env_var_id}")
 
         logger.success("Env Var Job Overwrite deleted successfully.")
 

src/main.py

@@ -108,6 +108,10 @@ def sync(
         console.log(change_set.to_table())
     change_set.apply()
 
+    if not change_set.apply_success:
+        logger.error("-- SYNC -- There were some errors during the sync. Check the logs.")
+        sys.exit(1)
+
 
 @cli.command()
 @option_disable_ssl_verification