Merge pull request #640 from hmcts/feature-error-handling-deleting-tables

Add error handling on deletion of files incase they have already been…

Author: AndrewHMCTS

Databricks/ACTIVE/MVP/Active_Publish_EH_HTML_JSON.ipynb

@@ -281,23 +281,41 @@
     "states = [\"paymentPending\"]\n",
     "\n",
     "for state in states:\n",
-    "    spark.sql(f\"DELETE FROM delta.`abfss://{silver_curated_container}@{curated_storage_account}.dfs.core.windows.net/ARIADM/ACTIVE/CCD/APPEALS/{state}/publish_audit_db_eh` WHERE true\")\n",
-    "    logger.info(f\"{state}: old data successfully deleted from publish_audit_db_eh\")\n",
+    "    try:\n",
+    "        spark.sql(f\"DELETE FROM delta.`abfss://{silver_curated_container}@{curated_storage_account}.dfs.core.windows.net/ARIADM/ACTIVE/CCD/APPEALS/{state}/publish_audit_db_eh` WHERE true\")\n",
+    "        logger.info(f\"{state}: old data successfully deleted from publish_audit_db_eh\")\n",
+    "    except Exception as e:\n",
+    "        logger.warning(f\"{state}: Could not delete data from publish_audit_db_eh - {e}\")\n",
     "\n",
-    "spark.sql(\"TRUNCATE TABLE hive_metastore.paymentpending_gold.dashboard_segmentation_state\")\n",
-    "logger.info(\"dashboard_segmentation_state table truncated\")\n",
+    "try:\n",
+    "    spark.sql(\"TRUNCATE TABLE hive_metastore.paymentpending_gold.dashboard_segmentation_state\")\n",
+    "    logger.info(\"dashboard_segmentation_state table truncated\")\n",
+    "except Exception as e:\n",
+    "    logger.warning(f\"Could not truncate dashboard_segmentation_state as data does not exist - {e}\")\n",
     "\n",
-    "spark.sql(\"TRUNCATE TABLE hive_metastore.paymentpending_gold.dashboard_valid_json_files\")\n",
-    "logger.info(\"dashboard_valid_json_files table truncated\")\n",
+    "try:\n",
+    "    spark.sql(\"TRUNCATE TABLE hive_metastore.paymentpending_gold.dashboard_valid_json_files\")\n",
+    "    logger.info(\"dashboard_valid_json_files table truncated\")\n",
+    "except Exception as e:\n",
+    "    logger.warning(f\"Could not truncate dashboard_valid_json_files as data does not exist - {e}\")\n",
     "\n",
-    "spark.sql(\"TRUNCATE TABLE hive_metastore.paymentpending_gold.dashboard_invalid_json_files\")\n",
-    "logger.info(\"dashboard_invalid_json_files table truncated\")\n",
+    "try:\n",
+    "    spark.sql(\"TRUNCATE TABLE hive_metastore.paymentpending_gold.dashboard_invalid_json_files\")\n",
+    "    logger.info(\"dashboard_invalid_json_files table truncated\")\n",
+    "except Exception as e:\n",
+    "    logger.warning(f\"Could not truncate dashboard_invalid_json_files  as data does not exist - {e}\")\n",
     "\n",
-    "spark.sql(f\"DELETE FROM delta.`abfss://silver@ingest{lz_key}curated{env}.dfs.core.windows.net/ARIADM/ACTIVE/CCD/AUDIT/APPEALS/all_active_states/ack_audit` WHERE true\")\n",
-    "logger.info(\"ack_audit data deleted\")\n",
+    "try:\n",
+    "    spark.sql(f\"DELETE FROM delta.`abfss://silver@ingest{lz_key}curated{env}.dfs.core.windows.net/ARIADM/ACTIVE/CCD/AUDIT/APPEALS/all_active_states/ack_audit` WHERE true\")\n",
+    "    logger.info(\"ack_audit data deleted\")\n",
+    "except Exception as e:\n",
+    "    logger.warning(f\"No ack audit data exists to delete - {e}\")\n",
     "\n",
-    "spark.sql(f\"DELETE FROM delta.`abfss://{silver_curated_container}@{curated_storage_account}.dfs.core.windows.net/ARIADM/ACTIVE/CCD/APPEALS/publish_payload_audit` WHERE true\")\n",
-    "logger.info(\"publish_payload_audit data deleted\")"
+    "try:\n",
+    "    spark.sql(f\"DELETE FROM delta.`abfss://{silver_curated_container}@{curated_storage_account}.dfs.core.windows.net/ARIADM/ACTIVE/CCD/APPEALS/publish_payload_audit` WHERE true\")\n",
+    "    logger.info(\"publish_payload_audit data deleted\")\n",
+    "except Exception as e:\n",
+    "    logger.warning(f\"No publish_payload_audit data exists to delete - {e}\")"
    ]
   },
   {