Make a start on merging Bedrock GI Databases

Author: JoostGevaert

examples/hk_kaitak_ags3/hk_kaitak_ags3_to_brgi_geodb.py

@@ -225,19 +225,30 @@ def _(mo):
     return
 
 
+@app.cell
+def _():
+    none = None
+    boolean = not none
+    boolean
+    return
+
+
 @app.cell
 def _(CRS, zip, zipfile):
     from bedrock_ge.gi.ags3 import ags3_to_brgi_db_mapping
     from bedrock_ge.gi.ags_parser import ags_to_brgi_db_mapping
     from bedrock_ge.gi.mapping_models import BedrockGIMapping
     from bedrock_ge.gi.mapper import map_to_brgi_db
+    from bedrock_ge.gi.db_operations import merge_databases
 
     projected_crs = CRS("EPSG:2326")
     vertrical_crs = CRS("EPSG:5738")
+    brgi_db_from_1_ags3_file = None
+    brgi_db_obj = None
 
     with zipfile.ZipFile(zip) as zip_ref:
         # Iterate over files and directories in the .zip archive
-        for file_name in zip_ref.namelist():
+        for i, file_name in enumerate(zip_ref.namelist()):
             # Only process files that have an .ags or .AGS extension
             if file_name.lower().endswith(".ags"):
                 print(f"\n🖥️ Processing {file_name} ...")
@@ -246,7 +257,15 @@ def _(CRS, zip, zipfile):
                     ags3_mapping = ags_to_brgi_db_mapping(
                         ags3_file, projected_crs, vertrical_crs
                     )
-                    brgi_db_obj = map_to_brgi_db(ags3_mapping)
+                    brgi_db_from_1_ags3_file = map_to_brgi_db(ags3_mapping)
+
+            # if not brgi_db_obj:
+            #     brgi_db_obj = brgi_db_from_1_ags3_file
+        
+            # if brgi_db_obj and brgi_db_from_1_ags3_file:
+            #     brgi_db_obj = merge_databases(brgi_db_obj, brgi_db_from_1_ags3_file)
+        
+            # print(f"i = {i}: brgi_db = {brgi_db_obj}")
 
     # with zipfile.ZipFile(zip) as zip_ref:
     #     file_name = "21659/9508008.AGS"

src/bedrock_ge/gi/db_operations.py

@@ -0,0 +1,74 @@
+import pandas as pd
+
+from bedrock_ge.gi.schemas import (
+    BedrockGIDatabase,
+    InSituTestSchema,
+    LocationSchema,
+    ProjectSchema,
+    SampleSchema,
+)
+from bedrock_ge.gi.validate import check_foreign_key
+
+
+def merge_databases(
+    target_db: BedrockGIDatabase,
+    incoming_db: BedrockGIDatabase,
+) -> BedrockGIDatabase:
+    """Merges the incoming Bedrock GI database into the target Bedrock GI database.
+
+    The function concatenates the pandas DataFrames of the second dict of
+    DataFrames to the first dict of DataFrames for the keys they have in common.
+    Keys that are unique to either dictionary will be included in the final
+    concatenated dictionary.
+
+    Args:
+        target_db (BedrockGIDatabase): The Bedrock GI database into which the incoming data will be merged.
+        incoming_db (BedrockGIDatabase): The Bedrock GI database containing the data to be merged.
+
+    Returns:
+        BedrockGIDatabase: Merged Bedrock GI database.
+    """
+    # write the body of this function that merges the incoming_db (BedrockGIDatabase) into the target_db (BedrockGIDatabase).
+    # duplicate rows in the incoming_db (BedrockGIDatabase) will be dropped.
+    # After merging tables validate them with the schemas from bedrock_ge.gi.schemas and check that foreign keys are correct.
+    # In case the incoming_db contains tables that are not in the target_db, add them to the target_db.
+    # The function must return a BedrockGIDatabase object.
+    merged_project = pd.concat(
+        [target_db.Project, incoming_db.Project], ignore_index=True
+    )
+    ProjectSchema.validate(merged_project)
+
+    merged_location = pd.concat(
+        [target_db.Location, incoming_db.Location], ignore_index=True
+    )
+    LocationSchema.validate(merged_location)
+    check_foreign_key("project_uid", merged_project, merged_location)
+
+    merged_insitu = {}
+
+    merged_db = {
+        "Project": merged_project,
+        "Location": merged_location,
+    }
+
+    # merged_db = BedrockGIDatabase(
+    #     Project=target_db.Project.append(incoming_db.Project),
+    #     Location=target_db.Location.append(incoming_db.Location),
+    #     InSituTests={
+    #         k: target_db.InSituTests[k].append(incoming_db.InSituTests[k])
+    #         for k in target_db.InSituTests
+    #         if k in incoming_db.InSituTests
+    #     },
+    #     Sample=target_db.Sample.append(incoming_db.Sample),
+    #     LabTests={
+    #         k: target_db.LabTests[k].append(incoming_db.LabTests[k])
+    #         for k in target_db.LabTests
+    #         if k in incoming_db.LabTests
+    #     },
+    #     Other={
+    #         k: target_db.Other[k].append(incoming_db.Other[k])
+    #         for k in target_db.Other
+    #         if k in incoming_db.Other
+    #     },
+    # )
+    return merged_db