Implement database merge

Author: JoostGevaert

examples/hk_kaitak_ags3/hk_kaitak_ags3_to_brgi_geodb.py

@@ -225,14 +225,6 @@ def _(mo):
     return
 
 
-@app.cell
-def _():
-    none = None
-    boolean = not none
-    boolean
-    return
-
-
 @app.cell
 def _(CRS, zip, zipfile):
     from bedrock_ge.gi.ags3 import ags3_to_brgi_db_mapping
@@ -243,9 +235,7 @@ def _(CRS, zip, zipfile):
 
     projected_crs = CRS("EPSG:2326")
     vertrical_crs = CRS("EPSG:5738")
-    brgi_db_from_1_ags3_file = None
-    brgi_db_obj = None
-
+    brgi_dbs = []
     with zipfile.ZipFile(zip) as zip_ref:
         # Iterate over files and directories in the .zip archive
         for i, file_name in enumerate(zip_ref.namelist()):
@@ -257,14 +247,14 @@ def _(CRS, zip, zipfile):
                     ags3_mapping = ags_to_brgi_db_mapping(
                         ags3_file, projected_crs, vertrical_crs
                     )
-                    brgi_db_from_1_ags3_file = map_to_brgi_db(ags3_mapping)
+                    brgi_dbs.append(map_to_brgi_db(ags3_mapping))
 
             # if not brgi_db_obj:
             #     brgi_db_obj = brgi_db_from_1_ags3_file
-        
+
             # if brgi_db_obj and brgi_db_from_1_ags3_file:
             #     brgi_db_obj = merge_databases(brgi_db_obj, brgi_db_from_1_ags3_file)
-        
+
             # print(f"i = {i}: brgi_db = {brgi_db_obj}")
 
     # with zipfile.ZipFile(zip) as zip_ref:
@@ -279,6 +269,96 @@ def _(CRS, zip, zipfile):
     # brgi_db_obj = map_to_brgi_db(ags3_mapping_obj)
 
     # brgi_db_obj
+    return ags3_mapping, brgi_dbs, merge_databases
+
+
+@app.cell
+def _(project_data_jsons):
+    hash(project_data_jsons)
+    return
+
+
+@app.cell
+def _(ags3_mapping, pd):
+    import base64
+    import hashlib
+    import json
+
+    brgi_db_mapping = ags3_mapping
+    project_data_jsons = json.dumps(brgi_db_mapping.Project.data, sort_keys=True)
+    project_data_hash = hashlib.blake2b(
+        project_data_jsons.encode("utf-8"), digest_size=9
+    ).digest()
+    url_safe_hash = base64.b64encode(project_data_hash).decode()
+    project_uid = brgi_db_mapping.Project.project_id + "-" + url_safe_hash
+    location_df = pd.DataFrame(
+        {
+            "location_uid": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.location_id_column
+            ]
+            + f"_{project_uid}",
+            "location_source_id": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.location_id_column
+            ],
+            "project_uid": project_uid,
+            "easting": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.easting_column
+            ],
+            "northing": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.northing_column
+            ],
+            "ground_level_elevation": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.ground_level_elevation_column
+            ],
+            "depth_to_base": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.depth_to_base_column
+            ],
+        }
+    )
+    return base64, hashlib, project_data_jsons, project_uid, url_safe_hash
+
+
+@app.cell
+def _(url_safe_hash):
+    b64_hash = url_safe_hash
+    print(b64_hash)
+    return
+
+
+@app.cell
+def _(base64):
+    b = b'\xff\x00\xfa'
+    encoded = base64.b64encode(b).decode()
+    print(encoded)
+    return
+
+
+@app.cell
+def _(base64, hashlib, project_data_jsons, project_uid):
+    print(project_uid)
+    bytes_hash = hashlib.blake2b(
+        project_data_jsons.encode("utf-8"), digest_size=8
+    ).digest()
+    safe_hash = base64.b85encode(bytes_hash).decode()
+    safe_hash
+    return
+
+
+@app.cell
+def _(ags3_mapping):
+    ags3_mapping.Project
+    return
+
+
+@app.cell
+def _(brgi_dbs):
+    brgi_dbs[7].Project
+    return
+
+
+@app.cell
+def _(brgi_dbs, merge_databases):
+    brgi_db_obj = merge_databases(brgi_dbs)
     return
 
 

src/bedrock_ge/gi/db_operations.py

@@ -59,31 +59,73 @@ def merge_databases(
     elif len(dbs) == 1 and isinstance(dbs[0], BedrockGIDatabase):
         return dbs[0]
 
-    target_db = dbs.pop(0)
-
-    merged_db = {
-        "Project": "merged_project",
-        "Location": "merged_location",
-    }
-
-    # merged_db = BedrockGIDatabase(
-    #     Project=target_db.Project.append(incoming_db.Project),
-    #     Location=target_db.Location.append(incoming_db.Location),
-    #     InSituTests={
-    #         k: target_db.InSituTests[k].append(incoming_db.InSituTests[k])
-    #         for k in target_db.InSituTests
-    #         if k in incoming_db.InSituTests
-    #     },
-    #     Sample=target_db.Sample.append(incoming_db.Sample),
-    #     LabTests={
-    #         k: target_db.LabTests[k].append(incoming_db.LabTests[k])
-    #         for k in target_db.LabTests
-    #         if k in incoming_db.LabTests
-    #     },
-    #     Other={
-    #         k: target_db.Other[k].append(incoming_db.Other[k])
-    #         for k in target_db.Other
-    #         if k in incoming_db.Other
-    #     },
-    # )
-    return merged_db
+    merged_project = pd.concat([db.Project for db in dbs], ignore_index=True)
+    merged_project.drop_duplicates(inplace=True)
+    ProjectSchema.validate(merged_project)
+
+    merged_location = pd.concat([db.Location for db in dbs], ignore_index=True)
+    merged_location.drop_duplicates(inplace=True)
+    LocationSchema.validate(merged_location)
+    check_foreign_key("project_uid", merged_project, merged_location)
+
+    insitu_tables: set[str] = set()
+    lab_tables: set[str] = set()
+    other_tables: set[str] = set()
+    for db in dbs:
+        insitu_tables.update(db.InSituTests.keys())
+        if db.LabTests:
+            lab_tables.update(db.LabTests.keys())
+        if db.Other:
+            other_tables.update(db.Other.keys())
+
+    merged_insitu: dict[str, pd.DataFrame] = {}
+    for insitu_table in insitu_tables:
+        insitu_df = pd.concat(
+            [db.InSituTests.get(insitu_table) for db in dbs], ignore_index=True
+        )
+        insitu_df.drop_duplicates(inplace=True)
+        InSituTestSchema.validate(insitu_df)
+        check_foreign_key("project_uid", merged_project, insitu_df)
+        check_foreign_key("location_uid", merged_location, insitu_df)
+        merged_insitu[insitu_table] = insitu_df
+
+    sample_dfs = [db.Sample for db in dbs if db.Sample is not None]
+    if sample_dfs:
+        merged_sample = pd.concat(sample_dfs, ignore_index=True)
+        merged_sample.drop_duplicates(inplace=True)
+        SampleSchema.validate(merged_sample)
+        check_foreign_key("project_uid", merged_project, merged_sample)
+
+    merged_lab: dict[str, pd.DataFrame] = {}
+    for lab_table in lab_tables:
+        lab_dfs = [
+            db.LabTests.get(lab_table)
+            for db in dbs
+            if db.LabTests.get(lab_table) is not None
+        ]
+        lab_df = pd.concat(lab_dfs, ignore_index=True)
+        lab_df.drop_duplicates(inplace=True)
+        check_foreign_key("project_uid", merged_project, lab_df)
+        check_foreign_key("sample_uid", merged_sample, lab_df)
+        merged_lab[lab_table] = lab_df
+
+    merged_other: dict[str, pd.DataFrame] = {}
+    for other_table in other_tables:
+        other_dfs = [
+            db.Other.get(other_table)
+            for db in dbs
+            if db.Other.get(other_table) is not None
+        ]
+        other_df = pd.concat(other_dfs, ignore_index=True)
+        other_df.drop_duplicates(inplace=True)
+        check_foreign_key("project_uid", merged_project, other_df)
+        merged_other[other_table] = other_df
+
+    return BedrockGIDatabase(
+        Project=merged_project,
+        Location=merged_location,
+        InSituTests=merged_insitu,
+        Sample=merged_sample,
+        LabTests=merged_lab,
+        Other=merged_other,
+    )