Make sure the correct df is output on the BedrockGIDatabase

Author: JoostGevaert

examples/hk_kaitak_ags3/hk_kaitak_ags3_to_brgi_geodb.py

@@ -177,7 +177,7 @@ def _(mo):
 @app.cell
 def _(io, requests):
     # Read ZIP from disk after downloading manually
-    # zip = Path(r"C:\Users\joost\ReposWindows\bedrock-ge\examples\hk_kaitak_ags3\public\kaitak_ags3.zip")
+    # zip = Path.home() / "Downloads" / "kaitak_ags3.zip"
 
     # Request ZIP from GitHub
     raw_githubusercontent_url = "https://raw.githubusercontent.com/bedrock-engineer/bedrock-ge/main/examples/hk_kaitak_ags3/kaitak_ags3.zip"
@@ -226,13 +226,30 @@ def _(mo):
 
 
 @app.cell
-def _(CRS, zip, zipfile):
+def _():
+    import json
+    import hashlib
+    import base64
+
     from bedrock_ge.gi.ags3 import ags3_to_brgi_db_mapping
     from bedrock_ge.gi.ags_parser import ags_to_brgi_db_mapping
-    from bedrock_ge.gi.mapping_models import BedrockGIMapping
-    from bedrock_ge.gi.mapper import map_to_brgi_db
     from bedrock_ge.gi.db_operations import merge_databases
+    from bedrock_ge.gi.mapper import map_to_brgi_db
+    from bedrock_ge.gi.mapping_models import BedrockGIMapping
+    from bedrock_ge.gi.schemas import (
+        BedrockGIDatabase,
+        InSituTestSchema,
+        LabTestSchema,
+        LocationSchema,
+        ProjectSchema,
+        SampleSchema,
+    )
+
+    return ags_to_brgi_db_mapping, map_to_brgi_db, merge_databases
+
 
+@app.cell
+def _(CRS, ags_to_brgi_db_mapping, map_to_brgi_db, zip, zipfile):
     projected_crs = CRS("EPSG:2326")
     vertrical_crs = CRS("EPSG:5738")
     brgi_dbs = []
@@ -269,96 +286,12 @@ def _(CRS, zip, zipfile):
     # brgi_db_obj = map_to_brgi_db(ags3_mapping_obj)
 
     # brgi_db_obj
-    return ags3_mapping, brgi_dbs, merge_databases
-
-
-@app.cell
-def _(project_data_jsons):
-    hash(project_data_jsons)
-    return
-
-
-@app.cell
-def _(ags3_mapping, pd):
-    import base64
-    import hashlib
-    import json
-
-    brgi_db_mapping = ags3_mapping
-    project_data_jsons = json.dumps(brgi_db_mapping.Project.data, sort_keys=True)
-    project_data_hash = hashlib.blake2b(
-        project_data_jsons.encode("utf-8"), digest_size=9
-    ).digest()
-    url_safe_hash = base64.b64encode(project_data_hash).decode()
-    project_uid = brgi_db_mapping.Project.project_id + "-" + url_safe_hash
-    location_df = pd.DataFrame(
-        {
-            "location_uid": brgi_db_mapping.Location.data[
-                brgi_db_mapping.Location.location_id_column
-            ]
-            + f"_{project_uid}",
-            "location_source_id": brgi_db_mapping.Location.data[
-                brgi_db_mapping.Location.location_id_column
-            ],
-            "project_uid": project_uid,
-            "easting": brgi_db_mapping.Location.data[
-                brgi_db_mapping.Location.easting_column
-            ],
-            "northing": brgi_db_mapping.Location.data[
-                brgi_db_mapping.Location.northing_column
-            ],
-            "ground_level_elevation": brgi_db_mapping.Location.data[
-                brgi_db_mapping.Location.ground_level_elevation_column
-            ],
-            "depth_to_base": brgi_db_mapping.Location.data[
-                brgi_db_mapping.Location.depth_to_base_column
-            ],
-        }
-    )
-    return base64, hashlib, project_data_jsons, project_uid, url_safe_hash
-
-
-@app.cell
-def _(url_safe_hash):
-    b64_hash = url_safe_hash
-    print(b64_hash)
-    return
-
-
-@app.cell
-def _(base64):
-    b = b'\xff\x00\xfa'
-    encoded = base64.b64encode(b).decode()
-    print(encoded)
-    return
-
-
-@app.cell
-def _(base64, hashlib, project_data_jsons, project_uid):
-    print(project_uid)
-    bytes_hash = hashlib.blake2b(
-        project_data_jsons.encode("utf-8"), digest_size=8
-    ).digest()
-    safe_hash = base64.b85encode(bytes_hash).decode()
-    safe_hash
-    return
-
-
-@app.cell
-def _(ags3_mapping):
-    ags3_mapping.Project
-    return
-
-
-@app.cell
-def _(brgi_dbs):
-    brgi_dbs[7].Project
-    return
+    return (brgi_dbs,)
 
 
 @app.cell
 def _(brgi_dbs, merge_databases):
-    brgi_db_obj = merge_databases(brgi_dbs)
+    merged_brgi_db = merge_databases(brgi_dbs)
     return
 
 

src/bedrock_ge/gi/mapper.py

@@ -32,12 +32,16 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
         BedrockGIDatabase: The transformed Bedrock GI database containing validated
             DataFrames for each table type.
     """
+    # Create a base64 hash from the project data, such that project a project Unique ID
+    # can be created from the project_id and the hash of the project data.
     project_data_jsons = json.dumps(brgi_db_mapping.Project.data, sort_keys=True)
     project_data_bytes_hash = hashlib.blake2b(
         project_data_jsons.encode("utf-8"), digest_size=9
     ).digest()
     project_data_b64_hash = base64.b64encode(project_data_bytes_hash).decode()
     project_uid = brgi_db_mapping.Project.project_id + "-" + project_data_b64_hash
+
+    # Create the project table
     brgi_project = pd.DataFrame(
         {
             "project_uid": project_uid,
@@ -52,6 +56,7 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
     )
     ProjectSchema.validate(brgi_project)
 
+    # Create the location table
     location_df = pd.DataFrame(
         {
             "location_uid": brgi_db_mapping.Location.data[
@@ -79,6 +84,7 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
     location_df = pd.concat([location_df, brgi_db_mapping.Location.data], axis=1)
     LocationSchema.validate(location_df)
 
+    # Create the in-situ test tables
     insitu_tests = {}
     for insitu_mapping in brgi_db_mapping.InSitu:
         insitu_df = pd.DataFrame(
@@ -97,6 +103,7 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
         InSituTestSchema.validate(insitu_df)
         insitu_tests[insitu_mapping.table_name] = insitu_df.copy()
 
+    # Create the sample table
     if brgi_db_mapping.Sample:
         sample_df = pd.DataFrame(
             {
@@ -124,6 +131,7 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
         sample_df = pd.concat([sample_df, brgi_db_mapping.Sample.data], axis=1)
         SampleSchema.validate(sample_df)
 
+    # Create the lab test tables
     brgi_lab_tests = {}
     if brgi_db_mapping.Lab:
         for lab_mapping in brgi_db_mapping.Lab:
@@ -145,9 +153,10 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
             LabTestSchema.validate(lab_df)
             brgi_lab_tests[lab_mapping.table_name] = lab_df.copy()
 
-    brgi_db = BedrockGIDatabase(
+    # Create and return the Bedrock GI database
+    return BedrockGIDatabase(
         Project=brgi_project,
-        Location=pd.DataFrame(brgi_db_mapping.Location.data),
+        Location=location_df,
         InSituTests=insitu_tests,
         Sample=sample_df if brgi_db_mapping.Sample else None,
         LabTests=brgi_lab_tests,
@@ -160,4 +169,3 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
             else {}
         ),
     )
-    return brgi_db