Fix marimo dtypes

Author: JoostGevaert

examples/hk_kaitak_ags3/hk_kaitak_ags3_to_brgi_geodb.py

@@ -17,7 +17,7 @@
 
 import marimo
 
-__generated_with = "0.13.11"
+__generated_with = "0.13.15"
 app = marimo.App(
     app_title="Kai Tak, HK AGS 3 data to bedrock_ge.gi geodatabase",
 )
@@ -249,7 +249,14 @@ def _():
 
 
 @app.cell
-def _(CRS, ags_to_brgi_db_mapping, map_to_brgi_db, zip, zipfile):
+def _(
+    CRS,
+    ags_to_brgi_db_mapping,
+    map_to_brgi_db,
+    merge_databases,
+    zip,
+    zipfile,
+):
     projected_crs = CRS("EPSG:2326")
     vertrical_crs = CRS("EPSG:5738")
     brgi_dbs = []
@@ -266,32 +273,14 @@ def _(CRS, ags_to_brgi_db_mapping, map_to_brgi_db, zip, zipfile):
                     )
                     brgi_dbs.append(map_to_brgi_db(ags3_mapping))
 
-            # if not brgi_db_obj:
-            #     brgi_db_obj = brgi_db_from_1_ags3_file
-
-            # if brgi_db_obj and brgi_db_from_1_ags3_file:
-            #     brgi_db_obj = merge_databases(brgi_db_obj, brgi_db_from_1_ags3_file)
-
-            # print(f"i = {i}: brgi_db = {brgi_db_obj}")
-
-    # with zipfile.ZipFile(zip) as zip_ref:
-    #     file_name = "21659/9508008.AGS"
-    #     print(f"\n🖥️ Processing {file_name} ...")
-    #     with zip_ref.open(file_name) as ags3_file:
-    #         # Convert content of a single AGS 3 file to a Dictionary of pandas dataframes (a database)
-    #         ags3_mapping_obj = ags3_to_brgi_db_mapping(
-    #             ags3_file, projected_crs, vertrical_crs, "utf-8"
-    #         )
-
-    # brgi_db_obj = map_to_brgi_db(ags3_mapping_obj)
+    merged_brgi_db = merge_databases(brgi_dbs)
 
-    # brgi_db_obj
-    return (brgi_dbs,)
+    return (merged_brgi_db,)
 
 
 @app.cell
-def _(brgi_dbs, merge_databases):
-    merged_brgi_db = merge_databases(brgi_dbs)
+def _(merged_brgi_db):
+    merged_brgi_db.InSituTests["ISPT"]
     return
 
 

src/bedrock_ge/gi/db_operations.py

@@ -4,9 +4,11 @@
 
 import pandas as pd
 
+from bedrock_ge.gi.io_utils import convert_dtypes_object_to_string
 from bedrock_ge.gi.schemas import (
     BedrockGIDatabase,
     InSituTestSchema,
+    LabTestSchema,
     LocationSchema,
     ProjectSchema,
     SampleSchema,
@@ -40,11 +42,13 @@ def merge_databases(
     project_dataframes = _filter_dataframes([db.Project for db in dbs])
     merged_project = pd.concat(project_dataframes, ignore_index=True)
     merged_project = merged_project.drop_duplicates().reset_index(drop=True)
+    merged_project = convert_dtypes_object_to_string(merged_project.convert_dtypes())
     ProjectSchema.validate(merged_project)
 
     location_dataframes = _filter_dataframes([db.Location for db in dbs])
     merged_location = pd.concat(location_dataframes, ignore_index=True)
     merged_location = merged_location.drop_duplicates().reset_index(drop=True)
+    merged_location = convert_dtypes_object_to_string(merged_location.convert_dtypes())
     LocationSchema.validate(merged_location)
     check_foreign_key("project_uid", merged_project, merged_location)
 
@@ -65,6 +69,7 @@ def merge_databases(
         )
         insitu_df = pd.concat(insitu_dataframes, ignore_index=True)
         insitu_df = insitu_df.drop_duplicates().reset_index(drop=True)
+        insitu_df = convert_dtypes_object_to_string(insitu_df.convert_dtypes())
         InSituTestSchema.validate(insitu_df)
         check_foreign_key("project_uid", merged_project, insitu_df)
         check_foreign_key("location_uid", merged_location, insitu_df)
@@ -75,6 +80,7 @@ def merge_databases(
     if sample_dfs:
         merged_sample = pd.concat(sample_dfs, ignore_index=True)
         merged_sample = merged_sample.drop_duplicates().reset_index(drop=True)
+        merged_sample = convert_dtypes_object_to_string(merged_sample.convert_dtypes())
         SampleSchema.validate(merged_sample)
         check_foreign_key("project_uid", merged_project, merged_sample)
 
@@ -87,6 +93,8 @@ def merge_databases(
         ]
         lab_df = pd.concat(lab_dfs, ignore_index=True)
         lab_df = lab_df.drop_duplicates().reset_index(drop=True)
+        lab_df = convert_dtypes_object_to_string(lab_df.convert_dtypes())
+        LabTestSchema.validate(lab_df)
         check_foreign_key("project_uid", merged_project, lab_df)
         check_foreign_key("sample_uid", merged_sample, lab_df)
         merged_lab[lab_table] = lab_df
@@ -100,6 +108,7 @@ def merge_databases(
         ]
         other_df = pd.concat(other_dfs, ignore_index=True)
         other_df = other_df.drop_duplicates().reset_index(drop=True)
+        other_df = convert_dtypes_object_to_string(other_df.convert_dtypes())
         check_foreign_key("project_uid", merged_project, other_df)
         merged_other[other_table] = other_df
 

src/bedrock_ge/gi/io_utils.py

@@ -9,6 +9,7 @@
 from typing import IO, ContextManager
 
 import chardet
+import pandas as pd
 
 DEFAULT_ENCODING = "utf-8"
 
@@ -190,3 +191,9 @@ def coerce_string(string: str) -> None | bool | float | str:
                 return value
         except ValueError:
             return string
+
+
+def convert_dtypes_object_to_string(dataframe: pd.DataFrame) -> pd.DataFrame:
+    object_cols = dataframe.select_dtypes(include=["object"]).columns
+    dataframe[object_cols] = dataframe[object_cols].astype("string")
+    return dataframe

src/bedrock_ge/gi/mapper.py

@@ -107,6 +107,7 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
         insitu_tests[insitu_mapping.table_name] = insitu_df.copy()
 
     # Create the sample table
+    sample_df = None
     if brgi_db_mapping.Sample:
         sample_df = pd.DataFrame(
             {
@@ -167,7 +168,7 @@ def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
         Project=project_df,
         Location=location_df,
         InSituTests=insitu_tests,
-        Sample=sample_df if brgi_db_mapping.Sample else None,
+        Sample=sample_df,
         LabTests=lab_tests,
         Other=other_tables,
     )