Prevent circular imports and strip '?' from non-standard AGS 3 groups and headers

Author: JoostGevaert

examples/hk_kaitak_ags3/hk_kaitak_ags3_to_brgi_geodb.py

@@ -227,7 +227,10 @@ def _(mo):
 
 @app.cell
 def _(CRS, zip, zipfile):
+    from bedrock_ge.gi.ags3 import ags3_to_brgi_db_mapping
     from bedrock_ge.gi.ags_parser import ags_to_brgi_db_mapping
+    from bedrock_ge.gi.mapping_models import BedrockGIMapping
+    from bedrock_ge.gi.mapper import map_to_brgi_db
 
     projected_crs = CRS("EPSG:2326")
     vertrical_crs = CRS("EPSG:5738")
@@ -240,18 +243,23 @@ def _(CRS, zip, zipfile):
                 print(f"\n🖥️ Processing {file_name} ...")
                 with zip_ref.open(file_name) as ags3_file:
                     # Convert content of a single AGS 3 file to a Dictionary of pandas dataframes (a database)
-                    ags3_to_brgi_db_mapping = ags_to_brgi_db_mapping(
+                    ags3_mapping = ags_to_brgi_db_mapping(
                         ags3_file, projected_crs, vertrical_crs
                     )
+                    brgi_db_obj = map_to_brgi_db(ags3_mapping)
 
     # with zipfile.ZipFile(zip) as zip_ref:
-    #     file_name = "58358/GE201304.18A.ags"
+    #     file_name = "21659/9508008.AGS"
     #     print(f"\n🖥️ Processing {file_name} ...")
     #     with zip_ref.open(file_name) as ags3_file:
     #         # Convert content of a single AGS 3 file to a Dictionary of pandas dataframes (a database)
-    #         ags3_to_brgi_db_mapping = ags_to_brgi_db_mapping(
-    #             ags3_file, projected_crs, vertrical_crs
+    #         ags3_mapping_obj = ags3_to_brgi_db_mapping(
+    #             ags3_file, projected_crs, vertrical_crs, "utf-8"
     #         )
+
+    # brgi_db_obj = map_to_brgi_db(ags3_mapping_obj)
+
+    # brgi_db_obj
     return
 
 

src/bedrock_ge/gi/ags3.py

@@ -7,23 +7,26 @@
 from pyproj import CRS
 
 from bedrock_ge.gi.ags_schemas import Ags3HOLE, Ags3SAMP, check_ags_proj_group
-from bedrock_ge.gi.brgi_db_mapping import (
-    BedrockGIDatabaseMapping,
+from bedrock_ge.gi.io_utils import coerce_string, open_text_data_source
+from bedrock_ge.gi.mapping_models import (
+    BedrockGIMapping,
     InSituTestTableMapping,
     LabTestTableMapping,
     LocationTableMapping,
     OtherTable,
     ProjectTableMapping,
     SampleTableMapping,
 )
-from bedrock_ge.gi.io_utils import coerce_string, open_text_data_source
 
 
 def ags3_to_dfs(
     source: str | Path | IO[str] | IO[bytes] | bytes, encoding: str
 ) -> dict[str, pd.DataFrame]:
     """Converts AGS 3 data to a dictionary of pandas DataFrames.
 
+    Also strips '?' from non-standard AGS 3 group and header names, in order to
+    make the rest of the code more generic.
+
     Args:
         source (str | Path | IO[str] | IO[bytes] | bytes): The AGS 3 file (str or Path)
             or a file-like object that represents the AGS 3 file.
@@ -52,14 +55,14 @@ def ags3_to_dfs(
                 if group:
                     ags3_dfs[group] = pd.DataFrame(group_data, columns=headers)
 
-                group = line.strip(' ,"*')
+                group = line.strip(' ,"*?')
                 group_data = []
 
             # In AGS 3 header names are prefixed with "*
             elif line.startswith('"*'):
                 line_type = "headers"
                 new_headers = line.split('","')
-                new_headers = [h.strip(' ,"*') for h in new_headers]
+                new_headers = [h.strip(' ,"*?') for h in new_headers]
 
                 # Some groups have so many headers that they span multiple lines.
                 # Therefore we need to check whether the new headers are
@@ -131,7 +134,7 @@ def ags3_to_brgi_db_mapping(
     projected_crs: CRS,
     vertical_crs: CRS,
     encoding: str,
-) -> BedrockGIDatabaseMapping:
+) -> BedrockGIMapping:
     """Map AGS 3 data to the Bedrock GI data model.
 
     Args:
@@ -180,7 +183,6 @@ def ags3_to_brgi_db_mapping(
         )
         del ags3_dfs["SAMP"]
     else:
-        print("Your AGS 3 data doesn't contain a SAMP group, i.e. samples.")
         ags3_sample = None
 
     ags3_lab_tests = []
@@ -190,7 +192,7 @@ def ags3_to_brgi_db_mapping(
     for group, df in ags3_dfs.items():
         # Non-standard group names contain the "?" prefix.
         # => checking that "SAMP_TOP" / "HOLE_ID" is in the columns is too restrictive.
-        if any("SAMP_TOP" in col for col in df.columns):
+        if "SAMP_TOP" in df.columns:
             df = _add_sample_source_id(df)
             ags3_lab_tests.append(
                 LabTestTableMapping(
@@ -200,7 +202,7 @@ def ags3_to_brgi_db_mapping(
                     sample_id_column="sample_source_id",
                 )
             )
-        elif any("HOLE_ID" in col for col in df.columns):
+        elif "HOLE_ID" in df.columns:
             top_depth, base_depth = _get_depth_columns(group, list(df.columns))
             ags3_insitu_tests.append(
                 InSituTestTableMapping(
@@ -214,24 +216,26 @@ def ags3_to_brgi_db_mapping(
         else:
             ags3_other_tables.append(OtherTable(table_name=group, data=df))
 
-    ags3_brgi_db_mapping = BedrockGIDatabaseMapping(
+    brgi_db_mapping = BedrockGIMapping(
         Project=ags3_project,
         Location=ags3_location,
         InSitu=ags3_insitu_tests,
         Sample=ags3_sample,
         Lab=ags3_lab_tests,
         Other=ags3_other_tables,
     )
-    return ags3_brgi_db_mapping
+    return brgi_db_mapping
 
 
 def _add_sample_source_id(df: pd.DataFrame) -> pd.DataFrame:
     df["sample_source_id"] = (
         df["SAMP_REF"].astype(str)
-        + "_"
+        + "-"
         + df["SAMP_TYPE"].astype(str)
-        + "_"
+        + "-"
         + df["SAMP_TOP"].astype(str)
+        + "-"
+        + df["HOLE_ID"].astype(str)
     )
     return df
 

src/bedrock_ge/gi/ags_parser.py

@@ -7,16 +7,16 @@
 from pyproj import CRS
 
 from bedrock_ge.gi.ags3 import ags3_to_brgi_db_mapping
-from bedrock_ge.gi.brgi_db_mapping import BedrockGIDatabaseMapping
 from bedrock_ge.gi.io_utils import detect_encoding, open_text_data_source
+from bedrock_ge.gi.mapping_models import BedrockGIMapping
 
 
 def ags_to_brgi_db_mapping(
     source: str | Path | IO[str] | IO[bytes] | bytes,
     projected_crs: CRS,
     vertical_crs: CRS = CRS(3855),
     encoding: str | None = None,
-) -> BedrockGIDatabaseMapping:
+) -> BedrockGIMapping:
     """Map AGS 3 or AGS 4 data to the Bedrock GI data model.
 
     Args: