Rename to sar_pipeline

.github/workflows/run-pytest.yml

@@ -26,4 +26,4 @@ jobs:
     - name: Run tests
       shell: micromamba-shell {0}
       run: |
-        pytest tests/sar_antarctica/
+        pytest tests/sar_pipeline/

.gitignore

@@ -10,5 +10,5 @@ build
 data
 
 # include test data files
-!tests/sar_antarctica/data/**/*.tif
-!tests/sar_antarctica/data/**/*.vrt
+!tests/sar_pipeline/data/**/*.tif
+!tests/sar_pipeline/data/**/*.vrt

docs/workflows/README.MD

@@ -23,7 +23,7 @@ $ find-scene S1A_EW_GRDM_1SDH_20240129T091735_20240129T091828_052319_065379_0F1E
 
 ### Submit a workflow
 This will submit a job request to the NCI based on the job parameters and file paths in the supplied config. 
-The [default config](../../sar_antarctica/nci/configs/default.toml) will be used if no other config is provided.
+The [default config](../../sar_pipeline/nci/configs/default.toml) will be used if no other config is provided.
 
 Example usage
 ```

environment.yml

@@ -1,4 +1,4 @@
-name: sar-antarctica
+name: sar-pipeline
 dependencies:
   - _libgcc_mutex=0.1
   - _openmp_mutex=4.5

sar_antarctica/nci/cli.py → sar_pipeline/nci/cli.py

@@ -3,26 +3,27 @@
 import tomli
 import logging
 
-from sar_antarctica.nci.filesystem import get_orbits_nci
-from sar_antarctica.nci.submission.pyrosar_gamma.prepare_input import (
+from sar_pipeline.nci.filesystem import get_orbits_nci
+from sar_pipeline.nci.submission.pyrosar_gamma.prepare_input import (
     get_orbit_and_dem,
 )
-from sar_antarctica.nci.preparation.orbits import (
+from sar_pipeline.nci.preparation.orbits import (
     filter_orbits_to_cover_time_window,
 )
-from sar_antarctica.nci.preparation.scenes import (
+from sar_pipeline.nci.preparation.scenes import (
     parse_scene_file_sensor,
     parse_scene_file_dates,
     find_scene_file_from_id,
 )
-from sar_antarctica.nci.processing.pyroSAR.pyrosar_geocode import (
+from sar_pipeline.nci.processing.pyroSAR.pyrosar_geocode import (
     run_pyrosar_gamma_geocode,
 )
-from sar_antarctica.nci.submission.pyrosar_gamma.submit_job import submit_job
-from sar_antarctica.nci.upload.push_folder_to_s3 import push_files_in_folder_to_s3
+from sar_pipeline.nci.submission.pyrosar_gamma.submit_job import submit_job
+from sar_pipeline.nci.upload.push_folder_to_s3 import push_files_in_folder_to_s3
 
 logging.basicConfig(level=logging.INFO)
 
+
 @click.command()
 @click.argument("scene_name", type=str)
 def find_scene_file(scene_name):
@@ -178,31 +179,45 @@ def find_orbits_for_scene(scene: str):
 
 
 @click.command()
-@click.argument('src_folder', type=click.Path(exists=True, file_okay=False))
-@click.argument('s3_bucket')
-@click.argument('s3_bucket_folder')
-@click.option('--upload-folder', 
-              default=False, 
-              is_flag=True, 
-              help="Upload the whole folder to specified s3_bucket_folder.")
-@click.option('--exclude-extensions', '-e', multiple=True, help="File extensions to exclude, e.g., '.txt', '.log'")
-@click.option('--exclude-files', '-f', multiple=True, help="Specific files to exclude, e.g., 'config.json'")
-@click.option('--region-name', default='ap-southeast-2', show_default=True, help="AWS region name")
+@click.argument("src_folder", type=click.Path(exists=True, file_okay=False))
+@click.argument("s3_bucket")
+@click.argument("s3_bucket_folder")
+@click.option(
+    "--upload-folder",
+    default=False,
+    is_flag=True,
+    help="Upload the whole folder to specified s3_bucket_folder.",
+)
+@click.option(
+    "--exclude-extensions",
+    "-e",
+    multiple=True,
+    help="File extensions to exclude, e.g., '.txt', '.log'",
+)
+@click.option(
+    "--exclude-files",
+    "-f",
+    multiple=True,
+    help="Specific files to exclude, e.g., 'config.json'",
+)
+@click.option(
+    "--region-name", default="ap-southeast-2", show_default=True, help="AWS region name"
+)
 def upload_files_in_folder_to_s3(
-        src_folder : str,
-        s3_bucket : str,
-        s3_bucket_folder : str,
-        upload_folder : bool,
-        exclude_extensions : list[str] = [],
-        exclude_files : list[str] = [],
-        region_name : str = 'ap-southeast-2',
+    src_folder: str,
+    s3_bucket: str,
+    s3_bucket_folder: str,
+    upload_folder: bool,
+    exclude_extensions: list[str] = [],
+    exclude_files: list[str] = [],
+    region_name: str = "ap-southeast-2",
 ):
     push_files_in_folder_to_s3(
-        src_folder = src_folder,
-        s3_bucket = s3_bucket,
-        s3_bucket_folder = s3_bucket_folder, 
-        upload_folder = upload_folder,
-        exclude_extensions = exclude_extensions,
-        exclude_files = exclude_files,
-        region_name = region_name,
-    )
+        src_folder=src_folder,
+        s3_bucket=s3_bucket,
+        s3_bucket_folder=s3_bucket_folder,
+        upload_folder=upload_folder,
+        exclude_extensions=exclude_extensions,
+        exclude_files=exclude_files,
+        region_name=region_name,
+    )

sar_antarctica/nci/filesystem.py → sar_pipeline/nci/filesystem.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
-from sar_antarctica.nci.preparation.orbits import find_orbits
-from sar_antarctica.nci.preparation.dem import get_cop30_dem_for_bounds
+from sar_pipeline.nci.preparation.orbits import find_orbits
+from sar_pipeline.nci.preparation.dem import get_cop30_dem_for_bounds
 
 
 def get_orbits_nci(

sar_antarctica/nci/preparation/dem.py → sar_pipeline/nci/preparation/dem.py

@@ -9,14 +9,14 @@
 
 logger = logging.getLogger(__name__)
 
-from sar_antarctica.utils.spatial import BoundingBox, get_local_utm, adjust_bounds
-from sar_antarctica.utils.raster import reproject_raster, merge_arrays_with_geometadata
-from sar_antarctica.nci.preparation.dem_cop_glo30 import (
+from sar_pipeline.utils.spatial import BoundingBox, get_local_utm, adjust_bounds
+from sar_pipeline.utils.raster import reproject_raster, merge_arrays_with_geometadata
+from sar_pipeline.nci.preparation.dem_cop_glo30 import (
     get_cop_glo30_files_covering_bounds,
     buffer_bounds_cop_glo30,
     make_empty_cop_glo30_profile_for_bounds,
 )
-from sar_antarctica.nci.preparation.geoid import remove_geoid
+from sar_pipeline.nci.preparation.geoid import remove_geoid
 
 # Create a custom type that allows use of BoundingBox or tuple(xmin, ymin, xmax, ymax)
 BBox = BoundingBox | tuple[float | int, float | int, float | int, float | int]

sar_antarctica/nci/preparation/dem_cop_glo30.py → sar_pipeline/nci/preparation/dem_cop_glo30.py

@@ -3,7 +3,7 @@
 import numpy as np
 from rasterio.crs import CRS
 import rasterio.features
-from sar_antarctica.utils.raster import (
+from sar_pipeline.utils.raster import (
     adjust_pixel_coordinate_from_point_to_area,
     expand_bounding_box_to_pixel_edges,
 )
@@ -13,7 +13,7 @@
 
 logger = logging.getLogger(__name__)
 
-from sar_antarctica.utils.spatial import BoundingBox
+from sar_pipeline.utils.spatial import BoundingBox
 
 
 def buffer_bounds_cop_glo30(

sar_antarctica/nci/preparation/orbits.py → sar_pipeline/nci/preparation/orbits.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 import re
 
-from sar_antarctica.nci.preparation.scenes import (
+from sar_pipeline.nci.preparation.scenes import (
     parse_scene_file_dates,
 )
 

sar_antarctica/nci/processing/pyroSAR/pyrosar_geocode.py → sar_pipeline/nci/processing/pyroSAR/pyrosar_geocode.py

@@ -7,7 +7,7 @@
 import shutil
 import sys
 
-from sar_antarctica.nci.processing.GAMMA.GAMMA_utils import set_gamma_env_variables
+from sar_pipeline.nci.processing.GAMMA.GAMMA_utils import set_gamma_env_variables
 
 logging.basicConfig(
     format="%(asctime)s | %(levelname)s : %(message)s",

sar_antarctica/nci/submission/pyrosar_gamma/prepare_input.py → sar_pipeline/nci/submission/pyrosar_gamma/prepare_input.py

@@ -2,8 +2,8 @@
 from pathlib import Path
 from pyroSAR import identify
 
-from sar_antarctica.nci.preparation.orbits import find_latest_orbit_covering_window
-from sar_antarctica.nci.filesystem import get_orbits_nci, get_dem_nci
+from sar_pipeline.nci.preparation.orbits import find_latest_orbit_covering_window
+from sar_pipeline.nci.filesystem import get_orbits_nci, get_dem_nci
 
 
 def get_orbit_and_dem(

sar_antarctica/nci/submission/pyrosar_gamma/submit_job.py → sar_pipeline/nci/submission/pyrosar_gamma/submit_job.py

@@ -1,6 +1,6 @@
 import os
 from pathlib import Path
-from sar_antarctica.nci.submission.utils import populate_pbs_template
+from sar_pipeline.nci.submission.utils import populate_pbs_template
 
 
 ENVIRONMENT_COMMAND = """

tests/filesystem/test_filesystem.py

@@ -1,5 +1,5 @@
-from sar_antarctica.nci.preparation.orbits import find_latest_orbit_for_scene
-from sar_antarctica.nci.preparation.scenes import find_scene_file_from_id
+from sar_pipeline.nci.preparation.orbits import find_latest_orbit_for_scene
+from sar_pipeline.nci.preparation.scenes import find_scene_file_from_id
 
 import dataclasses
 from datetime import datetime

tests/sar_antarctica/test_dem.py → tests/sar_pipeline/test_dem.py

@@ -1,12 +1,12 @@
-from sar_antarctica.nci.preparation.dem import (
+from sar_pipeline.nci.preparation.dem import (
     get_cop30_dem_for_bounds,
     check_s1_bounds_cross_antimeridian,
     get_target_antimeridian_projection,
     split_s1_bounds_at_am_crossing,
     adjust_bounds_at_high_lat,
     find_required_dem_paths_from_index,
 )
-from sar_antarctica.nci.preparation.create_dem_vrt import find_tiles, build_tileindex
+from sar_pipeline.nci.preparation.create_dem_vrt import find_tiles, build_tileindex
 from dataclasses import dataclass
 import rasterio
 from pathlib import Path
@@ -19,7 +19,7 @@
 FOLDER_PATH = CURRENT_DIR / "data/copernicus_30m_world"
 GEOID_PATH = (
     CURRENT_DIR
-    / "data/geoid/tests/sar_antarctica/data/geoid/us_nga_egm2008_1_4326__agisoft_clipped.tif"
+    / "data/geoid/tests/sar_pipeline/data/geoid/us_nga_egm2008_1_4326__agisoft_clipped.tif"
 )
 TMP_PATH = CURRENT_DIR / "TMP"
 

tests/sar_antarctica/test_dem_cop_glo30.py → tests/sar_pipeline/test_dem_cop_glo30.py

@@ -7,7 +7,7 @@
 from rasterio.crs import CRS
 from pathlib import Path
 
-from sar_antarctica.nci.preparation.dem_cop_glo30 import (
+from sar_pipeline.nci.preparation.dem_cop_glo30 import (
     get_cop_glo30_spacing,
     get_cop_glo30_tile_transform,
     make_empty_cop_glo30_profile_for_bounds,

tests/sar_antarctica/test_dem_vrt.py → tests/sar_pipeline/test_dem_vrt.py

@@ -1,4 +1,4 @@
-from sar_antarctica.nci.preparation.create_dem_vrt import (
+from sar_pipeline.nci.preparation.create_dem_vrt import (
     find_tiles,
     build_vrt,
     build_tileindex,

tests/sar_antarctica/test_orbits.py → tests/sar_pipeline/test_orbits.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 import pytest
 
-from sar_antarctica.nci.preparation.orbits import (
+from sar_pipeline.nci.preparation.orbits import (
     find_orbits,
     parse_orbit_file_dates,
     filter_orbits_to_cover_time_window,

tests/sar_antarctica/test_scenes.py → tests/sar_pipeline/test_scenes.py

@@ -1,5 +1,5 @@
 import datetime
-from sar_antarctica.nci.preparation.scenes import (
+from sar_pipeline.nci.preparation.scenes import (
     parse_scene_file_dates,
     parse_scene_file_sensor,
 )