architecture changed

Author: PennyHow

setup.py

@@ -41,10 +41,10 @@
     'console_scripts': [
         'get_promice_data = pypromice.get.get_promice_data:get_promice_data',
         'get_l0tx = pypromice.tx.get_l0tx:get_l0tx',
-        'join_l2 = pypromice.process.join_l2:main',
-        'join_l3 = pypromice.process.join_l3:main',
-        'get_l2 = pypromice.process.get_l2:main',
-        'get_l2tol3 = pypromice.process.get_l2tol3:main',
+        'join_l2 = pypromice.pipeline.join_l2:main',
+        'join_l3 = pypromice.pipeline.join_l3:main',
+        'get_l2 = pypromice.pipeline.get_l2:main',
+        'get_l2tol3 = pypromice.pipeline.get_l2tol3:main',
         'make_metadata_csv = pypromice.postprocess.make_metadata_csv:main',
         'get_watsontx = pypromice.tx.get_watsontx:get_watsontx',
         'get_bufr = pypromice.postprocess.get_bufr:main',

src/pypromice/pipeline/__init__.py

@@ -0,0 +1 @@
+from pypromice.pipeline.aws import *

src/pypromice/process/aws.py src/pypromice/pipeline/aws.pysrc/pypromice/process/aws.py renamed to src/pypromice/pipeline/aws.py

@@ -16,9 +16,9 @@
 
 
 import pypromice.resources
-from pypromice.process.L0toL1 import toL1
-from pypromice.process.L1toL2 import toL2
-from pypromice.process.L2toL3 import toL3
+from pypromice.pipeline.l0_to_l1 import to_l1
+from pypromice.pipeline.l1_to_l2 import to_l2
+from pypromice.pipeline.l2_to_l3 import to_l3
 from pypromice.process import write, load, utilities
 from pypromice.utilities.git import get_commit_hash_and_check_dirty
 
@@ -127,15 +127,15 @@ def getL1(self):
         """Perform L0 to L1 data processing"""
         logger.info("Level 1 processing...")
         self.L0 = [utilities.addBasicMeta(item, self.vars) for item in self.L0]
-        self.L1 = [toL1(item, self.vars) for item in self.L0]
+        self.L1 = [to_l1(item, self.vars) for item in self.L0]
         self.L1A = reduce(xr.Dataset.combine_first, reversed(self.L1))
         self.L1A.attrs["format"] = self.format
 
     def getL2(self):
         """Perform L1 to L2 data processing"""
         logger.info("Level 2 processing...")
 
-        self.L2 = toL2(
+        self.L2 = to_l2(
             self.L1A,
             vars_df=self.vars,
             data_flags_dir=self.data_issues_repository / "flags",
@@ -146,7 +146,7 @@ def getL3(self):
         """Perform L2 to L3 data processing, including resampling and metadata
         and attribute population"""
         logger.info("Level 3 processing...")
-        self.L3 = toL3(self.L2, data_adjustments_dir=self.data_issues_repository / "adjustments")
+        self.L3 = to_l3(self.L2, data_adjustments_dir=self.data_issues_repository / "adjustments")
 
     def loadConfig(self, config_file, inpath):
         """Load configuration from .toml file

src/pypromice/process/get_l2.py src/pypromice/pipeline/get_l2.pysrc/pypromice/process/get_l2.py renamed to src/pypromice/pipeline/get_l2.py

@@ -5,7 +5,7 @@
 from argparse import ArgumentParser
 from pathlib import Path
 
-from pypromice.process.aws import AWS
+from pypromice.pipeline.aws import AWS
 from pypromice.process.write import prepare_and_write
 
 

src/pypromice/process/get_l2tol3.py src/pypromice/pipeline/get_l2tol3.pysrc/pypromice/process/get_l2tol3.py renamed to src/pypromice/pipeline/get_l2tol3.py

@@ -5,7 +5,7 @@
 import xarray as xr
 from argparse import ArgumentParser
 import pypromice
-from pypromice.process.L2toL3 import toL3
+from pypromice.pipeline.l2_to_l3 import to_l3
 import pypromice.resources
 from pypromice.process.write import prepare_and_write
 logger = logging.getLogger(__name__)
@@ -84,7 +84,7 @@ def get_l2tol3(config_folder: Path|str, inpath, outpath, variables, metadata, da
     data_adjustments_dir = data_issues_path / "adjustments"
 
     # Perform Level 3 processing
-    l3 = toL3(l2, data_adjustments_dir, station_config)
+    l3 = to_l3(l2, data_adjustments_dir, station_config)
 
     # Write Level 3 dataset to file if output directory given
     v = pypromice.resources.load_variables(variables)

src/pypromice/process/join_l2.py src/pypromice/pipeline/join_l2.pysrc/pypromice/process/join_l2.py renamed to src/pypromice/pipeline/join_l2.py

@@ -3,7 +3,7 @@
 import pandas as pd
 import xarray as xr
 from argparse import ArgumentParser
-from pypromice.process.L1toL2 import correctPrecip
+from pypromice.pipeline.l1_to_l2 import correctPrecip
 from pypromice.process.write import prepare_and_write
 logger = logging.getLogger(__name__)
 

src/pypromice/process/join_l3.py src/pypromice/pipeline/join_l3.pysrc/pypromice/process/join_l3.py renamed to src/pypromice/pipeline/join_l3.py

@@ -11,7 +11,7 @@
 logger = logging.getLogger(__name__)
 
 
-def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100):
+def to_l1(L0, vars_df, T_0=273.15, tilt_threshold=-100):
     '''Process one Level 0 (L0) product to Level 1
 
     Parameters

src/pypromice/process/L0toL1.py src/pypromice/pipeline/l0_to_l1.pysrc/pypromice/process/L0toL1.py renamed to src/pypromice/pipeline/l0_to_l1.py

@@ -21,7 +21,7 @@
 logger = logging.getLogger(__name__)
 
 
-def toL2(
+def to_l2(
     L1: xr.Dataset,
     vars_df: pd.DataFrame,
     data_flags_dir: Path,

src/pypromice/process/L1toL2.py src/pypromice/pipeline/l1_to_l2.pysrc/pypromice/process/L1toL2.py renamed to src/pypromice/pipeline/l1_to_l2.py

@@ -13,7 +13,7 @@
 
 logger = logging.getLogger(__name__)
 
-def toL3(L2,
+def to_l3(L2,
          data_adjustments_dir: Path,
          station_config={},
          T_0=273.15):