revisited qc for argo temp and salt

parm/jcb-gdas/observations/marine/insitu_salt_profile_argo.yaml.j2

@@ -29,168 +29,203 @@
     vertical coordinate: sea_water_depth
     observation vertical coordinate: depth
     interpolation method: linear
-    obs error:
+
+  obs error:
     covariance model: diagonal
 
-  #-------------------------------------------------------------------------------
-  #                   START OF OBS FILTERS (work done by Kriti Bhargava)
-  # The QC filters used here are based on the document by IODE that can be found at
-  # https://cdn.ioos.noaa.gov/media/2017/12/recommendations_in_situ_data_real_time_qc.pdf
-  #-------------------------------------------------------------------------------
-
-  obs filters:
-
-    # land check
-    - filter: Domain Check
-      where:
-      - variable: {name: GeoVaLs/sea_area_fraction}
-        value: is_valid
-        minvalue: 0.5
-
-
-  ## Filters for S:
-  #-------------------------------------------------------------------------------
-    #-----------------------------------------------------------------------------
-    ### Global range test
-    #-----------------------------------------------------------------------------
-
-    - filter: Bounds Check
-      filter variables: [{name: salinity}]
-      minvalue: 2.0
-      maxvalue: 41.0
-
-    #-----------------------------------------------------------------------------
-    ### Regional range test
-    #-----------------------------------------------------------------------------
-    #### Red Sea
-    #-----------------------------------------------------------------------------
-    ####
-    #### the document linked here describes Red sea as the are between 10N, 40E;
-    #### 20N, 50E; 30N, 30E; 10N, 40E. But that would also include Gulf of Aden.
-    #### A more reasonable choice seemed to be a box that extends from 10 N to
-    #### 30 N and 30 E to 45 East .
-
-    - filter: Bounds Check
-      filter variables: [{name: salinity}]
-      minvalue: 2.0
-      maxvalue: 41.0
-      where:
-      - variable:
-          name: MetaData/latitude
-        minvalue: 10
-        maxvalue: 30
-      - variable:
-          name: MetaData/longitude
-        minvalue: 30
-        maxvalue: 45
-
-    #### Mediterranean Sea
-    #-----------------------------------------------------------------------------
-    ##### Area 1/3 for Mediterranean Sea
-    - filter: Bounds Check
-      filter variables: [{name: salinity}]
-      minvalue: 2.0
-      maxvalue: 40.0
-      where:
-      - variable:
-          name: MetaData/latitude
-        minvalue: 30
-        maxvalue: 40
-      - variable:
-          name: MetaData/longitude
-        minvalue: -6
-        maxvalue: 40
-    ##### Area 2/3 for Mediterranean Sea
-    - filter: Bounds Check
-      filter variables: [{name: salinity}]
-      minvalue: 2.0
-      maxvalue: 40.0
-      where:
-      - variable:
-          name: MetaData/latitude
-        minvalue: 40
-        maxvalue: 41.5
-      - variable:
-          name: MetaData/longitude
-        minvalue: 20
-        maxvalue: 30
-    ##### Area 3/3 for Mediterranean Sea
-    - filter: Bounds Check
-      filter variables: [{name: salinity}]
-      minvalue: 2.0
-      maxvalue: 40.0
-      where:
-      - variable:
-          name: MetaData/latitude
-        minvalue: 40
-        maxvalue: 46
-      - variable:
-          name: MetaData/longitude
-        minvalue: 0
-        maxvalue: 20
-
-
-    #### Northwestern shelves
-    #-----------------------------------------------------------------------------
-
-    - filter: Bounds Check
-      filter variables: [{name: salinity}]
-      minvalue: 0.0
-      maxvalue: 37.0
-      where:
-      - variable:
-          name: MetaData/latitude
-        minvalue: 50
-        maxvalue: 60
-      - variable:
-          name: MetaData/longitude
-        minvalue: -20
-        maxvalue: 10
-
-    #### Southwestern shelves
-    #-----------------------------------------------------------------------------
-
-    - filter: Bounds Check
-      filter variables: [{name: salinity}]
-      minvalue: 0.0
-      maxvalue: 38
-      where:
-      - variable:
-          name: MetaData/latitude
-        minvalue: 25
-        maxvalue: 50
-      - variable:
-          name: MetaData/longitude
-        minvalue: -30
-        maxvalue: 0
-
-    #### Arctic Ocean
-    #-----------------------------------------------------------------------------
-
-    - filter: Bounds Check
-      filter variables: [{name: salinity}]
-      minvalue: 2.0
-      maxvalue: 40.0
-      where:
-      - variable:
-          name: MetaData/latitude
-        minvalue: 60
-
-    - filter: Background Check
-      filter variables: [{name: salinity}]
-      threshold: 5.0
-      absolute threshold: 5.0
-
-    - filter: Perform Action
-      action:
-        name: assign error
-        error function:
-          name: ObsFunction/LinearCombination
-          options:
-            variables:
-            - ObsError/salinity
-            coefs:
-            - 100.0
+  obs pre filters:
+  # Pre-filters credits to Kriti Bhargava
+  # Global range test
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    minvalue: 2.0
+    maxvalue: 41.0
+
+  # Regional range test
+  # Red Sea
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    minvalue: 2.0
+    maxvalue: 41.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 10
+      maxvalue: 30
+    - variable:
+        name: MetaData/longitude
+      minvalue: 30
+      maxvalue: 45
+
+  # Mediterranean Sea
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    minvalue: 2.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 30
+      maxvalue: 40
+    - variable:
+        name: MetaData/longitude
+      minvalue: -6
+      maxvalue: 40
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    minvalue: 2.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 41.5
+    - variable:
+        name: MetaData/longitude
+      minvalue: 20
+      maxvalue: 30
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    minvalue: 2.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 40
+      maxvalue: 46
+    - variable:
+        name: MetaData/longitude
+      minvalue: 0
+      maxvalue: 20
+
+  # Northwestern shelves
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    minvalue: 0.0
+    maxvalue: 37.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 50
+      maxvalue: 60
+    - variable:
+        name: MetaData/longitude
+      minvalue: -20
+      maxvalue: 10
+
+  # Southwestern shelves
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    minvalue: 0.0
+    maxvalue: 38
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 25
+      maxvalue: 50
+    - variable:
+        name: MetaData/longitude
+      minvalue: -30
+      maxvalue: 0
+
+  # Arctic
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    minvalue: 2.0
+    maxvalue: 40.0
+    where:
+    - variable:
+        name: MetaData/latitude
+      minvalue: 60
+
+  obs prior filters:
+  - filter: Domain Check
+    where:
+    - variable:
+        name: GeoVaLs/sea_area_fraction
+      value: is_valid
+      minvalue: 0.5
+
+  # Reject obs where MetaData/depth > GeoVaLs/sea_floor_depth_below_sea_surface - maxvalue
+  - filter: Bounds Check
+    filter variables:
+    - name: salinity
+    test variables:
+    - name: ObsFunction/Arithmetic
+      options:
+        variables: [MetaData/depth, GeoVaLs/sea_floor_depth_below_sea_surface]
+        coefs: [1.0, -1.0]   # depth - sea_floor_depth_below_sea_surface
+    maxvalue: -100.0         # i.e., depth - sea_floor_depth_below_sea_surface <= 0
+
+  # Reject stair-step profiles (rounded values)
+  - filter: Create Diagnostic Flags
+    filter variables:
+    - name: salinity
+    flags:
+    - name: ProfileSpike
+      initial value: false
+    - name: ProfileStep
+      initial value: false
+
+  - filter: Spike and Step Check
+    independent: MetaData/depth
+    dependent: ObsValue/salinity
+    count spikes: false
+    count steps: true
+    tolerance:
+      nominal value: 0.05                   # tune: < 0.1 if obs rounded to 0.1
+    action:
+      name: reject
+
+  obs post filters:
+  # Reject profiles with large differences from background
+  - filter: Profile Background Check
+    filter variables:
+    - name: salinity
+    absolute threshold: 0.5
+
+  # Reject obs that are 5PSU away from the background
+  - filter: Background Check
+    filter variables:
+    - name: salinity
+    absolute threshold: 5.0
+
+  # Force constant observation error everywhere
+  - filter: Perform Action
+    filter variables:
+    - name: salinity
+    action:
+      name: assign error
+      error parameter: 0.05
+
+  # Linear decay from surface to 2000m depth:
+  #    - minimum error of 0.1K at 2000m (and below)
+  #    - maximum error of 1.0K at surface
+  # original error is rescaled and added to reflect the out of window error
+  - filter: Perform Action
+    where:
+    - variable:
+        name: MetaData/depth
+      maxvalue: 2000.0
+    action:
+      name: assign error
+      error function:
+        name: ObsFunction/Arithmetic
+        options:
+          variables: [MetaData/depth, ObsError/salinity]
+          # sigo_total = 0.5 + (-0.00045)*depth + 0.1*sigo_original
+          coefs:     [-0.00025, 0.1]
+          intercept: 0.5
+
+
 {% if marine_letkf_app | default(false) %}
   obs localizations:
   - localization method: Rossby