fix: collection configs (#130)

botanical · Jennifer Tran · web-flow · commit eed1b3c271e1 · 2024-05-23T14:17:09.000-07:00
* fix: campire-data transfer config

* fix: lis-global-da-gpp-trend transfer-config and discovery-items

* fix: update snow-projection configs and lis-global-da-tws-trend

---------

Co-authored-by: Jennifer Tran &lt;tranjennif@gmail.com&gt;
diff --git a/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json b/ingestion-data/production/discovery-items/lis-global-da-gpp-trend.json
@@ -1,8 +1,28 @@
-{
-    "collection": "lis-global-da-gpp-trend",
-    "bucket": "veda-data-store",
-    "prefix": "lis-global-da-gpp-trend/",
-    "filename_regex": "^(.*)DAGPP_STL_based_trend",
-    "discovery": "s3",
-    "upload": false
-}
+[
+    {
+        "collection": "lis-global-da-gpp-trend",
+        "bucket": "veda-data-store",
+        "prefix": "lis-global-da-gpp-trend/",
+        "filename_regex": "^(.*)DAGPP_STL_based_trend_v3_2003_2021.cog.tif",
+        "discovery": "s3",
+        "upload": false
+    },
+    {
+        "collection": "lis-global-da-gpp-trend",
+        "bucket": "veda-data-store",
+        "prefix": "lis-global-da-gpp-trend/",
+        "filename_regex": "^(.*)DAGPP_STL_based_trendv2_2003_2021.cog.tif",
+        "discovery": "s3",
+        "upload": false
+    },
+    {
+        "collection": "lis-global-da-gpp-trend",
+        "bucket": "veda-data-store",
+        "prefix": "lis-global-da-gpp-trend/",
+        "filename_regex": "^(.*)DAGPP_STL_based_trend.cog.tif",
+        "discovery": "s3",
+        "upload": false,
+        "start_datetime": "2003-01-01T00:00:00Z",
+        "end_datetime": "2021-12-31T23:59:59Z"
+    }
+]
diff --git a/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json b/ingestion-data/production/discovery-items/lis-global-da-tws-trend.json
@@ -21,6 +21,8 @@
         "prefix": "lis-global-da-tws-trend/",
         "filename_regex":"^(.*)DATWS_STL_based_trend.cog.tif",
         "discovery": "s3",
-        "upload": false
+        "upload": false,
+        "start_datetime": "2003-01-01T00:00:00Z",
+        "end_datetime": "2021-12-31T23:59:59Z"
     }
 ]
diff --git a/ingestion-data/production/discovery-items/snow-projections-diff-585.json b/ingestion-data/production/discovery-items/snow-projections-diff-585.json
@@ -1,7 +1,7 @@
 [
     {
         "collection": "snow-projections-diff-585",
-        "prefix": "snow-projections-diff-245/",
+        "prefix": "snow-projections-diff-585/",
         "bucket": "veda-data-store",
         "filename_regex": "^(.*)ssp585(.*)percChange.cog.tif$",
         "discovery": "s3",
diff --git a/ingestion-data/production/discovery-items/snow-projections-median-245.json b/ingestion-data/production/discovery-items/snow-projections-median-245.json
@@ -1,7 +1,7 @@
 [
     {
         "collection": "snow-projections-median-245",
-        "prefix": "snow-projections-diff-245/",
+        "prefix": "snow-projections-median-245/",
         "bucket": "veda-data-store",
         "filename_regex": "^(.*)ssp245(.*)median.cog.tif$",
         "discovery": "s3",
diff --git a/ingestion-data/production/discovery-items/snow-projections-median-585.json b/ingestion-data/production/discovery-items/snow-projections-median-585.json
@@ -1,7 +1,7 @@
 [
     {
         "collection": "snow-projections-median-585",
-        "prefix": "snow-projections-diff-245/",
+        "prefix": "snow-projections-median-585/",
         "bucket": "veda-data-store",
         "filename_regex": "^(.*)ssp585(.*)median.cog.tif$",
         "discovery": "s3",
diff --git a/ingestion-data/transfer-config/campfire-nlcd.json b/ingestion-data/transfer-config/campfire-nlcd.json
@@ -4,6 +4,6 @@
         "origin_prefix": "campfire-data/",
         "target_bucket": "veda-data-store",
         "collection": "campfire-nlcd",
-        "filename_regex": "^(.*)CampFire_NLCD_2019"
+        "filename_regex": "^(.*)CampFire_NLCD_201\\d"
     }
 ]
diff --git a/ingestion-data/transfer-config/lis-global-da-gpp-trend.json b/ingestion-data/transfer-config/lis-global-da-gpp-trend.json
@@ -4,6 +4,20 @@
         "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/",
         "target_bucket": "veda-data-store",
         "collection": "lis-global-da-gpp-trend",
-        "filename_regex": "^(.*)DAGPP_STL_based_trend"
+        "filename_regex": "^(.*)DAGPP_STL_based_trend.cog.tif"
+    },
+    {
+        "origin_bucket": "veda-data-store-staging",
+        "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/",
+        "target_bucket": "veda-data-store",
+        "collection": "lis-global-da-gpp-trend",
+        "filename_regex": "^(.*)DAGPP_STL_based_trend_v3_2003_2021.cog.tif"
+    },
+    {
+        "origin_bucket": "veda-data-store-staging",
+        "origin_prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/",
+        "target_bucket": "veda-data-store",
+        "collection": "lis-global-da-gpp-trend",
+        "filename_regex": "^(.*)DAGPP_STL_based_trendv2_2003_2021.cog.tif"
     }
 ]

Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,8 @@`
`21`	`21`	`"prefix": "lis-global-da-tws-trend/",`
`22`	`22`	`"filename_regex":"^(.*)DATWS_STL_based_trend.cog.tif",`
`23`	`23`	`"discovery": "s3",`
`24`		`- "upload": false`
	`24`	`+ "upload": false,`
	`25`	`+ "start_datetime": "2003-01-01T00:00:00Z",`
	`26`	`+ "end_datetime": "2021-12-31T23:59:59Z"`
`25`	`27`	`}`
`26`	`28`	`]`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`[`
`2`	`2`	`{`
`3`	`3`	`"collection": "snow-projections-diff-585",`
`4`		`- "prefix": "snow-projections-diff-245/",`
	`4`	`+ "prefix": "snow-projections-diff-585/",`
`5`	`5`	`"bucket": "veda-data-store",`
`6`	`6`	`"filename_regex": "^(.)ssp585(.)percChange.cog.tif$",`
`7`	`7`	`"discovery": "s3",`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`[`
`2`	`2`	`{`
`3`	`3`	`"collection": "snow-projections-median-245",`
`4`		`- "prefix": "snow-projections-diff-245/",`
	`4`	`+ "prefix": "snow-projections-median-245/",`
`5`	`5`	`"bucket": "veda-data-store",`
`6`	`6`	`"filename_regex": "^(.)ssp245(.)median.cog.tif$",`
`7`	`7`	`"discovery": "s3",`
Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,6 @@`
`4`	`4`	`"origin_prefix": "campfire-data/",`
`5`	`5`	`"target_bucket": "veda-data-store",`
`6`	`6`	`"collection": "campfire-nlcd",`
`7`		`- "filename_regex": "^(.*)CampFire_NLCD_2019"`
	`7`	`+ "filename_regex": "^(.*)CampFire_NLCD_201\\d"`
`8`	`8`	`}`
`9`	`9`	`]`