Skip to content

Commit 371ec8c

Browse files
p-phungp-phung
authored
Merge pull request #53 from rodekruis/dev
Dev
2 parents 747e788 + 8034a49 commit 371ec8c

File tree

2 files changed

+115
-30
lines changed

2 files changed

+115
-30
lines changed

‎README.md‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ Furthermore, for the ECMWF extractors one needs to accept the *Terms of use*.
6363
This is done on the ECMWF website and requires you to be logged in (corresponding to the account of the api-key).
6464
For example, for the extreme heat dataset see: https://cds.climate.copernicus.eu/datasets/derived-utci-historical?tab=download.
6565

66+
#### CHIRPS
67+
TBA
68+
69+
#### WorldPop
70+
TBA
71+
6672
#### IOM DTM
6773
IOM DTM data requires API key to access data. See more about the [DTM API](https://dtm.iom.int/data-and-analysis/dtm-api) latest version V3.
6874
Set the api_key in environment variable `DTM_API_KEY` in the `.env` file.
@@ -71,13 +77,14 @@ Set the api_key in environment variable `DTM_API_KEY` in the `.env` file.
7177
For IPC extractor, API key is needed.
7278
Set api key: TBA
7379

80+
#### FEWS NET
81+
TBA
82+
7483
### Data transformation
7584

76-
While the retrieval pipeline focuses on downloading raw data, the transform pipeline turns those
77-
datasets into district‑level climate indices suitable for multi‑criteria prioritisation. It operates on
85+
After extractions, the transformations turns those datasets into district‑level suitable for multi‑criteria prioritisation. It operates on
7886
three broad climate hazards—drought, flood and extreme heat—and produces baseline, recent and
79-
trend indicators for each. These pipelines live in the `transformpipeline/` package and are
80-
assembled from reusable `Step` classes defined in `transformpipeline.core`.
87+
trend indicators for each, as well as other transformations.
8188

8289
### Indicators
8390

@@ -94,6 +101,9 @@ The table below summarises what each climate indicator measures and the datasets
94101
| **Drought trend** | Evaluates 30‑year trends in drought risk by fitting linear regressions to time series of drought frequency, intensity and persistency. Negative slopes are clipped to zero before normalisation. | CHIRPS monthly NetCDF; SPI‑based metrics are computed for each year, trends estimated by linear regression and normalised before combination. |
95102
| **Flood trend** | Tracks trends in flood extent over 1984–2021 by fitting a linear regression to annual flood percentages. Negative trends are set to zero and positive slopes normalised. | JRC Monthly Water History v1.4 rasters; annual flooded area percentages are computed, trends estimated via linear regression and normalised. |
96103
| **Extreme heat trend** | Assesses trends in extreme heat frequency, intensity and persistency over the last 30 years using UTCI data. Negative trends are clipped and remaining slopes normalised and combined. | ERA5‑derived UTCI hourly NetCDF; annual time series of exceedance frequency/intensity/persistency are built, linear trends computed and normalised. |
104+
| **Displacement** | TBA | TBA |
105+
| **Food insecurity** | TBA | TBA |
106+
| **Population** | TBA | TBA |
97107

98108
## To contribute
99109

logic_app/demo_logic_app.json renamed to logic_app/logic-app-prod.json

Lines changed: 101 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,18 @@
55
"triggers": {
66
"When_an_HTTP_request_is_received": {
77
"type": "Request",
8-
"kind": "Http"
8+
"kind": "Http",
9+
"inputs": {
10+
"method": "POST",
11+
"schema": {
12+
"properties": {
13+
"country": {
14+
"type": "string"
15+
}
16+
},
17+
"type": "object"
18+
}
19+
}
920
}
1021
},
1122
"actions": {
@@ -15,7 +26,7 @@
1526
"inputs": {
1627
"host": {
1728
"connection": {
18-
"name": "@parameters('$connections')['keyvault']['connectionId']"
29+
"name": "@parameters('$connections')['keyvault-1']['connectionId']"
1930
}
2031
},
2132
"method": "get",
@@ -44,6 +55,12 @@
4455
"Get_container-registry-password": [
4556
"Succeeded"
4657
],
58+
"Get_dtm-api-key": [
59+
"Succeeded"
60+
],
61+
"Get_ipc-api-key": [
62+
"Succeeded"
63+
],
4764
"containerIsDone_=_false": [
4865
"Succeeded"
4966
],
@@ -55,7 +72,7 @@
5572
"inputs": {
5673
"host": {
5774
"connection": {
58-
"name": "@parameters('$connections')['aci']['connectionId']"
75+
"name": "@parameters('$connections')['aci-1']['connectionId']"
5976
}
6077
},
6178
"method": "put",
@@ -64,9 +81,17 @@
6481
"properties": {
6582
"containers": [
6683
{
67-
"name": "tjebbe-demo",
84+
"name": "container",
6885
"properties": {
69-
"image": "tjebbedemo-gnh9duaghsdfhyfx.azurecr.io/etl-pipeline:latest",
86+
"command": [
87+
"poetry",
88+
"run",
89+
"python",
90+
"run_pipeline.py",
91+
"--country-iso3",
92+
"@triggerBody()?['country']"
93+
],
94+
"image": "datapipelineregistry-a5eudbdee8djhnh8.azurecr.io/exposure-vulnerability-retrieval:prod",
7095
"environmentVariables": [
7196
{
7297
"name": "BLOB_ACCOUNT_NAME",
@@ -79,6 +104,14 @@
79104
{
80105
"name": "ECMWF_DATASTORES_KEY",
81106
"secureValue": "@body('Get_ecmwf-datastores-key')?['value']"
107+
},
108+
{
109+
"name": "IPC_API_KEY",
110+
"secureValue": "@body('Get_ipc-api-key')?['value']"
111+
},
112+
{
113+
"name": "DTM_API_KEY",
114+
"secureValue": "@body('Get_dtm-api-key')?['value']"
82115
}
83116
],
84117
"resources": {
@@ -93,8 +126,8 @@
93126
"osType": "Linux",
94127
"imageRegistryCredentials": [
95128
{
96-
"server": "tjebbedemo-gnh9duaghsdfhyfx.azurecr.io",
97-
"username": "tjebbedemo",
129+
"server": "datapipelineregistry-a5eudbdee8djhnh8.azurecr.io",
130+
"username": "datapipelineregistry",
98131
"password": "@body('Get_container-registry-password')?['value']"
99132
}
100133
],
@@ -103,11 +136,11 @@
103136
"identity": {
104137
"type": "UserAssigned",
105138
"userAssignedIdentities": {
106-
"/subscriptions/57b0d17a-5429-4dbb-8366-35c928e3ed94/resourceGroups/ibf-data-exploratory-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/tjebbe-demo": {}
139+
"/subscriptions/57b0d17a-5429-4dbb-8366-35c928e3ed94/resourceGroups/exposure-vulnerability/providers/Microsoft.ManagedIdentity/userAssignedIdentities/exposure-etl-managed-identity": {}
107140
}
108141
}
109142
},
110-
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('ibf-data-exploratory-rg')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('tjebbe-demo')}",
143+
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('exposure-vulnerability')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('exposure-vulnerability-retrieval-prod-',toLower(triggerBody()?['country']),'-cg')}",
111144
"queries": {
112145
"x-ms-api-version": "2023-05-01"
113146
}
@@ -123,11 +156,11 @@
123156
"inputs": {
124157
"host": {
125158
"connection": {
126-
"name": "@parameters('$connections')['aci']['connectionId']"
159+
"name": "@parameters('$connections')['aci-1']['connectionId']"
127160
}
128161
},
129162
"method": "get",
130-
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('ibf-data-exploratory-rg')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('tjebbe-demo')}/containers/@{encodeURIComponent('tjebbe-demo')}/logs",
163+
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('exposure-vulnerability')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('exposure-vulnerability-retrieval-prod-',toLower(triggerBody()?['country']),'-cg')}/containers/@{encodeURIComponent('container')}/logs",
131164
"queries": {
132165
"x-ms-api-version": "2023-05-01"
133166
}
@@ -146,11 +179,11 @@
146179
"inputs": {
147180
"host": {
148181
"connection": {
149-
"name": "@parameters('$connections')['aci']['connectionId']"
182+
"name": "@parameters('$connections')['aci-1']['connectionId']"
150183
}
151184
},
152185
"method": "delete",
153-
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('ibf-data-exploratory-rg')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('tjebbe-demo')}",
186+
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('exposure-vulnerability')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('exposure-vulnerability-retrieval-prod-',toLower(triggerBody()?['country']),'-cg')}",
154187
"queries": {
155188
"x-ms-api-version": "2023-05-01"
156189
}
@@ -163,11 +196,11 @@
163196
"inputs": {
164197
"host": {
165198
"connection": {
166-
"name": "@parameters('$connections')['aci']['connectionId']"
199+
"name": "@parameters('$connections')['aci-1']['connectionId']"
167200
}
168201
},
169202
"method": "get",
170-
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('ibf-data-exploratory-rg')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('tjebbe-demo')}",
203+
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('exposure-vulnerability')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('exposure-vulnerability-retrieval-prod-',toLower(triggerBody()?['country']),'-cg')}",
171204
"queries": {
172205
"x-ms-api-version": "2023-05-01"
173206
}
@@ -238,7 +271,7 @@
238271
"inputs": {
239272
"host": {
240273
"connection": {
241-
"name": "@parameters('$connections')['keyvault']['connectionId']"
274+
"name": "@parameters('$connections')['keyvault-1']['connectionId']"
242275
}
243276
},
244277
"method": "get",
@@ -259,7 +292,7 @@
259292
"inputs": {
260293
"host": {
261294
"connection": {
262-
"name": "@parameters('$connections')['keyvault']['connectionId']"
295+
"name": "@parameters('$connections')['keyvault-1']['connectionId']"
263296
}
264297
},
265298
"method": "get",
@@ -280,7 +313,7 @@
280313
"inputs": {
281314
"host": {
282315
"connection": {
283-
"name": "@parameters('$connections')['keyvault']['connectionId']"
316+
"name": "@parameters('$connections')['keyvault-1']['connectionId']"
284317
}
285318
},
286319
"method": "get",
@@ -314,15 +347,57 @@
314347
"inputs": {
315348
"host": {
316349
"connection": {
317-
"name": "@parameters('$connections')['aci']['connectionId']"
350+
"name": "@parameters('$connections')['aci-1']['connectionId']"
318351
}
319352
},
320353
"method": "delete",
321-
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('ibf-data-exploratory-rg')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('tjebbe-demo')}",
354+
"path": "/subscriptions/@{encodeURIComponent('57b0d17a-5429-4dbb-8366-35c928e3ed94')}/resourceGroups/@{encodeURIComponent('exposure-vulnerability')}/providers/Microsoft.ContainerInstance/containerGroups/@{encodeURIComponent('exposure-vulnerability-retrieval-prod-',toLower(triggerBody()?['country']),'-cg')}",
322355
"queries": {
323356
"x-ms-api-version": "2023-05-01"
324357
}
325358
}
359+
},
360+
"Get_dtm-api-key": {
361+
"runAfter": {},
362+
"type": "ApiConnection",
363+
"inputs": {
364+
"host": {
365+
"connection": {
366+
"name": "@parameters('$connections')['keyvault-1']['connectionId']"
367+
}
368+
},
369+
"method": "get",
370+
"path": "/secrets/@{encodeURIComponent('dtm-api-key')}/value"
371+
},
372+
"runtimeConfiguration": {
373+
"secureData": {
374+
"properties": [
375+
"inputs",
376+
"outputs"
377+
]
378+
}
379+
}
380+
},
381+
"Get_ipc-api-key": {
382+
"runAfter": {},
383+
"type": "ApiConnection",
384+
"inputs": {
385+
"host": {
386+
"connection": {
387+
"name": "@parameters('$connections')['keyvault-1']['connectionId']"
388+
}
389+
},
390+
"method": "get",
391+
"path": "/secrets/@{encodeURIComponent('ipc-api-key')}/value"
392+
},
393+
"runtimeConfiguration": {
394+
"secureData": {
395+
"properties": [
396+
"inputs",
397+
"outputs"
398+
]
399+
}
400+
}
326401
}
327402
},
328403
"outputs": {},
@@ -337,20 +412,20 @@
337412
"$connections": {
338413
"type": "Object",
339414
"value": {
340-
"keyvault": {
415+
"keyvault-1": {
341416
"id": "/subscriptions/57b0d17a-5429-4dbb-8366-35c928e3ed94/providers/Microsoft.Web/locations/westeurope/managedApis/keyvault",
342-
"connectionId": "/subscriptions/57b0d17a-5429-4dbb-8366-35c928e3ed94/resourceGroups/ibf-data-exploratory-rg/providers/Microsoft.Web/connections/keyvault",
343-
"connectionName": "keyvault",
417+
"connectionId": "/subscriptions/57b0d17a-5429-4dbb-8366-35c928e3ed94/resourceGroups/exposure-vulnerability/providers/Microsoft.Web/connections/keyvault-1",
418+
"connectionName": "keyvault-1",
344419
"connectionProperties": {
345420
"authentication": {
346421
"type": "ManagedServiceIdentity"
347422
}
348423
}
349424
},
350-
"aci": {
425+
"aci-1": {
351426
"id": "/subscriptions/57b0d17a-5429-4dbb-8366-35c928e3ed94/providers/Microsoft.Web/locations/westeurope/managedApis/aci",
352-
"connectionId": "/subscriptions/57b0d17a-5429-4dbb-8366-35c928e3ed94/resourceGroups/ibf-data-exploratory-rg/providers/Microsoft.Web/connections/aci",
353-
"connectionName": "aci",
427+
"connectionId": "/subscriptions/57b0d17a-5429-4dbb-8366-35c928e3ed94/resourceGroups/exposure-vulnerability/providers/Microsoft.Web/connections/aci-1",
428+
"connectionName": "aci-1",
354429
"connectionProperties": {
355430
"authentication": {
356431
"type": "ManagedServiceIdentity"

0 commit comments

Comments
 (0)