Skip to content

Commit f1e107e

Browse files
tnagorratnagorra
committed
Move usage of Geocoder in MontyGeoCoder
- Refactor models for ETL - Use TypedDict for IFRCEvent params - Create abstract model Tracing - Update admin panel settings for ETL - Remove validate_source_func - Update use of library ee (pyright not reading re-exports)
1 parent fa46712 commit f1e107e

File tree

23 files changed

+375
-323
lines changed

23 files changed

+375
-323
lines changed

‎apps/common/models.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
from django.db import models
22

33

4+
class Tracing(models.Model):
5+
trace_id = models.UUIDField(editable=False, default="00000000-0000-0000-0000-000000000000", db_index=True)
6+
7+
class Meta:
8+
abstract = True
9+
10+
411
class Resource(models.Model):
512
created_at = models.DateTimeField(auto_now_add=True)
613
modified_at = models.DateTimeField(auto_now=True)
@@ -10,4 +17,3 @@ class Resource(models.Model):
1017

1118
class Meta:
1219
abstract = True
13-
ordering = ["-id"]

‎apps/etl/admin.py‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,20 @@ def get_readonly_fields(self, request, obj=None):
1313
return []
1414

1515
list_display = (
16+
"trace_id",
1617
"id",
18+
"parent__id",
1719
"source",
18-
"resp_code",
20+
"metadata",
1921
"status",
20-
"parent__id",
21-
"resp_data_type",
2222
"source_validation_status",
23+
"resp_code",
24+
"resp_data",
25+
"resp_data_type",
2326
"hazard_type",
24-
"trace_id",
2527
"created_at",
2628
)
27-
list_filter = ("status", "source")
29+
list_filter = ("status", "source", "source_validation_status")
2830
autocomplete_fields = ["parent"]
2931
search_fields = ["parent"]
3032

@@ -37,7 +39,14 @@ def get_readonly_fields(self, request, obj=None):
3739
return [field.name for field in self.model._meta.fields]
3840
return []
3941

40-
list_display = ("id", "extraction", "status", "trace_id", "is_loaded")
42+
list_display = (
43+
"trace_id",
44+
"id",
45+
"extraction",
46+
"status",
47+
"is_loaded",
48+
"created_at",
49+
)
4150
list_filter = ("status",)
4251
autocomplete_fields = ["extraction"]
4352
search_fields = ["extraction"]
@@ -52,13 +61,17 @@ def get_readonly_fields(self, request, obj=None):
5261
return []
5362

5463
list_display = (
64+
"trace_id",
5565
"id",
5666
"transform_id",
67+
"collection_id",
68+
"item_type",
69+
"load_status",
70+
"created_at",
71+
)
72+
list_filter = (
5773
"item_type",
5874
"load_status",
59-
"collection_id",
60-
"trace_id",
6175
)
62-
list_filter = ("load_status",)
6376
autocomplete_fields = ["transform_id"]
6477
search_fields = ["transform_id"]

‎apps/etl/etl_tasks/desinventar.py‎

Lines changed: 110 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -8,124 +8,126 @@
88

99
logger = logging.getLogger(__name__)
1010

11+
# list of countries which has same country_code and iso3
12+
country_code_iso3_list: list[str] = [
13+
"ago",
14+
"alb",
15+
"arg",
16+
"arm",
17+
"atg",
18+
"bfa",
19+
"bdi",
20+
"blr",
21+
"blz",
22+
"bol",
23+
"brb",
24+
"btn",
25+
"chl",
26+
"col",
27+
"com",
28+
"cpv",
29+
"cri",
30+
"dji",
31+
"dma",
32+
"dom",
33+
"ecu",
34+
"egy",
35+
"esp",
36+
"eth",
37+
"gha",
38+
"gin",
39+
"gmb",
40+
"gnb",
41+
"gnq",
42+
"grd",
43+
"gtm",
44+
"guy",
45+
"hnd",
46+
"idn",
47+
"irn",
48+
"irq",
49+
"jam",
50+
"jor",
51+
"ken",
52+
"khm",
53+
"kna",
54+
"lao",
55+
"lbn",
56+
"lbr",
57+
"lca",
58+
"lka",
59+
"mar",
60+
"mdg",
61+
"mdv",
62+
"mex",
63+
"mli",
64+
"mmr",
65+
"mne",
66+
"mng",
67+
"moz",
68+
"mus",
69+
"mwi",
70+
"nam",
71+
"ner",
72+
"nga",
73+
"nic",
74+
"npl",
75+
"pac",
76+
"pak",
77+
"pan",
78+
"per",
79+
"prt",
80+
"pry",
81+
"pse",
82+
"rwa",
83+
"sdn",
84+
"sen",
85+
"sle",
86+
"slv",
87+
"som",
88+
"srb",
89+
"swz",
90+
"syc",
91+
"syr",
92+
"tgo",
93+
"tls",
94+
"tto",
95+
"tun",
96+
"tur",
97+
"tza",
98+
"uga",
99+
"ury",
100+
"vct",
101+
"ven",
102+
"vnm",
103+
"xkx",
104+
"yem",
105+
"zmb",
106+
]
11107

12-
@shared_task
13-
def ext_and_transform_desinventar_data():
14-
# list of countries which has same country_code and iso3
15-
country_code_iso3_list = [
16-
"ago",
17-
"alb",
18-
"arg",
19-
"arm",
20-
"atg",
21-
"bfa",
22-
"bdi",
23-
"blr",
24-
"blz",
25-
"bol",
26-
"brb",
27-
"btn",
28-
"chl",
29-
"col",
30-
"com",
31-
"cpv",
32-
"cri",
33-
"dji",
34-
"dma",
35-
"dom",
36-
"ecu",
37-
"egy",
38-
"esp",
39-
"eth",
40-
"gha",
41-
"gin",
42-
"gmb",
43-
"gnb",
44-
"gnq",
45-
"grd",
46-
"gtm",
47-
"guy",
48-
"hnd",
49-
"idn",
50-
"irn",
51-
"irq",
52-
"jam",
53-
"jor",
54-
"ken",
55-
"khm",
56-
"kna",
57-
"lao",
58-
"lbn",
59-
"lbr",
60-
"lca",
61-
"lka",
62-
"mar",
63-
"mdg",
64-
"mdv",
65-
"mex",
66-
"mli",
67-
"mmr",
68-
"mne",
69-
"mng",
70-
"moz",
71-
"mus",
72-
"mwi",
73-
"nam",
74-
"ner",
75-
"nga",
76-
"nic",
77-
"npl",
78-
"pac",
79-
"pak",
80-
"pan",
81-
"per",
82-
"prt",
83-
"pry",
84-
"pse",
85-
"rwa",
86-
"sdn",
87-
"sen",
88-
"sle",
89-
"slv",
90-
"som",
91-
"srb",
92-
"swz",
93-
"syc",
94-
"syr",
95-
"tgo",
96-
"tls",
97-
"tto",
98-
"tun",
99-
"tur",
100-
"tza",
101-
"uga",
102-
"ury",
103-
"vct",
104-
"ven",
105-
"vnm",
106-
"xkx",
107-
"yem",
108-
"zmb",
109-
]
108+
# list of countries / regions which has different country_code and iso3
109+
additional_region_code_to_iso3_map: dict[str, str] = {
110+
"etm": "tls",
111+
"mal": "mdv",
112+
"sy11": "syr",
113+
"znz": "tza",
114+
"019": "ind",
115+
"033": "ind",
116+
"005": "ind",
117+
}
110118

111-
# list of countries / regions which has different country_code and iso3
112-
additional_region_code_to_iso3_map = {
113-
"etm": "tls",
114-
"mal": "mdv",
115-
"sy11": "syr",
116-
"znz": "tza",
117-
"019": "ind",
118-
"033": "ind",
119-
"005": "ind",
120-
}
121119

120+
@shared_task
121+
def ext_and_transform_desinventar_data():
122122
for country_code in country_code_iso3_list:
123+
# FIXME: Not sure if URL should be passed from outside the Extraction function
123124
DATA_URL = f"{settings.DESINVENTAR_DATA_URL}/DesInventar/download/DI_export_{country_code}.zip"
124125
chain(
125126
DesinventarExtraction.task.s(DATA_URL),
126127
DesinventarTransformHandler.task.s(country_code, country_code),
127128
).apply_async()
128129

130+
# FIXME: Rename country_code to region_code
129131
for country_code, iso3 in additional_region_code_to_iso3_map.items():
130132
DATA_URL = f"{settings.DESINVENTAR_DATA_URL}/DesInventar/download/DI_export_{country_code}.zip"
131133
chain(

‎apps/etl/etl_tasks/gfd.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from datetime import datetime
1+
from datetime import date, datetime
22

33
from celery import chain, shared_task
44
from django.conf import settings
@@ -26,11 +26,11 @@ def ext_and_transform_gfd_latest_data():
2626
.first()
2727
)
2828

29-
end_date = datetime.now().date()
3029
if ext_object:
31-
start_date = ext_object.created_at.date()
30+
start_date: date = ext_object.created_at.date()
3231
else:
3332
start_date = datetime.strptime(settings.GFD_START_DATE, "%Y-%m-%d").date()
33+
end_date = datetime.now().date()
3434

3535
chain(
3636
GFDExtraction.task.s(start_date, end_date),

‎apps/etl/etl_tasks/idu.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,6 @@
88

99
logger = logging.getLogger(__name__)
1010

11-
HISTORICAL_DATA_URL = f"{settings.IDMC_DATA_URL}/external-api/idus/all/"
12-
LATEST_DATA_URL = f"{settings.IDMC_DATA_URL}/external-api/idus/last-180-days/"
13-
1411

1512
@shared_task
1613
def extract_and_transform_idu_data(url):
@@ -24,10 +21,12 @@ def extract_and_transform_idu_data(url):
2421
@shared_task
2522
def ext_and_transform_idu_historical_data():
2623
"""Extract and Transform IDU historical data"""
27-
extract_and_transform_idu_data(HISTORICAL_DATA_URL)
24+
url = f"{settings.IDMC_DATA_URL}/external-api/idus/all/"
25+
extract_and_transform_idu_data(url)
2826

2927

3028
@shared_task
3129
def ext_and_transform_idu_latest_data():
3230
"""Extract and Transform IDU latest data"""
33-
extract_and_transform_idu_data(LATEST_DATA_URL)
31+
url = f"{settings.IDMC_DATA_URL}/external-api/idus/last-180-days/"
32+
extract_and_transform_idu_data(url)

0 commit comments

Comments
 (0)