Skip to content

Commit 2460290

Browse files
thommodinthommodin
committed
Added all config WIP
1 parent fea0747 commit 2460290

14 files changed

+1850
-4
lines changed

‎aodn_cloud_optimised/bin/aggregated_kelp_nonqc.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
generic_launcher.py

‎aodn_cloud_optimised/bin/aggregated_seabird_nonqc.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
generic_launcher.py

‎aodn_cloud_optimised/bin/aggregated_seagrass_nonqc.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
generic_launcher.py

‎aodn_cloud_optimised/config/common.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
2-
"BUCKET_RAW_DEFAULT": "imos-data",
3-
"BUCKET_OPTIMISED_DEFAULT": "aodn-cloud-optimised",
4-
"ROOT_PREFIX_CLOUD_OPTIMISED_PATH": "",
2+
"BUCKET_RAW_DEFAULT": "data-uplift-public",
3+
"BUCKET_OPTIMISED_DEFAULT": "data-uplift-public",
4+
"ROOT_PREFIX_CLOUD_OPTIMISED_PATH": "cloud_optimised_testing",
55
"BUCKET_INTEGRATION_TESTING_RAW_DEFAULT": "imos-data",
66
"BUCKET_INTEGRATION_TESTING_OPTIMISED_DEFAULT": "imos-data-lab-optimised",
77
"ROOT_PREFIX_CLOUD_OPTIMISED_INTEGRATION_TESTING_PATH": ""

‎aodn_cloud_optimised/config/dataset/aggregated_kelp_nonqc.json‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
{
2+
"dataset_name": "aggregated_kelp_nonqc",
3+
"logger_name": "aggregated_kelp_nonqc",
4+
"cloud_optimised_format": "parquet",
5+
"run_settings": {
6+
"paths": [
7+
{
8+
"type": "parquet",
9+
"partitioning": null,
10+
"s3_uri": "s3://data-uplift-public/stored/datauplift/kelp/kelp.parquet"
11+
}
12+
],
13+
"cluster": {
14+
"mode": "local",
15+
"restart_every_path": false
16+
},
17+
"clear_existing_data": true,
18+
"raise_error": false,
19+
"batch_size": 4,
20+
"force_previous_parquet_deletion": false
21+
},
22+
"metadata_uuid": null,
23+
"schema": {
24+
"occurrenceID": {
25+
"type": "string",
26+
"nullable": "False"
27+
},
28+
"eventDate": {
29+
"type": "timestamp[ms]",
30+
"nullable": "False"
31+
},
32+
"decimalLatitude": {
33+
"type": "double",
34+
"nullable": "False"
35+
},
36+
"decimalLongitude": {
37+
"type": "double",
38+
"nullable": "False"
39+
},
40+
"label.id": {
41+
"type": "int64",
42+
"nullable": "True"
43+
},
44+
"verbatimIdentification": {
45+
"type": "string",
46+
"nullable": "False"
47+
},
48+
"occurrenceStatus": {
49+
"type": "string",
50+
"nullable": "False"
51+
},
52+
"basisOfRecord": {
53+
"type": "string",
54+
"nullable": "False"
55+
},
56+
"associatedMedia": {
57+
"type": "string",
58+
"nullable": "False"
59+
},
60+
"scientificName": {
61+
"type": "string",
62+
"nullable": "False"
63+
},
64+
"scientificNameID": {
65+
"type": "string",
66+
"nullable": "False"
67+
},
68+
"taxonRank": {
69+
"type": "string",
70+
"nullable": "False"
71+
},
72+
"kingdom": {
73+
"type": "string",
74+
"nullable": "False"
75+
},
76+
"phylum": {
77+
"type": "string",
78+
"nullable": "False"
79+
},
80+
"class": {
81+
"type": "string",
82+
"nullable": "True"
83+
},
84+
"order": {
85+
"type": "string",
86+
"nullable": "True"
87+
},
88+
"family": {
89+
"type": "string",
90+
"nullable": "True"
91+
},
92+
"genus": {
93+
"type": "string",
94+
"nullable": "True"
95+
},
96+
"scientificNameAuthorship": {
97+
"type": "string",
98+
"nullable": "False"
99+
}
100+
},
101+
"aws_opendata_registry": {
102+
"Name": "FILL UP MANUALLY - CHECK DOCUMENTATION",
103+
"Description": "FILL UP MANUALLY - CHECK DOCUMENTATION",
104+
"Documentation": "FILL UP MANUALLY - CHECK DOCUMENTATION",
105+
"Contact": "FILL UP MANUALLY - CHECK DOCUMENTATION",
106+
"ManagedBy": "FILL UP MANUALLY - CHECK DOCUMENTATION",
107+
"UpdateFrequency": "FILL UP MANUALLY - CHECK DOCUMENTATION",
108+
"Tags": [
109+
"FILL UP MANUALLY - CHECK DOCUMENTATION"
110+
],
111+
"License": "FILL UP MANUALLY - CHECK DOCUMENTATION",
112+
"Resources": [
113+
{
114+
"Description": "FILL UP MANUALLY - CHECK DOCUMENTATION",
115+
"ARN": "FILL UP MANUALLY - CHECK DOCUMENTATION",
116+
"Region": "FILL UP MANUALLY - CHECK DOCUMENTATION",
117+
"Type": "FILL UP MANUALLY - CHECK DOCUMENTATION",
118+
"Explore": [
119+
"FILL UP MANUALLY - CHECK DOCUMENTATION"
120+
]
121+
}
122+
],
123+
"DataAtWork": {
124+
"Tutorials": [
125+
{
126+
"Title": "FILL UP MANUALLY - CHECK DOCUMENTATION",
127+
"URL": "FILL UP MANUALLY - CHECK DOCUMENTATION",
128+
"Services": "FILL UP MANUALLY - CHECK DOCUMENTATION",
129+
"AuthorName": "FILL UP MANUALLY - CHECK DOCUMENTATION",
130+
"AuthorURL": "FILL UP MANUALLY - CHECK DOCUMENTATION"
131+
}
132+
],
133+
"Tools & Applications": [
134+
{
135+
"Title": "FILL UP MANUALLY - CHECK DOCUMENTATION",
136+
"URL": "FILL UP MANUALLY - CHECK DOCUMENTATION",
137+
"AuthorName": "FILL UP MANUALLY - CHECK DOCUMENTATION",
138+
"AuthorURL": "FILL UP MANUALLY - CHECK DOCUMENTATION"
139+
}
140+
],
141+
"Publications": [
142+
{
143+
"Title": "FILL UP MANUALLY - CHECK DOCUMENTATION",
144+
"URL": "FILL UP MANUALLY - CHECK DOCUMENTATION",
145+
"AuthorName": "FILL UP MANUALLY - CHECK DOCUMENTATION"
146+
}
147+
]
148+
}
149+
},
150+
"schema_transformation": {
151+
"drop_variables": [],
152+
"add_variables": {
153+
"filename": {
154+
"source": "@filename",
155+
"schema": {
156+
"type": "string",
157+
"units": "1",
158+
"long_name": "Filename of the source file"
159+
}
160+
},
161+
"timestamp": {
162+
"source": "@partitioning:time_extent",
163+
"schema": {
164+
"type": "int64",
165+
"units": "1",
166+
"long_name": "Partition timestamp"
167+
}
168+
},
169+
"polygon": {
170+
"source": "@partitioning:spatial_extent",
171+
"schema": {
172+
"type": "string",
173+
"units": "1",
174+
"long_name": "Spatial partition polygon"
175+
}
176+
}
177+
},
178+
"partitioning": [
179+
{
180+
"source_variable": "timestamp",
181+
"type": "time_extent",
182+
"time_extent": {
183+
"time_varname": "eventDate",
184+
"partition_period": "Y"
185+
}
186+
},
187+
{
188+
"source_variable": "polygon",
189+
"type": "spatial_extent",
190+
"spatial_extent": {
191+
"lat_varname": "decimalLatitude",
192+
"lon_varname": "decimalLongitude",
193+
"spatial_resolution": 90
194+
}
195+
}
196+
]
197+
}
198+
}

0 commit comments

Comments
 (0)