Skip to content

Commit 526b373

Browse files
ehavazliehavazli
committed
add cdsapi to authentication files and do some housekeeping
1 parent ab9380f commit 526b373

File tree

1 file changed

+82
-57
lines changed

1 file changed

+82
-57
lines changed

‎prep/ARIA_prep.ipynb‎

Lines changed: 82 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@
164164
"print(f\"MintPy directory: {mintpy_dir}\")\n",
165165
"\n",
166166
"# Configuration file path\n",
167-
"site_code = sitedata['sites'][site]['calval_location']\n",
167+
"site_code = site_info.get('calval_location')\n",
168168
"config_file = os.path.join(mintpy_dir, f\"{site_code}.cfg\")"
169169
]
170170
},
@@ -187,42 +187,64 @@
187187
},
188188
"outputs": [],
189189
"source": [
190-
"# Earthdata login to download GUNWs\n",
191-
"fnetrc = '/home/jovyan/.netrc'\n",
190+
"def ensure_permission(path, mode=0o600):\n",
191+
" if os.path.exists(path):\n",
192+
" os.chmod(path, mode)\n",
193+
"\n",
194+
"# === Earthdata Login ===\n",
195+
"fnetrc = os.path.expanduser(\"~/.netrc\")\n",
196+
"earthdata_host = \"urs.earthdata.nasa.gov\"\n",
192197
"earthdata = False\n",
198+
"\n",
193199
"if os.path.exists(fnetrc):\n",
194-
" os.system('chmod 0600 ' + fnetrc)\n",
195-
" remoteHostName = \"urs.earthdata.nasa.gov\"\n",
196-
" netrc = netrc.netrc()\n",
197-
" with open(fnetrc) as file:\n",
198-
" if remoteHostName in file.read():\n",
199-
" authTokens = netrc.authenticators(remoteHostName)\n",
200-
" earthdata_user = authTokens[0]\n",
201-
" earthdata_password = authTokens[2]\n",
202-
" earthdata = True \n",
203-
"if not earthdata: \n",
204-
" print('NEEDED to Download ARIA GUNWs: \\n Link to create account : https://urs.earthdata.nasa.gov/')\n",
205-
" earthdata_user = input('Please type your Earthdata username:')\n",
206-
" earthdata_password = input('Please type your Earthdata password:')\n",
207-
" with open(fnetrc, 'a') as file:\n",
208-
" file.write('machine urs.earthdata.nasa.gov\\n')\n",
209-
" file.write('login ' + earthdata_user + '\\n')\n",
210-
" file.write('password ' + earthdata_password)\n",
211-
" os.system('chmod 0600 ' + fnetrc)\n",
212-
"\n",
213-
"# OpenTopography login to download DEMs\n",
214-
"fopentopo = '/home/jovyan/.topoapi'\n",
200+
" ensure_permission(fnetrc)\n",
201+
" nrc = netrc.netrc()\n",
202+
" credentials = nrc.authenticators(earthdata_host)\n",
203+
" if credentials:\n",
204+
" earthdata_user, _, earthdata_password = credentials\n",
205+
" earthdata = True\n",
206+
" print(f\"Earthdata credentials found for user: {earthdata_user}\")\n",
207+
"\n",
208+
"if not earthdata:\n",
209+
" print(\"\\nNEEDED to Download ARIA GUNWs\")\n",
210+
" print(\"Create account at: https://urs.earthdata.nasa.gov/\")\n",
211+
" earthdata_user = input(\"Earthdata username: \").strip()\n",
212+
" earthdata_password = input(\"Earthdata password: \").strip()\n",
213+
" with open(fnetrc, \"a\") as f:\n",
214+
" f.write(f\"machine {earthdata_host}\\nlogin {earthdata_user}\\npassword {earthdata_password}\\n\")\n",
215+
" ensure_permission(fnetrc)\n",
216+
" print(\"Earthdata credentials saved.\")\n",
217+
"\n",
218+
"\n",
219+
"# === OpenTopography API Key ===\n",
220+
"fopentopo = os.path.expanduser(\"~/.topoapi\")\n",
215221
"if os.path.exists(fopentopo):\n",
216-
" os.system('chmod 0600 ' + fopentopo)\n",
217-
" with open(fopentopo) as file:\n",
218-
" opentopography_api_key = file.read()\n",
219-
"else: \n",
220-
" print('NEEDED To Download DEMs: \\n Link to get API Key : https://portal.opentopography.org/login' + \n",
221-
" '\\n Goto: My Account > myOpenTopo Authorizations and API Key > Request API key')\n",
222-
" opentopography_api_key = input('Please type your OpenTopo API key:')\n",
223-
" with open(fopentopo, 'a') as file:\n",
224-
" file.write(opentopography_api_key)\n",
225-
" os.system('chmod 0600 ' + fopentopo)"
222+
" ensure_permission(fopentopo)\n",
223+
" with open(fopentopo) as f:\n",
224+
" opentopography_api_key = f.read().strip()\n",
225+
"else:\n",
226+
" print(\"\\nNEEDED To Download DEMs:\")\n",
227+
" print(\"Register at: https://portal.opentopography.org/login\")\n",
228+
" print(\"Navigate: My Account → myOpenTopo Authorizations and API Key → Request API key\")\n",
229+
" opentopography_api_key = input(\"OpenTopo API key: \").strip()\n",
230+
" with open(fopentopo, \"w\") as f:\n",
231+
" f.write(opentopography_api_key + \"\\n\")\n",
232+
" ensure_permission(fopentopo)\n",
233+
" print(\"OpenTopography API key saved.\")\n",
234+
"\n",
235+
"# === CDS (ERA5) API Key ===\n",
236+
"cds_config_path = os.path.expanduser(\"~/.cdsapirc\")\n",
237+
"if not os.path.exists(cds_config_path):\n",
238+
" print(\"\\nNEEDED to use ERA5 correction:\")\n",
239+
" print(\"Register and get token: https://cds.climate.copernicus.eu/how-to-api\")\n",
240+
" cds_key = input(\"CDS API key (uid:api-key): \").strip()\n",
241+
" with open(cds_config_path, \"w\") as f:\n",
242+
" f.write(\"url: https://cds.climate.copernicus.eu/api\\n\")\n",
243+
" f.write(f\"key: {cds_key}\\n\")\n",
244+
" ensure_permission(cds_config_path)\n",
245+
" print(\"CDS API config created.\")\n",
246+
"else:\n",
247+
" print(\"CDS API config file detected. (Ensure it is current)\")"
226248
]
227249
},
228250
{
@@ -261,11 +283,10 @@
261283
"print(f\"CalVal site: {site}\")\n",
262284
"\n",
263285
"# Extract site-specific metadata\n",
264-
"site_info = sitedata['sites'][site]\n",
265-
"bbox = site_info['download_region']\n",
266-
"startdate = site_info['download_start_date']\n",
267-
"enddate = site_info['download_end_date']\n",
268-
"track = site_info['sentinel_track']\n",
286+
"bbox = site_info.get('download_region')\n",
287+
"startdate = site_info.get('download_start_date')\n",
288+
"enddate = site_info.get('download_end_date')\n",
289+
"track = site_info.get('sentinel_track')\n",
269290
"\n",
270291
"# Base command template\n",
271292
"aria_cmd_template = (\n",
@@ -329,8 +350,8 @@
329350
"outputs": [],
330351
"source": [
331352
"# Parse date range from site metadata\n",
332-
"start_date = int(sitedata['sites'][site]['download_start_date'])\n",
333-
"end_date = int(sitedata['sites'][site]['download_end_date'])\n",
353+
"start_date = int(site_info.get('download_start_date'))\n",
354+
"end_date = int(site_info.get('download_end_date'))\n",
334355
"\n",
335356
"# Filter GUNW files based on date range and version\n",
336357
"gunw_list = []\n",
@@ -367,11 +388,11 @@
367388
"optional_layers = []\n",
368389
"\n",
369390
"layer_conditions = [\n",
370-
" (\"solidEarthTide\", sitedata['sites'][site].get('do_SET') == 'True'),\n",
371-
" (\"ionosphere\", sitedata['sites'][site].get('do_iono') == 'True'),\n",
391+
" (\"solidEarthTide\", site_info.get('do_SET') == 'True'),\n",
392+
" (\"ionosphere\", site_info.get('do_iono') == 'True'),\n",
372393
" (\"troposphereTotal\", (\n",
373-
" sitedata['sites'][site].get('do_tropo') == 'True' and \n",
374-
" sitedata['sites'][site].get('tropo_model') == 'HRRR'\n",
394+
" site_info.get('do_tropo') == 'True' and \n",
395+
" site_info.get('tropo_model') == 'HRRR'\n",
375396
" ))\n",
376397
"]\n",
377398
"\n",
@@ -399,15 +420,15 @@
399420
" cmd_parts = [\n",
400421
" \"ariaTSsetup.py\",\n",
401422
" f\"-f {product_file}\",\n",
402-
" f\"-b {sitedata['sites'][site]['analysis_region']}\",\n",
423+
" f\"-b {site_info.get('analysis_region')}\",\n",
403424
" f\"-l '{', '.join(optional_layers)}'\",\n",
404425
" \"--croptounion\",\n",
405426
" \"-nt 8\",\n",
406427
" \"--log-level info\"\n",
407428
" ]\n",
408429
"\n",
409430
" # Add water mask option if enabled\n",
410-
" if sitedata['sites'][site]['maskWater'] != 'False':\n",
431+
" if site_info.get('maskWater') != 'False':\n",
411432
" cmd_parts.append(\"--mask Download\")\n",
412433
" mask_file = f\"{work_dir}/mask/watermask.msk\"\n",
413434
" else:\n",
@@ -416,6 +437,10 @@
416437
" subprocess.run(\" \".join(cmd_parts), shell=True, text=True)\n",
417438
"\n",
418439
"else:\n",
440+
" if os.path.isfile(f\"{work_dir}/mask/watermask.msk\"):\n",
441+
" mask_file = f\"{work_dir}/mask/watermask.msk\"\n",
442+
" else:\n",
443+
" mask_file = \"auto\"\n",
419444
" print(\"Stack directory detected and not overwritten.\")\n"
420445
]
421446
},
@@ -472,13 +497,13 @@
472497
" \"mintpy.topographicResidual.pixelwiseGeometry\": \"no\",\n",
473498
" \"mintpy.troposphericDelay.method\": \"no\",\n",
474499
" \"mintpy.topographicResidual\": \"no\",\n",
475-
" \"mintpy.network.tempBaseMax\": sitedata['sites'][site]['tempBaseMax'],\n",
476-
" \"mintpy.network.startDate\": sitedata['sites'][site]['download_start_date'],\n",
477-
" \"mintpy.network.endDate\": sitedata['sites'][site]['download_end_date'],\n",
478-
" \"mintpy.velocity.startDate\": sitedata['sites'][site]['download_start_date'],\n",
479-
" \"mintpy.velocity.endDate\": sitedata['sites'][site]['download_end_date'],\n",
480-
" \"mintpy.reference.lalo\": sitedata['sites'][site]['reference_lalo'],\n",
481-
" \"mintpy.network.excludeIfgIndex\": sitedata['sites'][site]['ifgExcludeList'],\n",
500+
" \"mintpy.network.tempBaseMax\": site_info.get('tempBaseMax'),\n",
501+
" \"mintpy.network.startDate\": site_info.get('download_start_date'),\n",
502+
" \"mintpy.network.endDate\": site_info.get('download_end_date'),\n",
503+
" \"mintpy.velocity.startDate\": site_info.get('download_start_date'),\n",
504+
" \"mintpy.velocity.endDate\": site_info.get('download_end_date'),\n",
505+
" \"mintpy.reference.lalo\": site_info.get('reference_lalo'),\n",
506+
" \"mintpy.network.excludeIfgIndex\": site_info.get('ifgExcludeList'),\n",
482507
"}\n",
483508
"\n",
484509
"# Write config dictionary to text file\n",
@@ -573,7 +598,7 @@
573598
"troposphere_f = None\n",
574599
"\n",
575600
"# Solid Earth Tides correction\n",
576-
"if sitedata['sites'][site].get('do_SET') == 'True':\n",
601+
"if site_info.get('do_SET') == 'True':\n",
577602
" path = os.path.join(stack_dir, 'setStack.vrt')\n",
578603
" if os.path.isfile(path):\n",
579604
" solidearthtides_f = path\n",
@@ -582,7 +607,7 @@
582607
" print(f\"File Not Found: {path}\")\n",
583608
"\n",
584609
"# Ionosphere correction\n",
585-
"if sitedata['sites'][site].get('do_iono') == 'True':\n",
610+
"if site_info.get('do_iono') == 'True':\n",
586611
" path = os.path.join(stack_dir, 'ionoStack.vrt')\n",
587612
" if os.path.isfile(path):\n",
588613
" ionosphere_f = path\n",
@@ -591,7 +616,7 @@
591616
" print(f\"File Not Found: {path}\")\n",
592617
"\n",
593618
"# Troposphere correction\n",
594-
"if sitedata['sites'][site].get('do_tropo') == 'True' and sitedata['sites'][site].get('tropo_model') == 'HRRR':\n",
619+
"if site_info.get('do_tropo') == 'True' and site_info.get('tropo_model') == 'HRRR':\n",
595620
" path = os.path.join(stack_dir, 'troposphereTotal', 'HRRRStack.vrt')\n",
596621
" if os.path.isfile(path):\n",
597622
" troposphere_f = path\n",

0 commit comments

Comments
 (0)