Skip to content

Commit c1ef3f1

Browse files
authored
Merge pull request #641 from dbekaert/dev
Release v0.5.0 -- orbits and stand-alone HyP3 job support
2 parents 85a26cd + 42d77d3 commit c1ef3f1

File tree

15 files changed

+366
-329
lines changed

15 files changed

+366
-329
lines changed

CHANGELOG.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
77
and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
88

9+
## [0.5.0]
10+
### Added
11+
* A `--input-bucket-prefix` argument to `calcDelaysGUNW` which will allow RAiDER to process ARIA GUNW products under one prefix and upload the final products to another prefix provided by the `--bucket-prefix` argument.
12+
### Fixed
13+
* [613](https://github.com/dbekaert/RAiDER/issues/613) - ensure NASA Earthdata credentials for downloading orbits from ASF
14+
* [634](https://github.com/dbekaert/RAiDER/issues/634) - download orbits from ASF before trying ESA
15+
* [630](https://github.com/dbekaert/RAiDER/pull/630) - use correct model name so (hrrr-ak) in azimuth_timing_grid
16+
* [620](https://github.com/dbekaert/RAiDER/issues/620) - Fix MERRA-2 access because API was updated
17+
918

1019
## [0.4.7]
1120
### Fixed

test/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
WM_DIR = os.path.join(TEST_DIR, 'weather_files')
1717
ORB_DIR = os.path.join(TEST_DIR, 'orbit_files')
1818

19-
WM = 'GMAO'
19+
WM = 'MERRA2'
2020

2121
@contextmanager
2222
def pushd(dir):

test/test_GUNW.py

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ def test_GUNW_hyp3_metadata_update(test_gunw_json_path, test_gunw_json_schema_pa
100100

101101
# We only need to make sure the json file is passed, the netcdf file name will not have
102102
# any impact on subsequent testing
103-
mocker.patch("RAiDER.aws.get_s3_file", side_effect=['foo.nc', temp_json_path])
103+
mocker.patch("RAiDER.aws.get_s3_file", side_effect=['foo.nc', temp_json_path, 'foo.png'])
104104
mocker.patch("RAiDER.aws.upload_file_to_s3")
105105
mocker.patch("RAiDER.aria.prepFromGUNW.main", return_value=['my_path_cfg', 'my_wavelength'])
106106
mocker.patch('RAiDER.aria.prepFromGUNW.check_hrrr_dataset_availablity_for_s1_azimuth_time_interpolation',
@@ -111,7 +111,7 @@ def test_GUNW_hyp3_metadata_update(test_gunw_json_path, test_gunw_json_schema_pa
111111

112112
iargs = ['--weather-model', 'HRES',
113113
'--bucket', 'myBucket',
114-
'--bucket-prefix', 'myPrefix']
114+
'--bucket-prefix', 'myPrefix',]
115115
calcDelaysGUNW(iargs)
116116

117117
metadata = json.loads(temp_json_path.read_text())
@@ -123,6 +123,7 @@ def test_GUNW_hyp3_metadata_update(test_gunw_json_path, test_gunw_json_schema_pa
123123
assert aws.get_s3_file.mock_calls == [
124124
unittest.mock.call('myBucket', 'myPrefix', '.nc'),
125125
unittest.mock.call('myBucket', 'myPrefix', '.json'),
126+
unittest.mock.call('myBucket', 'myPrefix', '.png'),
126127
]
127128

128129
RAiDER.aria.prepFromGUNW.main.assert_called_once()
@@ -138,6 +139,60 @@ def test_GUNW_hyp3_metadata_update(test_gunw_json_path, test_gunw_json_schema_pa
138139
assert aws.upload_file_to_s3.mock_calls == [
139140
unittest.mock.call('foo.nc', 'myBucket', 'myPrefix'),
140141
unittest.mock.call(temp_json_path, 'myBucket', 'myPrefix'),
142+
unittest.mock.call('foo.png', 'myBucket', 'myPrefix'),
143+
]
144+
145+
146+
def test_GUNW_hyp3_metadata_update_different_prefix(test_gunw_json_path, test_gunw_json_schema_path, tmp_path, mocker):
147+
"""This test performs the GUNW entrypoint using a different input bucket/prefix than the output bucket/prefix.
148+
Only updates the json. Monkey patches the upload/download to/from s3 and the actual computation.
149+
"""
150+
temp_json_path = tmp_path / 'temp.json'
151+
shutil.copy(test_gunw_json_path, temp_json_path)
152+
153+
# We only need to make sure the json file is passed, the netcdf file name will not have
154+
# any impact on subsequent testing
155+
mocker.patch("RAiDER.aws.get_s3_file", side_effect=['foo.nc', temp_json_path, 'foo.png'])
156+
mocker.patch("RAiDER.aws.upload_file_to_s3")
157+
mocker.patch("RAiDER.aria.prepFromGUNW.main", return_value=['my_path_cfg', 'my_wavelength'])
158+
mocker.patch('RAiDER.aria.prepFromGUNW.check_hrrr_dataset_availablity_for_s1_azimuth_time_interpolation',
159+
side_effect=[True])
160+
mocker.patch("RAiDER.aria.prepFromGUNW.check_weather_model_availability", return_value=True)
161+
mocker.patch("RAiDER.cli.raider.calcDelays", return_value=['file1', 'file2'])
162+
mocker.patch("RAiDER.aria.calcGUNW.tropo_gunw_slc")
163+
164+
iargs = ['--weather-model', 'HRES',
165+
'--bucket', 'myBucket',
166+
'--bucket-prefix', 'myOutputPrefix',
167+
'--input-bucket-prefix', 'myInputPrefix',]
168+
calcDelaysGUNW(iargs)
169+
170+
metadata = json.loads(temp_json_path.read_text())
171+
schema = json.loads(test_gunw_json_schema_path.read_text())
172+
173+
assert metadata['metadata']['weather_model'] == ['HRES']
174+
assert (jsonschema.validate(instance=metadata, schema=schema) is None)
175+
176+
assert aws.get_s3_file.mock_calls == [
177+
unittest.mock.call('myBucket', 'myInputPrefix', '.nc'),
178+
unittest.mock.call('myBucket', 'myInputPrefix', '.json'),
179+
unittest.mock.call('myBucket', 'myInputPrefix', '.png'),
180+
]
181+
182+
RAiDER.aria.prepFromGUNW.main.assert_called_once()
183+
184+
raider.calcDelays.assert_called_once_with(['my_path_cfg'])
185+
186+
RAiDER.aria.calcGUNW.tropo_gunw_slc.assert_called_once_with(
187+
['file1', 'file2'],
188+
'foo.nc',
189+
'my_wavelength',
190+
)
191+
192+
assert aws.upload_file_to_s3.mock_calls == [
193+
unittest.mock.call('foo.nc', 'myBucket', 'myOutputPrefix'),
194+
unittest.mock.call(temp_json_path, 'myBucket', 'myOutputPrefix'),
195+
unittest.mock.call('foo.png', 'myBucket', 'myOutputPrefix'),
141196
]
142197

143198

@@ -277,7 +332,7 @@ def test_azimuth_timing_interp_against_center_time_interp(weather_model_name: st
277332
assert np.nanmax(abs_diff_mm) < 1
278333

279334

280-
@pytest.mark.parametrize('weather_model_name', ['GMAO', 'HRRR', 'HRES', 'ERA5', 'ERA5'])
335+
@pytest.mark.parametrize('weather_model_name', ['HRRR', 'HRES', 'ERA5', 'ERA5T'])
281336
def test_check_weather_model_availability(test_gunw_path_factory, weather_model_name, mocker):
282337
# Should be True for all weather models
283338
# S1-GUNW-D-R-071-tops-20200130_20200124-135156-34956N_32979N-PP-913f-v2_0_4.nc
@@ -288,12 +343,12 @@ def test_check_weather_model_availability(test_gunw_path_factory, weather_model_
288343
mocker.patch("RAiDER.aria.prepFromGUNW.get_acq_time_from_slc_id", side_effect=[pd.Timestamp('2015-01-01'),
289344
pd.Timestamp('2014-01-01')])
290345
cond = check_weather_model_availability(test_gunw_path, weather_model_name)
291-
if weather_model_name in ['HRRR', 'GMAO']:
346+
if weather_model_name in ['HRRR', 'MERRA2']:
292347
cond = not cond
293348
assert cond
294349

295350

296-
@pytest.mark.parametrize('weather_model_name', ['GMAO', 'HRRR'])
351+
@pytest.mark.parametrize('weather_model_name', ['HRRR'])
297352
def test_check_weather_model_availability_over_alaska(test_gunw_path_factory, weather_model_name, mocker):
298353
# Should be True for all weather models
299354
# S1-GUNW-D-R-059-tops-20230320_20220418-180300-00179W_00051N-PP-c92e-v2_0_6.nc
@@ -309,7 +364,7 @@ def test_check_weather_model_availability_over_alaska(test_gunw_path_factory, we
309364
assert cond
310365

311366

312-
@pytest.mark.parametrize('weather_model_name', ['HRRR', 'GMAO'])
367+
@pytest.mark.parametrize('weather_model_name', ['HRRR'])
313368
@pytest.mark.parametrize('location', ['california-t71', 'alaska'])
314369
def test_weather_model_availability_integration_using_valid_range(location,
315370
test_gunw_path_factory,

test/test_s1_orbits.py

Lines changed: 114 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -12,101 +12,186 @@ class EmptyNetrc():
1212
def __init__(self, netrc_file):
1313
self.netrc_file = netrc_file
1414
self.hosts = {}
15+
1516
def __str__(self):
1617
return str(self.hosts)
1718

18-
# No .netrc, no ESA CDSE env variables
19+
# No .netrc, no ESA CDSE or Earthdata env variables
20+
with monkeypatch.context() as mp:
21+
mp.setattr(netrc, 'netrc', EmptyNetrc, raising=False)
22+
mp.delenv('ESA_USERNAME', raising=False)
23+
mp.delenv('ESA_PASSWORD', raising=False)
24+
mp.delenv('EARTHDATA_USERNAME', raising=False)
25+
mp.delenv('EARTHDATA_PASSWORD', raising=False)
26+
with pytest.raises(ValueError):
27+
s1_orbits.ensure_orbit_credentials()
28+
29+
# No .netrc or Earthdata env vars, set ESA CDSE env variables
30+
with monkeypatch.context() as mp:
31+
mp.setattr(netrc, 'netrc', EmptyNetrc, raising=False)
32+
mp.setenv('ESA_USERNAME', 'foo')
33+
mp.setenv('ESA_PASSWORD', 'bar')
34+
mp.delenv('EARTHDATA_USERNAME', raising=False)
35+
mp.delenv('EARTHDATA_PASSWORD', raising=False)
36+
with pytest.raises(ValueError):
37+
s1_orbits.ensure_orbit_credentials()
38+
39+
# No .netrc or ESA CDSE env vars, set Earthdata env variables
1940
with monkeypatch.context() as mp:
2041
mp.setattr(netrc, 'netrc', EmptyNetrc, raising=False)
2142
mp.delenv('ESA_USERNAME', raising=False)
2243
mp.delenv('ESA_PASSWORD', raising=False)
44+
mp.setenv('EARTHDATA_USERNAME', 'fizz')
45+
mp.setenv('EARTHDATA_PASSWORD', 'buzz')
2346
with pytest.raises(ValueError):
2447
s1_orbits.ensure_orbit_credentials()
2548

26-
# No .netrc, set ESA CDSE env variables
49+
# No .netrc, set Earthdata and ESA CDSE env variables
2750
with monkeypatch.context() as mp:
2851
mp.setattr(netrc, 'netrc', EmptyNetrc, raising=False)
2952
mp.setenv('ESA_USERNAME', 'foo')
3053
mp.setenv('ESA_PASSWORD', 'bar')
54+
mp.setenv('EARTHDATA_USERNAME', 'fizz')
55+
mp.setenv('EARTHDATA_PASSWORD', 'buzz')
3156
mp.setattr(Path, 'write_text', lambda self, write_text: write_text)
3257
written_credentials = s1_orbits.ensure_orbit_credentials()
33-
assert written_credentials == str({s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar')})
58+
assert written_credentials == str({
59+
s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar'),
60+
s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz')
61+
})
3462

3563
class NoCDSENetrc():
3664
def __init__(self, netrc_file):
3765
self.netrc_file = netrc_file
38-
self.hosts = {'fizz.buzz.org': ('foo', None, 'bar')}
66+
self.hosts = {s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz')}
67+
3968
def __str__(self):
4069
return str(self.hosts)
4170

42-
# No CDSE in .netrc, no ESA CDSE env variables
71+
# No CDSE in .netrc or ESA CDSE env variables, Earthdata in .netrc
4372
with monkeypatch.context() as mp:
4473
mp.setattr(netrc, 'netrc', NoCDSENetrc, raising=False)
4574
mp.delenv('ESA_USERNAME', raising=False)
4675
mp.delenv('ESA_PASSWORD', raising=False)
4776
with pytest.raises(ValueError):
4877
s1_orbits.ensure_orbit_credentials()
4978

50-
# No CDSE in .netrc, set ESA CDSE env variables
79+
# No CDSE in .netrc, set ESA CDSE env variables, Earthdata in .netrc
5180
with monkeypatch.context() as mp:
5281
mp.setattr(netrc, 'netrc', NoCDSENetrc, raising=False)
5382
mp.setenv('ESA_USERNAME', 'foo')
5483
mp.setenv('ESA_PASSWORD', 'bar')
5584
mp.setattr(Path, 'write_text', lambda self, write_text: write_text)
5685
written_credentials = s1_orbits.ensure_orbit_credentials()
57-
assert written_credentials == str({'fizz.buzz.org': ('foo', None, 'bar'), s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar')})
86+
assert written_credentials == str({
87+
s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz'),
88+
s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar'),
89+
})
5890

59-
class CDSENetrc():
91+
class NoEarthdataNetrc():
6092
def __init__(self, netrc_file):
6193
self.netrc_file = netrc_file
6294
self.hosts = {s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar')}
95+
6396
def __str__(self):
6497
return str(self.hosts)
6598

66-
# cdse in .netrc, no ESA CDSE env variables
99+
# cdse in .netrc, no ESA CDSE env variables, Earthdata env variables
67100
with monkeypatch.context() as mp:
68-
mp.setattr(netrc, 'netrc', CDSENetrc, raising=False)
101+
mp.setattr(netrc, 'netrc', NoEarthdataNetrc, raising=False)
69102
mp.delenv('ESA_USERNAME', raising=False)
70103
mp.delenv('ESA_PASSWORD', raising=False)
104+
mp.setenv('EARTHDATA_USERNAME', 'fizz')
105+
mp.setenv('EARTHDATA_PASSWORD', 'buzz')
106+
mp.setattr(Path, 'write_text', lambda self, write_text: write_text)
107+
written_credentials = s1_orbits.ensure_orbit_credentials()
108+
assert written_credentials == str({
109+
s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar'),
110+
s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz'),
111+
})
112+
113+
class CDSEAndEarthdataNetrc():
114+
def __init__(self, netrc_file):
115+
self.netrc_file = netrc_file
116+
self.hosts = {
117+
s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar'),
118+
s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz')
119+
}
120+
121+
def __str__(self):
122+
return str(self.hosts)
123+
124+
# cdse and Earthdata in netrc, no env variables
125+
with monkeypatch.context() as mp:
126+
mp.setattr(netrc, 'netrc', CDSEAndEarthdataNetrc, raising=False)
127+
written_credentials = s1_orbits.ensure_orbit_credentials()
128+
assert written_credentials is None
129+
130+
with monkeypatch.context() as mp:
131+
mp.setattr(netrc, 'netrc', CDSEAndEarthdataNetrc, raising=False)
132+
mp.delenv('ESA_USERNAME', raising=False)
133+
mp.delenv('ESA_PASSWORD', raising=False)
134+
mp.setenv('EARTHDATA_USERNAME', 'fizz')
135+
mp.setenv('EARTHDATA_PASSWORD', 'buzz')
71136
written_credentials = s1_orbits.ensure_orbit_credentials()
72137
assert written_credentials is None
73138

74-
# cdse in .netrc, set ESA CDSE env variables
75139
with monkeypatch.context() as mp:
76-
mp.setattr(netrc, 'netrc', CDSENetrc, raising=False)
140+
mp.setattr(netrc, 'netrc', CDSEAndEarthdataNetrc, raising=False)
77141
mp.setenv('ESA_USERNAME', 'foo')
78142
mp.setenv('ESA_PASSWORD', 'bar')
143+
mp.setenv('EARTHDATA_USERNAME', 'fizz')
144+
mp.setenv('EARTHDATA_PASSWORD', 'buzz')
145+
written_credentials = s1_orbits.ensure_orbit_credentials()
146+
assert written_credentials is None
147+
148+
with monkeypatch.context() as mp:
149+
mp.setattr(netrc, 'netrc', CDSEAndEarthdataNetrc, raising=False)
150+
mp.setenv('ESA_USERNAME', 'foo')
151+
mp.setenv('ESA_PASSWORD', 'bar')
152+
mp.delenv('ESA_USERNAME', raising=False)
153+
mp.delenv('ESA_PASSWORD', raising=False)
79154
written_credentials = s1_orbits.ensure_orbit_credentials()
80155
assert written_credentials is None
81156

82157

83158
def test_get_orbits_from_slc_ids(mocker):
84159
side_effect = [
85-
[Path('foo.txt')],
86-
[Path('bar.txt'), Path('fiz.txt')],
160+
[Path('foo_start.txt'), Path('foo_stop.txt')],
161+
[Path('bar_start.txt'), Path('bar_end.txt'),
162+
Path('fiz_start.txt'), Path('fiz_end')],
87163
]
88164
mocker.patch('eof.download.download_eofs',
89-
side_effect=side_effect)
165+
side_effect=side_effect[0])
90166

91167
orbit_files = s1_orbits.get_orbits_from_slc_ids(
92168
['S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8']
93169
)
94-
assert orbit_files == [Path('foo.txt')]
95-
assert eof.download.download_eofs.call_count == 1
96-
eof.download.download_eofs.assert_called_with(
97-
[ '20150621T120220', '20150621T120232'],
98-
['S1A'] * 2,
99-
save_dir=str(Path.cwd())
100-
)
170+
assert orbit_files == side_effect[0]
171+
assert eof.download.download_eofs.call_count == 2
172+
for dt in '20150621T120220 20150621T120232'.split():
173+
eof.download.download_eofs.assert_any_call(
174+
[dt],
175+
['S1A'],
176+
save_dir=str(Path.cwd()),
177+
force_asf=True
178+
)
179+
180+
mocker.patch('eof.download.download_eofs',
181+
side_effect=side_effect[1])
101182

102183
orbit_files = s1_orbits.get_orbits_from_slc_ids(
103184
['S1B_IW_SLC__1SDV_20201115T162313_20201115T162340_024278_02E29D_5C54',
104185
'S1A_IW_SLC__1SDV_20201203T162353_20201203T162420_035524_042744_6D5C']
105186
)
106-
assert orbit_files == [Path('bar.txt'), Path('fiz.txt')]
107-
assert eof.download.download_eofs.call_count == 2
108-
eof.download.download_eofs.assert_called_with(
109-
['20201115T162313', '20201203T162353', '20201115T162340', '20201203T162420'],
110-
['S1B', 'S1A'] * 2,
111-
save_dir=str(Path.cwd())
112-
)
187+
assert orbit_files == side_effect[1]
188+
assert eof.download.download_eofs.call_count == 4
189+
missions = 'S1B S1B S1A S1A'.split()
190+
dts = '20201115T162313 20201115T162340 20201203T162353 20201203T162420'.split()
191+
for dt, mission in zip(dts, missions):
192+
eof.download.download_eofs.assert_any_call(
193+
[dt],
194+
[mission],
195+
save_dir=str(Path.cwd()),
196+
force_asf=True
197+
)

0 commit comments

Comments
 (0)