Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
f0153d0
DAS-2446 - Refactor variable names.
lyonthefrog Dec 2, 2025
e8baf52
DAS-2446 - Write URL formatting functions.
lyonthefrog Dec 2, 2025
5fbfc6d
DAS-2446 - Incorporate OPeNDAP request URL as an output option.
lyonthefrog Dec 2, 2025
de0d6a4
DAS-2446 - Update existing unit tests to include a test mimetype value.
lyonthefrog Dec 3, 2025
979fa46
DAS-2446 - Add opendap url end-to-end test, update catalog comparision.
lyonthefrog Dec 10, 2025
545ddcf
DAS-2446 - Add get_opendap_nc4 subtest and new unexecuted_url_request…
lyonthefrog Dec 10, 2025
7994835
DAS-2446 - Update service version and changelog.
lyonthefrog Dec 11, 2025
511ba0e
DAS-2446 - Minor typo in changelog.
lyonthefrog Dec 11, 2025
dcf1481
DAS-2446 - Add release tag link to changelog.
lyonthefrog Dec 12, 2025
ef3ced7
DAS-2446 - Update comments for clarity.
lyonthefrog Dec 12, 2025
33cc5c3
DAS-2446 - Add logger output for formatted url.
lyonthefrog Dec 12, 2025
3c0ddca
DAS-2446 - Remove mimetype dependency in tests.
lyonthefrog Dec 12, 2025
e3fcff1
DAS-2446 - Add None type option to functions that include opendap_url…
lyonthefrog Dec 16, 2025
faf2f86
DAS-2446 - Correct verb tense in changelog.
lyonthefrog Dec 16, 2025
f8b0844
DAS-2446 - Add fourth opendap url format string option and update com…
lyonthefrog Dec 16, 2025
56ace23
DAS-2446 - Update optional argument code style in get_opendap_nc4() f…
lyonthefrog Dec 16, 2025
21a1af7
DAS-2446 - Add catalog verification to opendap_url_request test.
lyonthefrog Dec 16, 2025
793e058
Merge in main branch (snyk vulnerabilities library update).
lyonthefrog Dec 17, 2025
e50cab9
Merge branch 'main' into DAS-2446
lyonthefrog Dec 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Comment thread
sudha-murthy marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
## [v1.2.0] - 2025-12-11

### Added

- Allow HOSS to format and return an unexecuted OPeNDAP URL instead of a subset
file when requested by the user.

## [v1.1.17] - 2025-11-25

### Changed
Expand Down
2 changes: 1 addition & 1 deletion docker/service_version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.1.17
1.2.0
53 changes: 32 additions & 21 deletions hoss/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,11 @@
from hoss.dimension_utilities import is_index_subset
from hoss.harmony_log_context import set_logger
from hoss.subset import subset_granule
from hoss.utilities import get_file_mimetype, raise_from_hoss_exception
from hoss.utilities import (
get_file_mimetype,
raise_from_hoss_exception,
unexecuted_url_requested,
)


class HossAdapter(BaseHarmonyAdapter):
Expand Down Expand Up @@ -105,32 +109,39 @@ def process_item(self, item: Item, source: Source):

self.logger.info(f'Collection short name: {source.shortName}')

# Invoke service logic to retrieve subset of file from OPeNDAP
output_file_path = subset_granule(
# Invoke service logic to retrieve request output from OPeNDAP.
output_url = subset_granule(
asset.href, source, workdir, self.message, self.config
)

# Stage the output file with a conventional filename
mime, _ = get_file_mimetype(output_file_path)
staged_filename = generate_output_filename(
asset.href,
variable_subset=source.variables,
ext='.nc4',
is_subsetted=(
is_index_subset(self.message) or len(source.variables) > 0
),
)
url = stage(
output_file_path,
staged_filename,
mime,
location=self.message.stagingLocation,
logger=self.logger,
)
# Check if an unexecuted OPeNDAP URL or subset file was requested.
if unexecuted_url_requested(self.message.format.mime):
asset_name = 'OPeNAP Request URL'
url = output_url
mime = self.message.format.mime
# Otherwise, stage the returned output file using a conventional
# filename.
else:
asset_name = generate_output_filename(
asset.href,
variable_subset=source.variables,
ext='.nc4',
is_subsetted=(
is_index_subset(self.message) or len(source.variables) > 0
),
)
mime, _ = get_file_mimetype(output_url)
url = stage(
output_url,
asset_name,
mime,
location=self.message.stagingLocation,
logger=self.logger,
)
Comment thread
sudha-murthy marked this conversation as resolved.

# Update the STAC record
result.assets['data'] = Asset(
url, title=staged_filename, media_type=mime, roles=['data']
url, title=asset_name, media_type=mime, roles=['data']
)

except NoDataException as no_data_exception:
Expand Down
20 changes: 11 additions & 9 deletions hoss/subset.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@


Comment thread
sudha-murthy marked this conversation as resolved.
def subset_granule(
opendap_url: str,
input_granule_url: str,
harmony_source: Source,
output_dir: str,
harmony_message: Message,
Expand Down Expand Up @@ -70,7 +70,7 @@ def subset_granule(

# Produce map of variable dependencies with `earthdata-varinfo` and `.dmr`.
varinfo = get_varinfo(
opendap_url,
input_granule_url,
output_dir,
harmony_source.shortName,
harmony_message.accessToken,
Expand All @@ -96,7 +96,7 @@ def subset_granule(
if request_is_index_subset:
# Prefetch all dimension variables in full:
dimensions_path = get_prefetch_variables(
opendap_url,
input_granule_url,
varinfo,
required_variables,
output_dir,
Expand Down Expand Up @@ -159,21 +159,23 @@ def subset_granule(
'variables_with_ranges: ' f'{format_variable_set_string(variables_with_ranges)}'
)

# Retrieve OPeNDAP data including only the specified variables in the
# specified ranges.
output_path = get_opendap_nc4(
opendap_url,
# Retrieve requested OPeNDAP output including only the specified variables
# in the specified ranges. Depending on the user input, this will either
Comment thread
sudha-murthy marked this conversation as resolved.
# be the subset data or a link to the OPeNDAP request.
output_url = get_opendap_nc4(
input_granule_url,
variables_with_ranges,
output_dir,
harmony_message.accessToken,
config,
harmony_message.format.mime,
)

# Fill the data outside the requested ranges for variables that cross a
# dimensional discontinuity (for example longitude and the anti-meridian).
fill_variables(output_path, varinfo, required_variables, index_ranges)
fill_variables(output_url, varinfo, required_variables, index_ranges)

return output_path
return output_url


def get_varinfo(
Expand Down
28 changes: 27 additions & 1 deletion hoss/utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,17 +46,25 @@ def get_opendap_nc4(
output_dir: str,
access_token: str,
config: Config,
mimetype=None,
Comment thread
flamingbear marked this conversation as resolved.
Outdated
) -> str:
"""Construct a semi-colon separated string of the required variables and
use as a constraint expression to retrieve those variables from
OPeNDAP.

Returns the path of the downloaded granule containing those variables.
Returns either the unexecuted OPeNDAP request URL or the path of the
downloaded granule containing those variables, depending on the requested
mimetype.

"""
constraint_expression = get_constraint_expression(required_variables)
netcdf4_url = f'{url}.dap.nc4'

# Check if the user requested an unexecuted OPeNDAP URL.
if unexecuted_url_requested(mimetype):
Comment thread
sudha-murthy marked this conversation as resolved.
get_logger().info('Returning unexecuted OPeNAP URL.')
return format_request_url(netcdf4_url, constraint_expression)

Comment thread
sudha-murthy marked this conversation as resolved.
if constraint_expression != '':
request_data = {'dap4.ce': constraint_expression}
else:
Expand All @@ -75,6 +83,24 @@ def get_opendap_nc4(
return move_downloaded_nc4(output_dir, downloaded_nc4)


def unexecuted_url_requested(mimetype: str) -> bool:
Comment thread
flamingbear marked this conversation as resolved.
Outdated
"""Return True if the user requests the unexecuted OPeNDAP URL."""
opendap_url_only_mimetype = [
'application/x-netcdf4; profile="opendap_url"',
'application/x-netcdf4;profile="opendap_url"',
'application/x-netcdf4;profile=opendap_url',
]
Comment thread
flamingbear marked this conversation as resolved.
return mimetype in opendap_url_only_mimetype


Comment thread
sudha-murthy marked this conversation as resolved.
def format_request_url(url: str, constraint_expression: str):
"""This function formats the request NetCDF4 URL to return a dap4
OPeNDAP request URL.

"""
return f'{url}?dap4.ce={constraint_expression}'
Comment thread
sudha-murthy marked this conversation as resolved.


def get_constraint_expression(variables: Set[str]) -> str:
"""Take a set of variables and return a URL encoded, semi-colon separated
DAP4 constraint expression to retrieve those variables. Each variable
Expand Down
Loading