Skip to content

Adjusting data downloader and subscriber to use earthaccess login methods #191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jpl-btlunsfo merged 3 commits into from
Jun 10, 2026
Merged

Adjusting data downloader and subscriber to use earthaccess login methods #191

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
686b9ce
adjusting downloader and subscriber to use earthaccess login auth, re…
jpl-btlunsfo May 14, 2026
00ef12d
changes per review
jpl-btlunsfo May 19, 2026
1650b45
get_search_results token refetch pushed down into podaac_access method
jpl-btlunsfo Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Subscriber.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ $> podaac-data-subscriber -h
Specify a provider for collection search. Default is POCLOUD.
--dry-run Search and identify files to download, but do not actually download them
--subset Flag to enable subsetting on the specified collection
--collection-version Restrict download to files within a specific collection version
```

## Run the Script
Expand Down
1,140 changes: 1,093 additions & 47 deletions poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@ packages = [
]

[tool.poetry.dependencies]
python = ">=3.10,<4.0"
python = ">=3.11,<4.0"
Comment thread
jpl-btlunsfo marked this conversation as resolved.
requests = "^2.27.1"
tenacity = "^8.0.1"
packaging = "^23.0"
harmony-py = "^0.4.12"
earthaccess = "^0.17.0"

[tool.poetry.dev-dependencies]
pytest = "^7.1.2"
Expand Down
139 changes: 0 additions & 139 deletions subscriber/podaac_access.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,145 +37,6 @@

IPAddr = "127.0.0.1" # socket.gethostbyname(hostname)


# ## Authentication setup
#
# The function below will allow Python scripts to log into any Earthdata Login
# application programmatically. To avoid being prompted for
# credentials every time you run and also allow clients such as curl to log in,
# you can add the following to a `.netrc` (`_netrc` on Windows) file in
# your home directory:
#
# ```
# machine urs.earthdata.nasa.gov
# login <your username>
# password <your password>
# ```
#
# Make sure that this file is only readable by the current user
# or you will receive an error stating
# "netrc access too permissive."
#
# `$ chmod 0600 ~/.netrc`
#
# You'll need to authenticate using the netrc method when running from
# command line with [`papermill`](https://papermill.readthedocs.io/en/latest/).
# You can log in manually by executing the cell below when running in the
# notebook client in your browser.*


def setup_earthdata_login_auth(endpoint):
"""
Set up the request library so that it authenticates against the given
Earthdata Login endpoint and is able to track cookies between requests.
This looks in the .netrc file first and if no credentials are found,
it prompts for them.

Valid endpoints include:
urs.earthdata.nasa.gov - Earthdata Login production
"""
try:
username, _, password = netrc.netrc().authenticators(endpoint)
except (FileNotFoundError, TypeError):
# FileNotFound = There's no .netrc file
# TypeError = The endpoint isn't in the netrc file,
# causing the above to try unpacking None
logging.warning("There's no .netrc file or the The endpoint isn't in the netrc file")

manager = request.HTTPPasswordMgrWithDefaultRealm()
manager.add_password(None, endpoint, username, password)
auth = request.HTTPBasicAuthHandler(manager)

jar = CookieJar()
processor = request.HTTPCookieProcessor(jar)
opener = request.build_opener(auth, processor)
opener.addheaders = [('User-agent', 'podaac-subscriber-' + __version__)]
request.install_opener(opener)



def get_token(url: str) -> str:
tokens = list_tokens(url)
if len(tokens) == 0 :
return create_token(url)
else:
return tokens[0]

###############################################################################
# GET TOKEN FROM CMR
###############################################################################
@tenacity.retry(wait=tenacity.wait_random_exponential(multiplier=1, max=60),
stop=tenacity.stop_after_attempt(3),
reraise=True,
retry=(tenacity.retry_if_result(lambda x: x == ''))
)
def create_token(url: str) -> str:
try:
token: str = ''
username, _, password = netrc.netrc().authenticators(edl)
headers: Dict = {'Accept': 'application/json'} # noqa E501


resp = requests.post(url+"/token", headers=headers, auth=HTTPBasicAuth(username, password))
response_content: Dict = json.loads(resp.content)
if "error" in response_content:
if response_content["error"] == "max_token_limit":
logging.error("Max tokens acquired from URS. Using existing token")
tokens=list_tokens(url)
return tokens[0]
token = response_content['access_token']

# Add better error handling there
# Max tokens
# Wrong Username/Passsword
# Other
except: # noqa E722
logging.warning("Error getting the token - check user name and password", exc_info=True)
return token


###############################################################################
# DELETE TOKEN FROM CMR
###############################################################################
def delete_token(url: str, token: str) -> bool:
try:
username, _, password = netrc.netrc().authenticators(edl)
headers: Dict = {'Accept': 'application/json'}
resp = requests.post(url+"/revoke_token",params={"token":token}, headers=headers, auth=HTTPBasicAuth(username, password))

if resp.status_code == 200:
logging.info("EDL token successfully deleted")
return True
else:
logging.info("EDL token deleting failed.")

except: # noqa E722
logging.warning("Error deleting the token", exc_info=True)

return False

def list_tokens(url: str):
try:
tokens = []
username, _, password = netrc.netrc().authenticators(edl)
headers: Dict = {'Accept': 'application/json'} # noqa E501
resp = requests.get(url+"/tokens", headers=headers, auth=HTTPBasicAuth(username, password))
response_content = json.loads(resp.content)

for x in response_content:
tokens.append(x['access_token'])

except: # noqa E722
logging.warning("Error getting the token - check user name and password", exc_info=True)
return tokens


def refresh_token(old_token: str):
setup_earthdata_login_auth(edl)
delete_token(token_url,old_token)
return get_token(token_url)


def validate(args):
if args.bbox is not None:
bounds = args.bbox.split(',')
Expand Down
28 changes: 14 additions & 14 deletions subscriber/podaac_data_downloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,17 @@
from os.path import isdir, basename, join, exists
from urllib.error import HTTPError

import earthaccess
from subscriber import podaac_access as pa
from subscriber import subsetting
from subscriber import token_formatter

__version__ = pa.__version__

page_size = 2000
edl = pa.edl
# edl = pa.edl
cmr = pa.cmr
token_url = pa.token_url
# token_url = pa.token_url

# The lines below are to get the IP address. You can make this static and
# assign a fixed value to the IPAddr variable
Expand Down Expand Up @@ -138,8 +139,8 @@ def run(args=None):
logging.error(str(v))
exit(1)

pa.setup_earthdata_login_auth(edl)
token = pa.get_token(token_url)
earthaccess.login(strategy="netrc")
token = earthaccess.get_edl_token()["access_token"]

data_path = args.outputDirectory
if not isdir(data_path):
Expand Down Expand Up @@ -261,12 +262,13 @@ def cmr_downloader(args, token, data_path):
results = pa.get_search_results(params, args.verbose)
except HTTPError as e:
if e.code == 401:
token = pa.refresh_token(token)
# Updated: This is not always a dictionary...
# in fact, here it's always a list of tuples
for i, p in enumerate(params):
if p[1] == "token":
params[i] = ("token", token)
# token = pa.refresh_token(token)
# # Updated: This is not always a dictionary...
# # in fact, here it's always a list of tuples
# for i, p in enumerate(params):
# if p[1] == "token":
# params[i] = ("token", token)
token = earthaccess.get_edl_token()["access_token"]
results = pa.get_search_results(params, args.verbose)
else:
raise e
Expand Down Expand Up @@ -294,10 +296,8 @@ def cmr_downloader(args, token, data_path):
results['items']]
checksums = pa.extract_checksums(results['items'])

for f in downloads_data:
downloads_all.append(f)
for f in downloads_metadata:
downloads_all.append(f)
downloads_all.extend(downloads_data)
downloads_all.extend(downloads_metadata)

downloads = [item for sublist in downloads_all for item in sublist]

Expand Down
24 changes: 13 additions & 11 deletions subscriber/podaac_data_subscriber.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from os.path import isdir, basename, join, isfile, exists
from urllib.error import HTTPError

import earthaccess
from subscriber import podaac_access as pa
from subscriber import subsetting
from subscriber import token_formatter
Expand All @@ -29,9 +30,9 @@

page_size = 2000

edl = pa.edl
# edl = pa.edl
cmr = pa.cmr
token_url = pa.token_url
# token_url = pa.token_url


def get_update_file(data_dir, collection_name):
Expand Down Expand Up @@ -126,8 +127,8 @@ def run(args=None):
logging.error(str(v))
exit(1)

pa.setup_earthdata_login_auth(edl)
token = pa.get_token(token_url)
earthaccess.login(strategy="netrc")
token = earthaccess.get_edl_token()["access_token"]

mins = args.minutes # In this case download files ingested in the last 60 minutes -- change this to whatever setting is needed
provider = args.provider
Expand Down Expand Up @@ -223,13 +224,14 @@ def run(args=None):
results = pa.get_search_results(params, args.verbose)
except HTTPError as e:
if e.code == 401:
token = pa.refresh_token(token)
# Updated: This is not always a dictionary...
# in fact, here it's always a list of tuples
for i, p in enumerate(params) :
if p[1] == "token":
params[i] = ("token", token)
#params['token'] = token
# token = pa.refresh_token(token)
# # Updated: This is not always a dictionary...
# # in fact, here it's always a list of tuples
# for i, p in enumerate(params) :
# if p[1] == "token":
# params[i] = ("token", token)
# #params['token'] = token
token = earthaccess.get_edl_token()["access_token"]
results = pa.get_search_results(params, args.verbose)
Comment thread
jpl-btlunsfo marked this conversation as resolved.
Outdated
else:
raise e
Expand Down
47 changes: 0 additions & 47 deletions tests/test_token_regression.py
View file
Open in desktop
Comment thread
jpl-btlunsfo marked this conversation as resolved.
Outdated

This file was deleted.

Loading