Adjusting data downloader and subscriber to use earthaccess login methods

.github/workflows/python-app.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       max-parallel: 2
       matrix:
-        python-version: [ "3.10", "3.11", "3.12", "3.13" ]
+        python-version: [ "3.11", "3.12", "3.13" ]
         poetry-version: [ "1.8.4" ]
         os: [ ubuntu-22.04, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
@@ -62,7 +62,7 @@ jobs:
       fail-fast: false
       max-parallel: 1
       matrix:
-        python-version: [ "3.10", "3.11", "3.12", "3.13" ]
+        python-version: ["3.11", "3.12", "3.13" ]
         poetry-version: [ "1.8.4" ]
         os: [ ubuntu-22.04, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}

.github/workflows/release.yml

@@ -13,10 +13,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.10
+    - name: Set up Python 3.11
       uses: actions/setup-python@v2
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install Poetry
       uses: abatilo/actions-poetry@v2.0.0
       with:
@@ -43,7 +43,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.10", "3.11", "3.12", "3.13" ]
+        python-version: [ "3.11", "3.12", "3.13" ]
         os: [ ubuntu-22.04, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
     steps:

CHANGELOG.md

@@ -12,6 +12,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - **PODAAC-6756 (issues/181)**
   - Validate and added support for 3.12 and 3.13
   - Remove support for 3.8 and 3.9
+- **(issues/175)**
+  - Changed authentication flow to use earthaccess module
+  - Remove support for 3.10
 ### Fixed
 - **PODAAC-6303 (issues/167)**
   - Fixed issue where -gr and -sd/-ed (temporal) cannot be used together as a query

README.md

@@ -26,7 +26,7 @@ The subscriber is useful for users who need to continuously pull the latest data
 
 ## Installation
 
-Both subscriber and downloader require Python >= 3.10.
+Both subscriber and downloader require Python >= 3.11.
 
 The subscriber and downloader scripts are available in the [pypi python repository](https://pypi.org/project/podaac-data-subscriber/), it can be installed via pip:
 

Subscriber.md

@@ -36,6 +36,7 @@ $> podaac-data-subscriber -h
                       Specify a provider for collection search. Default is POCLOUD.
 --dry-run             Search and identify files to download, but do not actually download them
 --subset              Flag to enable subsetting on the specified collection
+--collection-version  Restrict download to files within a specific collection version
 ```
 
 ## Run the Script

pyproject.toml

@@ -12,11 +12,12 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.10,<4.0"
+python = ">=3.11,<4.0"
 requests = "^2.27.1"
 tenacity = "^8.0.1"
 packaging = "^23.0"
 harmony-py = "^0.4.12"
+earthaccess = "^0.17.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.2"

subscriber/podaac_access.py

@@ -15,6 +15,7 @@
 from urllib.request import Request, urlopen
 import hashlib
 import time
+import earthaccess
 from requests.auth import HTTPBasicAuth
 import harmony
 import concurrent.futures
@@ -37,145 +38,6 @@
 
 IPAddr = "127.0.0.1"  # socket.gethostbyname(hostname)
 
-
-# ## Authentication setup
-#
-# The function below will allow Python scripts to log into any Earthdata Login
-#  application programmatically.  To avoid being prompted for
-# credentials every time you run and also allow clients such as curl to log in,
-#  you can add the following to a `.netrc` (`_netrc` on Windows) file in
-#  your home directory:
-#
-# ```
-# machine urs.earthdata.nasa.gov
-#     login <your username>
-#     password <your password>
-# ```
-#
-# Make sure that this file is only readable by the current user
-# or you will receive an error stating
-# "netrc access too permissive."
-#
-# `$ chmod 0600 ~/.netrc`
-#
-# You'll need to authenticate using the netrc method when running from
-# command line with [`papermill`](https://papermill.readthedocs.io/en/latest/).
-# You can log in manually by executing the cell below when running in the
-# notebook client in your browser.*
-
-
-def setup_earthdata_login_auth(endpoint):
-    """
-    Set up the request library so that it authenticates against the given
-    Earthdata Login endpoint and is able to track cookies between requests.
-    This looks in the .netrc file first and if no credentials are found,
-    it prompts for them.
-
-    Valid endpoints include:
-        urs.earthdata.nasa.gov - Earthdata Login production
-    """
-    try:
-        username, _, password = netrc.netrc().authenticators(endpoint)
-    except (FileNotFoundError, TypeError):
-        # FileNotFound = There's no .netrc file
-        # TypeError = The endpoint isn't in the netrc file,
-        #  causing the above to try unpacking None
-        logging.warning("There's no .netrc file or the The endpoint isn't in the netrc file")
-
-    manager = request.HTTPPasswordMgrWithDefaultRealm()
-    manager.add_password(None, endpoint, username, password)
-    auth = request.HTTPBasicAuthHandler(manager)
-
-    jar = CookieJar()
-    processor = request.HTTPCookieProcessor(jar)
-    opener = request.build_opener(auth, processor)
-    opener.addheaders = [('User-agent', 'podaac-subscriber-' + __version__)]
-    request.install_opener(opener)
-
-
-
-def get_token(url: str) -> str:
-    tokens = list_tokens(url)
-    if len(tokens) == 0 :
-        return create_token(url)
-    else:
-        return tokens[0]
-
-###############################################################################
-# GET TOKEN FROM CMR
-###############################################################################
-@tenacity.retry(wait=tenacity.wait_random_exponential(multiplier=1, max=60),
-                stop=tenacity.stop_after_attempt(3),
-                reraise=True,
-                retry=(tenacity.retry_if_result(lambda x: x == ''))
-                )
-def create_token(url: str) -> str:
-    try:
-        token: str = ''
-        username, _, password = netrc.netrc().authenticators(edl)
-        headers: Dict = {'Accept': 'application/json'}  # noqa E501
-
-
-        resp = requests.post(url+"/token", headers=headers, auth=HTTPBasicAuth(username, password))
-        response_content: Dict = json.loads(resp.content)
-        if "error" in response_content:
-            if response_content["error"] == "max_token_limit":
-                logging.error("Max tokens acquired from URS. Using existing token")
-                tokens=list_tokens(url)
-                return tokens[0]
-        token = response_content['access_token']
-
-    # Add better error handling there
-    # Max tokens
-    # Wrong Username/Passsword
-    # Other
-    except:  # noqa E722
-        logging.warning("Error getting the token - check user name and password", exc_info=True)
-    return token
-
-
-###############################################################################
-# DELETE TOKEN FROM CMR
-###############################################################################
-def delete_token(url: str, token: str) -> bool:
-    try:
-        username, _, password = netrc.netrc().authenticators(edl)
-        headers: Dict = {'Accept': 'application/json'}
-        resp = requests.post(url+"/revoke_token",params={"token":token}, headers=headers, auth=HTTPBasicAuth(username, password))
-
-        if resp.status_code == 200:
-            logging.info("EDL token successfully deleted")
-            return True
-        else:
-            logging.info("EDL token deleting failed.")
-
-    except:  # noqa E722
-        logging.warning("Error deleting the token", exc_info=True)
-
-    return False
-
-def list_tokens(url: str):
-    try:
-        tokens = []
-        username, _, password = netrc.netrc().authenticators(edl)
-        headers: Dict = {'Accept': 'application/json'}  # noqa E501
-        resp = requests.get(url+"/tokens", headers=headers, auth=HTTPBasicAuth(username, password))
-        response_content = json.loads(resp.content)
-
-        for x in response_content:
-            tokens.append(x['access_token'])
-
-    except:  # noqa E722
-        logging.warning("Error getting the token - check user name and password", exc_info=True)
-    return tokens
-
-
-def refresh_token(old_token: str):
-    setup_earthdata_login_auth(edl)
-    delete_token(token_url,old_token)
-    return get_token(token_url)
-
-
 def validate(args):
     if args.bbox is not None:
         bounds = args.bbox.split(',')
@@ -230,6 +92,14 @@ def validate(args):
             )
 
 
+def refresh_token(params: list=None ) -> tuple[str, list] | str:
+    earthaccess.login(strategy="netrc")
+    token = earthaccess.get_edl_token()["access_token"]
+    if params:
+        return token, [('token', token) if param[0] == 'token' else param for param in params]
+    return token
+
+
 def check_dir(path):
     if not isdir(path):
         makedirs(path, exist_ok=True)
@@ -380,8 +250,7 @@ def download_file(remote_file, output_path, retries=3):
                 )
 def get_search_results(params, verbose=False):
     # Get the query parameters as a string and then the complete search url:
-    query = urlencode(params)
-    url = "https://" + cmr + "/search/granules.umm_json?" + query
+    url = f"https://{cmr}/search/collections.umm_json?{urlencode(params)}"
     if verbose:
         logging.info(url)
 
@@ -395,7 +264,17 @@ def get_search_results(params, verbose=False):
         req = Request(url)
         if search_after_header is not None:
             req.add_header('CMR-Search-After', search_after_header)
-        response = urlopen(req)
+        try:
+            response = urlopen(req)
+        except HTTPError as e:
+            if e.code == 401:
+                _, params = refresh_token(params)
+                req = Request(f"https://{cmr}/search/collections.umm_json?{urlencode(params)}")
+                if search_after_header:
+                    req.add_header('CMR-Search-After', search_after_header)
+                response = urlopen(req)
+            else:
+                raise e
 
         # Build the results object, load entire result if it's the first time.
         if results is None:
@@ -523,14 +402,20 @@ def make_checksum(file_path, algorithm):
     return hash_alg.hexdigest()
 
 def get_cmr_collections(params, verbose=False):
-    query = urlencode(params)
-    url = "https://" + cmr + "/search/collections.umm_json?" + query
+    url = f"https://{cmr}/search/collections.umm_json?{urlencode(params)}"
     if verbose:
         logging.info(url)
 
     # Build the request, add the search after header to it if it's not None (e.g. after the first iteration)
-    req = Request(url)
-    response = urlopen(req)
+    try:
+        response = urlopen(Request(url))
+    except HTTPError as e:
+        if e.code == 401:
+            # try to refresh the token in params
+            _, params = refresh_token(params)
+            response = urlopen(Request(f"https://{cmr}/search/collections.umm_json?{urlencode(params)}"))
+        else:
+            raise e
     result = json.loads(response.read().decode())
     return result
 

subscriber/podaac_data_downloader.py

@@ -15,9 +15,9 @@
 __version__ = pa.__version__
 
 page_size = 2000
-edl = pa.edl
+# edl = pa.edl
 cmr = pa.cmr
-token_url = pa.token_url
+# token_url = pa.token_url
 
 # The lines below are to get the IP address. You can make this static and
 # assign a fixed value to the IPAddr variable
@@ -138,8 +138,7 @@ def run(args=None):
         logging.error(str(v))
         exit(1)
 
-    pa.setup_earthdata_login_auth(edl)
-    token = pa.get_token(token_url)
+    token = pa.refresh_token()
 
     data_path = args.outputDirectory
     if not isdir(data_path):
@@ -256,20 +255,7 @@ def cmr_downloader(args, token, data_path):
     # Final token appending; seems to bug urlencode(params) when it's not last
     params.append(('token', token))
 
-    # If 401 is raised, refresh token and try one more time
-    try:
-        results = pa.get_search_results(params, args.verbose)
-    except HTTPError as e:
-        if e.code == 401:
-            token = pa.refresh_token(token)
-            # Updated: This is not always a dictionary...
-            # in fact, here it's always a list of tuples
-            for i, p in enumerate(params):
-                if p[1] == "token":
-                    params[i] = ("token", token)
-            results = pa.get_search_results(params, args.verbose)
-        else:
-            raise e
+    results = pa.get_search_results(params, args.verbose)
 
     if args.verbose:
         logging.info(str(results['hits']) + " granules found for " + short_name)  # noqa E501
@@ -294,10 +280,8 @@ def cmr_downloader(args, token, data_path):
                           results['items']]
     checksums = pa.extract_checksums(results['items'])
 
-    for f in downloads_data:
-        downloads_all.append(f)
-    for f in downloads_metadata:
-        downloads_all.append(f)
+    downloads_all.extend(downloads_data)
+    downloads_all.extend(downloads_metadata)
 
     downloads = [item for sublist in downloads_all for item in sublist]