podaac
diff --git a/‎.github/workflows/python-app.yml
+40-2 b/‎.github/workflows/python-app.yml
+40-2
diff --git a/‎.github/workflows/release.yml
+1-1 b/‎.github/workflows/release.yml
+1-1
diff --git a/‎CHANGELOG.md
+8 b/‎CHANGELOG.md
+8
diff --git a/‎Downloader.md
+26-21 b/‎Downloader.md
+26-21
diff --git a/‎README.md
+13-5 b/‎README.md
+13-5
@@ -17,7 +17,7 @@ jobs:
       matrix:
         python-version: [ "3.7", "3.8", "3.9", "3.10" ]
         poetry-version: [ "1.1.14" ]
-        os: [ ubuntu-18.04, macos-latest, windows-latest ]
+        os: [ ubuntu-22.04, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
@@ -39,8 +39,10 @@ jobs:
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        poetry run pytest -m "not regression"
+        poetry run pytest -m "not regression and not token"
     - name: netrc-gen
       uses: extractions/netrc@v1
       with:
@@ -50,5 +52,41 @@ jobs:
     - name: Regression Test with pytest
       env:
         PODAAC_LOGLEVEL: "DEBUG"
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
         poetry run pytest -o log_cli=true --log-cli-level=DEBUG -m "regression"
+
+  auth_regression:
+    needs: build
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+        poetry-version: [ "1.1.14" ]
+        os: [ ubuntu-22.04, macos-latest, windows-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: abatilo/[email protected]
+      with:
+        poetry-version: ${{ matrix.poetry-version }}
+    - name: Install dependencies
+      run: |
+        poetry install
+    - name: netrc-gen
+      uses: extractions/netrc@v1
+      with:
+       machine: urs.earthdata.nasa.gov
+       username: ${{ secrets.EDL_OPS_USERNAME }}
+       password: ${{ secrets.EDL_OPS_PASSWORD }}
+    - name: Regression Test with pytest
+      env:
+        PODAAC_LOGLEVEL: "DEBUG"
+      run: |
+        poetry run pytest -o log_cli=true --log-cli-level=DEBUG -m "token"
@@ -44,7 +44,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [ "3.7", "3.8", "3.9", "3.10" ]
-        os: [ ubuntu-18.04, macos-latest, windows-latest ]
+        os: [ ubuntu-22.04, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Set up Python
 
@@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
+## 1.13.0
+### Added
+- Added --dry-run option to subscriber and downloader to view the files that _would_ be downloaded without actuall downloading them. [102](https://github.com/podaac/data-subscriber/issues/102)
+- Added new feature allowing regex to be used in `--extension` `-e` options. For example using -e `PTM_\\d+` would match data files like `filename.PTM_1`, `filename.PTM_2` and `filename.PTM_10`, instead of specifying all possible combinations (`-e PTM_1, -e PTM_2, ...,  -e PMT_10`) [115](https://github.com/podaac/data-subscriber/issues/115)
+- Added check for updated version [70](https://github.com/podaac/data-subscriber/issues/70)
+- Removed CMR Token from log messages [127](https://github.com/podaac/data-subscriber/issues/127)
+
+
 ## 1.12.0
 ### Fixed
 - Added EDL based token downloading, removing CMR tokens [98](https://github.com/podaac/data-subscriber/issues/98),
 
@@ -6,7 +6,7 @@ For installation and dependency information, please see the [top-level README](R
 
 ```
 $> podaac-data-downloader -h
-usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [--granule-name GRANULE] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--limit LIMIT]
+usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [-gr GRANULENAME] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--limit LIMIT] [--dry-run]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -17,44 +17,39 @@ optional arguments:
   --cycle SEARCH_CYCLES
                         Cycle number for determining downloads. can be repeated for multiple cycles
   -sd STARTDATE, --start-date STARTDATE
-                        The ISO date time before which data should be retrieved. For Example, --start-date 2021-01-14T00:00:00Z
+                        The ISO date time after which data should be retrieved. For Example, --start-date 2021-01-14T00:00:00Z
   -ed ENDDATE, --end-date ENDDATE
-                        The ISO date time after which data should be retrieved. For Example, --end-date 2021-01-14T00:00:00Z
-   -f, --force          
-                        Flag to force downloading files that are listed in CMR query, even if the file exists and checksum matches
+                        The ISO date time before which data should be retrieved. For Example, --end-date 2021-01-14T00:00:00Z
+  -f, --force           Flag to force downloading files that are listed in CMR query, even if the file exists and checksum matches
   -b BBOX, --bounds BBOX
-                        The bounding rectangle to filter result in. Format is W Longitude,S Latitude,E Longitude,N Latitude without
-                        spaces. Due to an issue with parsing arguments, to use this command, please use the -b="-180,-90,180,90" syntax
-                        when calling from the command line. Default: "-180,-90,180,90".
+                        The bounding rectangle to filter result in. Format is W Longitude,S Latitude,E Longitude,N Latitude without spaces. Due to an issue with parsing arguments, to use this command, please use the -b="-180,-90,180,90" syntax when calling from the command line.
+                        Default: "-180,-90,180,90".
   -dc                   Flag to use cycle number for directory where data products will be downloaded.
   -dydoy                Flag to use start time (Year/DOY) of downloaded data for directory where data products will be downloaded.
-  -dymd                 Flag to use start time (Year/Month/Day) of downloaded data for directory where data products will be
-                        downloaded.
+  -dymd                 Flag to use start time (Year/Month/Day) of downloaded data for directory where data products will be downloaded.
   -dy                   Flag to use start time (Year) of downloaded data for directory where data products will be downloaded.
   --offset OFFSET       Flag used to shift timestamp. Units are in hours, e.g. 10 or -10.
   -e EXTENSIONS, --extensions EXTENSIONS
-                        The extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz]
-  -gr GRANULE, --granule-name GRANULE
-  						The name of the granule to download. Only one granule name can be specified. Script will download all files matching similar granule name sans extension.
+                        Regexps of extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz, .tiff]
+  -gr GRANULENAME, --granule-name GRANULENAME
+                        Flag to download specific granule from a collection. This parameter can only be used if you know the granule name. Only one granule name can be supplied
   --process PROCESS_CMD
                         Processing command to run on each downloaded file (e.g., compression). Can be specified multiple times.
   --version             Display script version information and exit.
   --verbose             Verbose mode.
   -p PROVIDER, --provider PROVIDER
                         Specify a provider for collection search. Default is POCLOUD.
-  --limit LIMIT         Integer limit for number of granules to download. Useful in testing. Defaults to 2000
+  --limit LIMIT         Integer limit for number of granules to download. Useful in testing. Defaults to no limit.
+  --dry-run             Search and identify files to download, but do not actually download them
 
 ```
 
-##Run the Script
-
 ## Step 2:  Run the Script
 
 Usage:
 ```
-usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f]
-                                    [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [--process PROCESS_CMD]
-                                    [--version] [--verbose] [-p PROVIDER] [--limit LIMIT]
+usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [-gr GRANULENAME] [--process PROCESS_CMD] [--version] [--verbose]
+                                    [-p PROVIDER] [--limit LIMIT] [--dry-run]
 ```
 
 To run the script, the following parameters are required:
@@ -215,17 +210,27 @@ podaac-data-downloader -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -b="-180,-90,180,90
 
 ### Setting extensions
 
-Some collections have many files. To download a specific set of files, you can set the extensions on which downloads are filtered. By default, ".nc", ".h5", and ".zip" files are downloaded by default.
+Some collections have many files. To download a specific set of files, you can set the extensions on which downloads are filtered. By default, ".nc", ".h5", and ".zip" files are downloaded by default. The `-e` option is a regular expression check so you can do advanced things like `-e PTM_\\d+` to match `PTM_` followed by one or more digits- useful when the ending of a file has no suffix and has a number (1-12 for PTM, in this example)
 
 ```
 -e EXTENSIONS, --extensions EXTENSIONS
-                       The extensions of products to download. Default is [.nc, .h5, .zip]
+                      Regexps of extensions of products to download. Default is [.nc, .h5, .zip, .tar.gz, .tiff]
 ```
 
 An example of the -e usage- note the -e option is additive:
 ```
 podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e .nc -e .h5 -sd 2020-06-01T00:46:02Z -ed 2020-07-01T00:46:02Z
 ```
+
+One may also specify a regular expression to select files. For example, the following are equivalent:
+
+`podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e PTM_1, -e PTM_2, ...,  -e PMT_10 -sd 2020-06-01T00:46:02Z -ed 2020-07-01T00:46:02Z`
+
+and
+
+`podaac-data-subscriber -c VIIRS_N20-OSPO-L2P-v2.61 -d ./data -e PTM_\\d+ -sd 2020-06-01T00:46:02Z -ed 2020-07-01T00:46:02Z`
+
+
 ### run a post download process
 
 Using the `--process` option, you can run a simple command agaisnt the "just" downloaded file. This will take the format of "<command> <path/to/file>". This means you can run a command like `--process gzip` to gzip all downloaded files. We do not support more advanced processes at this time (piping, running a process on a directory, etc).
 
@@ -37,16 +37,16 @@ pip install podaac-data-subscriber
 you should now have access to the downloader and subscriber Command line interfaces:
 
 ```
-$> podaac-data-subscriber -h
-usage: podaac_data_subscriber.py [-h] -c COLLECTION -d OUTPUTDIRECTORY [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-m MINUTES]
-                                 [-e EXTENSIONS] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER]
+$> usage: PO.DAAC data subscriber [-h] -c COLLECTION -d OUTPUTDIRECTORY [-f] [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-m MINUTES]
+                               [-e EXTENSIONS] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--dry-run]
 
 ...
 ```
 
 ```
-$> podaac-data-downloader -h
-usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy] [--offset OFFSET] [-e EXTENSIONS] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--limit LIMIT]
+$> usage: PO.DAAC bulk-data downloader [-h] -c COLLECTION -d OUTPUTDIRECTORY [--cycle SEARCH_CYCLES] [-sd STARTDATE] [-ed ENDDATE] [-f] [-b BBOX] [-dc] [-dydoy] [-dymd] [-dy]
+                                    [--offset OFFSET] [-e EXTENSIONS] [-gr GRANULENAME] [--process PROCESS_CMD] [--version] [--verbose] [-p PROVIDER] [--limit LIMIT] [--dry-run]
+
 
 ...
 ```
@@ -114,6 +114,14 @@ export PODAAC_LOGLEVEL=DEBUG
 
 And then run the script. This should give you more verbose output on URL requests to CMR, tokens, etc.
 
+### OTHER OPTIONS
+
+The podaac downloader and subscriber make calls to github for checking recent releases. Unauthenticated requests are limited to 60 per hour. If you start seeing errors like:
+```
+releases_json = {'documentation_url': 'https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting', 'message': "API... here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)"}
+```
+You'll want to set the environment variable GITHUB_TOKEN to a github personal access token- this allows for up to 5000 calls per hour. This requires a free github account. Most users will not run in to this issue.
+
 
 ### In need of Help?
 The PO.DAAC User Services Office is the primary point of contact for answering your questions concerning data and information held by the PO.DAAC. User Services staff members are knowledgeable about both the data ordering system and the data products themselves. We answer questions about data, route requests to other DAACs, and direct questions we cannot answer to the appropriate information source.