fetch-marketdata #172
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: fetch-marketdata | |
| on: | |
| # Nasdaq stock screener does not provide real-time data anymore | |
| # Pre-Market Trading: 9:00 AM - 2:30 PM UTC | |
| # Regular Session: 2:30 PM - 9:00 PM UTC | |
| # After-Hours Trading: 9:00 PM - 1:00 AM UTC (the following day) | |
| schedule: | |
| # - cron: '55 9-23 * * 1-5' | |
| # - cron: '55 0,1 * * 2-6' | |
| - cron: '55 1-11 * * 2-6' | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| fetch: | |
| runs-on: ubuntu-latest | |
| env: | |
| CI_COMMIT_AUTHOR: github-actions[bot] | |
| CI_COMMIT_AUTHOR_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com | |
| URL: https://api.nasdaq.com/api | |
| USER_AGENT: Mozilla/5.0 (iPad; CPU OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1 | |
| steps: | |
| - name : Set up environment variables | |
| run: | | |
| DATE=$(TZ='US/Eastern' date --date yesterday +'%Y/%m/%d') | |
| DATA_DIR=marketdata/$DATE | |
| echo "DATE=$DATE" >> $GITHUB_ENV | |
| echo "DATA_DIR=$DATA_DIR" >> $GITHUB_ENV | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| filter: blob:none | |
| sparse-checkout: | | |
| ${{ env.DATA_DIR }} | |
| sparse-checkout-cone-mode: true | |
| - name: Get equities | |
| run: | | |
| set -euo pipefail | |
| TMP="$(mktemp)" | |
| trap "rm -f '$TMP'" EXIT | |
| for exchange in "nasdaq" "nyse" "amex"; do | |
| response_code=$(curl --request GET \ | |
| --url "$URL/screener/stocks?tableonly=true&limit=25&offset=0&download=true&exchange=${exchange}" \ | |
| --user-agent "$USER_AGENT" \ | |
| --silent \ | |
| --write-out "%{http_code}" \ | |
| --output "$TMP") | |
| if [ "${response_code}" -ne 200 ]; then | |
| echo "::error::Request failed with response code: ${response_code}" | |
| exit 1 | |
| fi | |
| length=$(jq '.data.rows | length' "$TMP") | |
| if [ "$length" -eq "0" ]; then | |
| echo "::error::No data for ${exchange}" | |
| exit 1 | |
| fi | |
| jq '.data.rows' "$TMP" > "${exchange}.json" | |
| done | |
| - name: Get ETFs | |
| continue-on-error: true | |
| run: | | |
| set -euo pipefail | |
| response_code=$(curl --request GET \ | |
| --url "$URL/screener/etf?tableonly=true&limit=25&offset=0&download=true" \ | |
| --user-agent "$USER_AGENT" \ | |
| --silent \ | |
| --write-out "%{http_code}" \ | |
| --output "us-etf.json") | |
| if [ "${response_code}" -ne 200 ]; then | |
| echo "::error::Request failed with response code: ${response_code}" | |
| exit 1 | |
| fi | |
| length=$(jq '.data.data.rows | length' us-etf.json) | |
| if [ "$length" -eq "0" ]; then | |
| echo "::error::No ETF data" | |
| rm us-etf.json | |
| exit 1 | |
| fi | |
| - name: Commit and push | |
| run: | | |
| mkdir -p "$DATA_DIR/raw" | |
| mv *.json "${DATA_DIR}/raw/" | |
| git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}" | |
| git config --global user.email "${{ env.CI_COMMIT_AUTHOR_EMAIL }}" | |
| COMMIT_MESSAGE="Fetch: $(TZ='US/Eastern' date)" | |
| git add marketdata/ | |
| git diff-index --quiet HEAD || git commit -m "$COMMIT_MESSAGE" | |
| git push |