main.yml

# This is a basic workflow to help you get started with Actions

name: update-data

# Controls when the action will run. 
on:
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch: {}
  push:
    branches:
    - master
  schedule:
    - cron: "20 1 * * *"

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  # This workflow contains a single job called "build"
  fetch-data:
    runs-on: ubuntu-latest
    steps:
      - name: Setup deno
        uses: denoland/setup-deno@v1
        with:
          deno-version: v1.x
      - name: Check out repo
        uses: actions/checkout@v2
        with:
          ref: ${{ github.ref }}
      - name: Fetch data
        uses: githubocto/flat@v3
        with:
          http_url: https://www.ndbc.noaa.gov/activestations.xml
          downloaded_filename: ./noaa-ndbc-data/active-stations.xml
      - name: Fetch data
        uses: githubocto/flat@v3
        with:
          http_url: https://www.ndbc.noaa.gov/data/latest_obs/latest_obs.txt
          downloaded_filename: ./noaa-ndbc-data/latest-observations.txt
          postprocess: ./noaa-ndbc-data/postprocess.js
      - name: Fetch data
        uses: githubocto/flat@v3
        with:
          http_url: https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_week.geojson
          downloaded_filename: ./usgs-earthquake-data/all_week.geojson
          postprocess: ./usgs-earthquake-data/postprocess.js
  process-data:
    # The type of runner that the job will run on
    runs-on: ubuntu-latest
    needs: fetch-data
    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2
        with:
          ref: ${{ github.ref }}
      - name: Setup Ruby, JRuby and TruffleRuby
        uses: ruby/setup-ruby@v1.72.1
        with:
          ruby-version: 3.0.1
      - name: InfluxDB Action
        uses: influxdata/influxdb-action@v3
        with:
          influxdb_version: 2.0.7
          influxdb_org: influxdata
          influxdb_user: ci_user
          influxdb_password: password
          influxdb_bucket: dummy
      - name: Generate Air Sensor LP
        run: |
          ruby $GITHUB_WORKSPACE/air-sensor-data/air-sensor-data.rb > $GITHUB_WORKSPACE/air-sensor-data/air-sensor-data.lp
      - name: Generate Air Sensor CSV
        run: |
          /usr/local/bin/influx write -f $GITHUB_WORKSPACE/air-sensor-data/air-sensor-data.lp -b dummy
          /usr/local/bin/influx query "from(bucket: \"dummy\") |> range(start: -1y) |> drop(columns: [\"_start\",\"_stop\"])" --raw > $GITHUB_WORKSPACE/air-sensor-data/air-sensor-data-annotated.csv
          /usr/local/bin/influx bucket delete -n dummy
      - name: Generate NOAA LP and CSV
        run: |
          /usr/local/bin/influx bucket create -n dummy
          /usr/local/bin/influx write dryrun -f $GITHUB_WORKSPACE/noaa-ndbc-data/latest-observations.csv --format csv --header "#constant measurement,ndbc" --header "#datatype double,double,double,double,double,double,long,double,double,double,double,double,double,double,tag,double,double,double,tag,tag,tag,tag,string,string,string,string,dateTime:number" > $GITHUB_WORKSPACE/noaa-ndbc-data/latest-observations.lp
          /usr/local/bin/influx write -f $GITHUB_WORKSPACE/noaa-ndbc-data/latest-observations.lp -b dummy
          /usr/local/bin/influx query "from(bucket: \"dummy\") |> range(start: -1y) |> drop(columns: [\"_start\",\"_stop\"])" --raw > $GITHUB_WORKSPACE/noaa-ndbc-data/latest-observations-annotated.csv
          /usr/local/bin/influx bucket delete -n dummy
      - name: Generate USGS CSV
        run: |
          /usr/local/bin/influx bucket create -n dummy
          /usr/local/bin/influx write -f $GITHUB_WORKSPACE/usgs-earthquake-data/all_week.lp -b dummy
          /usr/local/bin/influx query "from(bucket: \"dummy\") |> range(start: -1y) |> drop(columns: [\"_start\",\"_stop\"])" --raw > $GITHUB_WORKSPACE/usgs-earthquake-data/all_week-annotated.csv
          /usr/local/bin/influx bucket delete -n dummy
      - name: Commit changes
        uses: stefanzweifel/git-auto-commit-action@v4
        with:
          commit_message: Adding LP and annotated CSV file
  sync-to-s3:
    runs-on: ubuntu-latest
    needs: process-data
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          ref: ${{ github.ref }}
      - name: Copy Air Sensor Data to S3
        uses: prewk/s3-cp-action@v2
        with:
          aws_access_key_id: ${{ secrets.S3_TOKEN }}
          aws_secret_access_key: ${{ secrets.S3_KEY }}
          aws_region: ${{ secrets.S3_REGION }}
          source: './air-sensor-data/air-sensor-data-annotated.csv'
          dest: 's3://${{ secrets.S3_BUCKET }}/air-sensor-data-annotated.csv'
          flags: --acl public-read --follow-symlinks
      - name: Copy NOAA Data to S3
        uses: prewk/s3-cp-action@v2
        with:
          aws_access_key_id: ${{ secrets.S3_TOKEN }}
          aws_secret_access_key: ${{ secrets.S3_KEY }}
          aws_region: ${{ secrets.S3_REGION }}
          source: './noaa-ndbc-data/latest-observations-annotated.csv'
          dest: 's3://${{ secrets.S3_BUCKET }}/noaa-ndbc-latest-observations-annotated.csv'
          flags: --acl public-read --follow-symlinks
      - name: Copy USGS Data to S3
        uses: prewk/s3-cp-action@v2
        with:
          aws_access_key_id: ${{ secrets.S3_TOKEN }}
          aws_secret_access_key: ${{ secrets.S3_KEY }}
          aws_region: ${{ secrets.S3_REGION }}
          source: './usgs-earthquake-data/all_week-annotated.csv'
          dest: 's3://${{ secrets.S3_BUCKET }}/usgs-earthquake-all-week-annotated.csv'
          flags: --acl public-read --follow-symlinks