Test against all NOAA stations, compute benchmarks

Author: bkeepers

.github/workflows/test.yml

@@ -22,3 +22,9 @@ jobs:
         with:
           fail_ci_if_error: true
           files: ./coverage/coverage-final.json
+
+      - name: Upload NOAA summary
+        uses: actions/upload-artifact@v6
+        with:
+          name: noaa-benchmarks
+          path: noaa-benchmarks.csv

package.json

@@ -19,6 +19,7 @@
   ],
   "devDependencies": {
     "@eslint/js": "^9.39.2",
+    "@neaps/tide-stations": "github:neaps/tide-database",
     "@types/node": "^20.10.6",
     "@vitest/coverage-v8": "^3.0.0",
     "eslint": "^9.39.2",

test/noaa.ts

@@ -1,19 +1,118 @@
-import { describe, it, expect } from 'vitest'
+import { describe, it, expect, afterAll } from 'vitest'
 import fs from 'fs/promises'
 import tidePrediction from '../src/index.js'
+import db from '@neaps/tide-stations'
+import { createWriteStream } from 'fs'
+
+const noaa = db.filter((station) =>
+  station.source.source_url.includes('noaa.gov')
+)
 
 // Create a directory for test cache
 await fs.mkdir('./.test-cache', { recursive: true })
 
-const stations = ['9413450', '9411340', '2695535', '8761724', '8410140']
+interface Stat {
+  station: string
+  count: number
+  mae: number
+  rmse: number
+  bias: number
+}
+
+const stats: Stat[] = []
+
+const knownOutliers = [
+  '9455920',
+  '9455963'
+  // ...
+]
+
+describe('NOAA benchmarks', () => {
+  noaa.forEach((station) => {
+    it(
+      station.source.id,
+      async () => {
+        const { harmonics, levels } = await getStation(station.source.id)
+        const tideStation = tidePrediction(harmonics.HarmonicConstituents)
+
+        // No predictions available
+        if (!levels.predictions) return
+
+        let count = 0
+        let sumError = 0
+        let sumAbsError = 0
+        let sumSqError = 0
+
+        levels.predictions.forEach((prediction: { t: string; v: string }) => {
+          const expected = parseFloat(prediction.v)
+
+          const { level: actual } = tideStation.getWaterLevelAtTime({
+            time: new Date(prediction.t)
+          })
+
+          const error = actual - expected
+
+          count += 1
+          sumError += error
+          sumAbsError += Math.abs(error)
+          sumSqError += error * error
+        })
+
+        const mae = sumAbsError / count
+        const rmse = Math.sqrt(sumSqError / count)
+        const bias = sumError / count
+
+        if (knownOutliers.includes(station.source.id)) {
+          expect(
+            mae,
+            `Remove ${station.source.id} from known outliers`
+          ).toBeGreaterThan(0.15)
+          expect(mae).toBeLessThan(0.3)
+        } else {
+          expect(mae).toBeLessThanOrEqual(0.15)
+        }
 
-const makeRequest = async (url: string) => {
+        stats.push({ station: station.source.id, count, mae, rmse, bias })
+      },
+      20000
+    )
+  })
+
+  afterAll(async () => {
+    // Write stats to file for later analysis
+    const summary = createWriteStream('noaa-benchmarks.csv')
+    summary.write('station,count,mae_m,rmse_m,bias_m\n')
+    stats.forEach(({ station, count, mae, rmse, bias }) => {
+      summary.write(
+        [station, count, mae.toFixed(4), rmse.toFixed(4), bias.toFixed(4)].join(
+          ','
+        ) + '\n'
+      )
+    })
+
+    // Baseline expectations based on current performance. The goal should be to move these toward zero over time.
+    const maeValues = stats.map((s) => s.mae).sort((a, b) => a - b)
+    const medianMAE = maeValues[Math.floor(stats.length / 2)]
+    const p90MAE = maeValues[Math.floor(stats.length * 0.9)]
+    const p95MAE = maeValues[Math.floor(stats.length * 0.95)]
+
+    expect(medianMAE).toBeLessThan(0.03) // 3 cm
+    expect(p90MAE).toBeLessThan(0.06) // 6 cm
+    expect(p95MAE).toBeLessThan(0.08) // 8 cm
+    expect(stats.length).toBeGreaterThanOrEqual(1187) // Ensure enough stations were tested
+
+    // Ensure only a small number of stations have very high errors
+    expect(stats.filter(({ mae }) => mae > 0.1).length).toBeLessThan(10)
+  })
+})
+
+async function makeRequest(url: string) {
   const res = await fetch(url)
   if (!res.ok) throw new Error(`Request failed with status ${res.status}`)
   return res.json()
 }
 
-const getStation = async (station: string) => {
+async function getStation(station: string) {
   const filePath = `./.test-cache/${station}.json`
 
   try {
@@ -36,18 +135,3 @@ const getStation = async (station: string) => {
     return data
   }
 }
-
-describe('Results compare to NOAA', () => {
-  stations.forEach((station) => {
-    it(`it compares with station ${station}`, async () => {
-      const { harmonics, levels } = await getStation(station)
-      const tideStation = tidePrediction(harmonics.HarmonicConstituents)
-      levels.predictions.forEach((prediction: { t: string; v: string }) => {
-        const neapsPrediction = tideStation.getWaterLevelAtTime({
-          time: new Date(prediction.t)
-        })
-        expect(neapsPrediction.level).toBeCloseTo(parseFloat(prediction.v), 0)
-      })
-    }, 20000)
-  })
-})