Skip to content

Commit 4e96cd7

Browse files
ch-basch-bas
authored
Merge pull request #229 from ch-bas/feat/ci-data-quality
ci: data-quality automation — lint gate, coverage, Wayback archiving, staleness sweep
2 parents f9994c8 + 0a2acf4 commit 4e96cd7

18 files changed

Lines changed: 729 additions & 18 deletions

‎.github/workflows/archive-sources.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: archive-sources
2+
3+
# Datasheet preservation (CI idea #3). On every push to main that touches a
4+
# camera file, submit that camera's source URLs to the Internet Archive's
5+
# Wayback "Save Page Now" API, so a permanent snapshot survives manufacturer
6+
# link-rot. Scoped to CHANGED cameras so we only archive what's new.
7+
#
8+
# Advisory only: Wayback rate-limits / downtime never fail the workflow.
9+
# Manual runs can re-archive a whole brand or the entire catalogue.
10+
on:
11+
push:
12+
branches: [main]
13+
paths:
14+
- "cameras/**"
15+
workflow_dispatch:
16+
inputs:
17+
scope:
18+
description: "Scope: 'changed' (default), a brand name (e.g. hikvision), or 'all'"
19+
required: false
20+
default: "changed"
21+
22+
permissions:
23+
contents: read
24+
25+
# One run at a time — keeps us well under Wayback's rate limits.
26+
concurrency:
27+
group: archive-sources
28+
cancel-in-progress: false
29+
30+
jobs:
31+
archive:
32+
runs-on: ubuntu-latest
33+
timeout-minutes: 30
34+
steps:
35+
- uses: actions/checkout@v4
36+
with:
37+
fetch-depth: 0 # need history to diff the pushed range
38+
- uses: actions/setup-node@v4
39+
with:
40+
node-version: "20"
41+
# No `npm ci` — scripts/archive-sources.js only uses Node built-ins + fetch.
42+
43+
- name: Archive source URLs to the Wayback Machine
44+
run: |
45+
scope="${{ github.event.inputs.scope || 'changed' }}"
46+
if [ "$scope" = "all" ]; then
47+
node scripts/archive-sources.js --all
48+
elif [ "$scope" != "changed" ] && [ -n "$scope" ]; then
49+
node scripts/archive-sources.js --brand "$scope"
50+
else
51+
base='${{ github.event.before }}'
52+
if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
53+
base="HEAD~1"
54+
fi
55+
files=$(git diff --name-only "$base" "${{ github.sha }}" -- 'cameras/**/*.json' 2>/dev/null || true)
56+
if [ -z "$files" ]; then
57+
echo "No changed camera files in this push — nothing to archive."
58+
exit 0
59+
fi
60+
echo "Changed camera files:"; echo "$files"
61+
node scripts/archive-sources.js $files
62+
fi

‎.github/workflows/build.yml‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,21 +35,31 @@ jobs:
3535
- name: Generate docs
3636
run: node scripts/gen-docs.js
3737

38-
# On push to main, keep the committed artifacts (data/, docs/) in sync by
39-
# regenerating and committing them here — so no contributor or maintainer
40-
# ever has to. Skipped on pull_request (validation above is the only gate).
38+
# Cross-field consistency lint (scripts/lint-data.js) — the invariants a
39+
# JSON Schema can't express (environment↔IP weatherproofing, ip_rating
40+
# shape, weight sanity, ISO last_verified …). ERRORS fail the PR; warnings
41+
# are advisory. Complements the schema gate above. See docs/ci.md.
42+
- name: Lint data consistency
43+
run: node scripts/lint-data.js
44+
45+
# On push to main, keep the committed artifacts (data/, docs/, COVERAGE.md)
46+
# in sync by regenerating and committing them here — so no contributor or
47+
# maintainer ever has to. Skipped on pull_request (validation is the gate).
4148
# The [skip ci] tag stops this commit from re-triggering workflows.
4249
- name: Sync generated artifacts (main only)
4350
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
4451
run: |
45-
if [ -z "$(git status --porcelain data/ docs/)" ]; then
52+
# Refresh the field-coverage report (COVERAGE.md), stamped with the
53+
# commit date for a deterministic, diff-friendly output.
54+
COVERAGE_DATE="$(git log -1 --format=%cs)" node scripts/coverage-report.js
55+
if [ -z "$(git status --porcelain data/ docs/ COVERAGE.md)" ]; then
4656
echo "Generated artifacts already up to date — nothing to commit."
4757
exit 0
4858
fi
4959
git config user.name "github-actions[bot]"
5060
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
51-
git add data/ docs/
52-
git commit -m "chore: regenerate data/docs artifacts [skip ci]"
61+
git add data/ docs/ COVERAGE.md
62+
git commit -m "chore: regenerate data/docs/coverage artifacts [skip ci]"
5363
if git push; then
5464
echo "Regenerated artifacts committed to main."
5565
else

‎.github/workflows/check-staleness.yml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
name: check-staleness
2+
3+
# Monthly `last_verified` staleness sweep (CI idea #4) — the freshness companion
4+
# to check-sources.yml (which verifies a source still *resolves*). This flags
5+
# entries not re-verified against their source in a while, plus any with no
6+
# last_verified at all, into a self-updating tracking issue. Advisory only.
7+
on:
8+
schedule:
9+
- cron: "0 8 1 * *" # 1st of each month, 08:00 UTC
10+
workflow_dispatch:
11+
inputs:
12+
months:
13+
description: "Staleness threshold in months (default 18)"
14+
required: false
15+
default: "18"
16+
17+
permissions:
18+
contents: read
19+
issues: write
20+
21+
jobs:
22+
check:
23+
runs-on: ubuntu-latest
24+
timeout-minutes: 10
25+
steps:
26+
- uses: actions/checkout@v4
27+
- uses: actions/setup-node@v4
28+
with:
29+
node-version: "20"
30+
31+
- name: Run staleness check
32+
id: check
33+
run: |
34+
months="${{ github.event.inputs.months || '18' }}"
35+
node scripts/check-staleness.js --months "$months" | tee staleness-report.txt
36+
stale=$(grep -oE 'Found [0-9]+ stale' staleness-report.txt | grep -oE '[0-9]+' | head -1)
37+
unver=$(grep -oE ', [0-9]+ unverified' staleness-report.txt | grep -oE '[0-9]+' | head -1)
38+
echo "stale=${stale:-0}" >> "$GITHUB_OUTPUT"
39+
echo "unver=${unver:-0}" >> "$GITHUB_OUTPUT"
40+
41+
- name: Write job summary
42+
run: |
43+
{
44+
echo "## last_verified staleness"
45+
echo "- Stale (> threshold): **${{ steps.check.outputs.stale }}**"
46+
echo "- Unverified (no date): **${{ steps.check.outputs.unver }}**"
47+
echo ""
48+
echo '```'
49+
sed -n '/--- STALENESS REPORT ---/,$p' staleness-report.txt | head -n 300
50+
echo '```'
51+
} >> "$GITHUB_STEP_SUMMARY"
52+
53+
- name: Upload full report
54+
uses: actions/upload-artifact@v4
55+
with:
56+
name: staleness-report
57+
path: staleness-report.txt
58+
59+
- name: Upsert tracking issue
60+
if: steps.check.outputs.stale != '0' || steps.check.outputs.unver != '0'
61+
env:
62+
GH_TOKEN: ${{ github.token }}
63+
run: |
64+
TITLE="🕰️ last_verified staleness (automated monthly check)"
65+
{
66+
echo "_Automated monthly \`last_verified\` staleness check (\`scripts/check-staleness.js\`). Last run: $(date -u '+%Y-%m-%d %H:%M UTC') — [run log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})._"
67+
echo ""
68+
echo "**${{ steps.check.outputs.stale }}** entries are stale (>threshold) and **${{ steps.check.outputs.unver }}** have no \`last_verified\` date. Re-verify each against its source and bump \`last_verified\`."
69+
echo ""
70+
sed -n '/--- STALENESS REPORT ---/,$p' staleness-report.txt | head -n 300
71+
} > issue-body.md
72+
NUM=$(gh issue list --state open --search "in:title last_verified staleness (automated monthly check)" --json number --jq '.[0].number // empty')
73+
if [ -n "$NUM" ]; then
74+
gh issue edit "$NUM" --body-file issue-body.md
75+
echo "Updated tracking issue #$NUM"
76+
else
77+
gh issue create --title "$TITLE" --label "help wanted" --body-file issue-body.md
78+
fi
79+
80+
- name: Close tracking issue (all fresh)
81+
if: steps.check.outputs.stale == '0' && steps.check.outputs.unver == '0'
82+
env:
83+
GH_TOKEN: ${{ github.token }}
84+
run: |
85+
NUM=$(gh issue list --state open --search "in:title last_verified staleness (automated monthly check)" --json number --jq '.[0].number // empty')
86+
if [ -n "$NUM" ]; then
87+
gh issue comment "$NUM" --body "✅ All entries have a recent last_verified as of run #${{ github.run_id }} — closing."
88+
gh issue close "$NUM"
89+
fi

‎CONTRIBUTING.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,15 @@ git push origin add/reolink-rlc-823a
5959

6060
1. Create `cameras/<brand-slug>/<model-slug>.json`
6161
2. Follow the schema below
62-
3. Run `npm run build` to validate
62+
3. Run `npm run build` to validate (schema) and `npm run lint` to check
63+
cross-field consistency (`npm run lint -- --fix` auto-formats the JSON)
6364
4. Open a pull request
6465

66+
Both the schema build and the consistency lint run on every PR — see
67+
[`docs/ci.md`](docs/ci.md) for the full CI overview (validation, coverage
68+
reporting, Wayback source archiving, and the weekly/monthly link & freshness
69+
sweeps).
70+
6571
> **You only commit source files** — the camera JSON under `cameras/`, and (for
6672
> tooling changes) `schema/` and `docs/glossary.md`. Everything under `cameras/`
6773
> is now pure source (`.json` only). **Do not commit the generated artifacts**:

‎COVERAGE.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# 📊 Field coverage
2+
3+
> Auto-generated by `scripts/coverage-report.js` (do not edit by hand). Last updated: 2026-08-07.
4+
5+
**2,884 cameras** across **74 brands**.
6+
7+
| Field | Coverage | Filled | Lane |
8+
|---|---|---:|:---:|
9+
| 🌍 Environment (in/outdoor) | `██████████` 100% | 2,884 | [#123](https://github.com/ch-bas/cctv-camera-database/issues/123) |
10+
| 🔎 Lens | `█████████░` 94% | 2,698 | |
11+
| 💧 IP rating | `█████████░` 89% | 2,563 | |
12+
| 📡 Field of view | `█████████░` 88% | 2,552 | [#179](https://github.com/ch-bas/cctv-camera-database/issues/179) |
13+
| 🔬 Sensor | `█████████░` 87% | 2,498 | |
14+
| 🔧 Frigate config | `████████░░` 82% | 2,375 | [#170](https://github.com/ch-bas/cctv-camera-database/issues/170) |
15+
| 🎞️ Video streams | `██████░░░░` 60% | 1,738 | [#177](https://github.com/ch-bas/cctv-camera-database/issues/177) |
16+
| 📐 Dimensions | `█████░░░░░` 54% | 1,546 | [#178](https://github.com/ch-bas/cctv-camera-database/issues/178) |
17+
| 🗺️ Markets | `█████░░░░░` 52% | 1,489 | [#166](https://github.com/ch-bas/cctv-camera-database/issues/166) |
18+
| ⚖️ Weight | `█████░░░░░` 47% | 1,347 | [#178](https://github.com/ch-bas/cctv-camera-database/issues/178) |
19+
| 📅 Release year | `████░░░░░░` 38% | 1,088 | [#166](https://github.com/ch-bas/cctv-camera-database/issues/166) |
20+
| 🌙 Min-lux (color) | `████░░░░░░` 36% | 1,052 | [#161](https://github.com/ch-bas/cctv-camera-database/issues/161) |
21+
| 🛡️ IK / impact rating | `████░░░░░░` 36% | 1,028 | [#162](https://github.com/ch-bas/cctv-camera-database/issues/162) |
22+
23+
Help raise a lane — see [open data-collection issues](https://github.com/ch-bas/cctv-camera-database/issues) (index: #163).

‎cameras/bosch/nds-5703-f360.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@
6464
"rtsp",
6565
"http"
6666
],
67-
"ip_rating": "none (indoor)",
6867
"ik_rating": "IK08",
6968
"audio": {
7069
"microphone": true

‎cameras/bosch/nuv-3703-f06.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@
5757
"rtsp",
5858
"http"
5959
],
60-
"ip_rating": "none (indoor)",
6160
"ik_rating": "IK08",
6261
"features": [
6362
"High Dynamic Range (HDR) 120 dB",

‎data/cameras.csv‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ bosch-ndp-5533-z30l,Bosch,AUTODOME IP starlight 5100i IR (NDP-5533-Z30L),ptz,4MP
752752
bosch-ndp-7802-z40,Bosch,AUTODOME 7100i (NDP-7802-Z40),ptz,1080p,2,1/2.8 inch CMOS,1.9-66.35,color,IP66,IK10,true,
753753
bosch-ndp-7802-z40l,Bosch,AUTODOME 7100i (NDP-7802-Z40L),ptz,1080p,2,1/2.8 inch CMOS,1.9-66.35,ir,IP66,IK10,true,2023
754754
bosch-ndp-7804-z12l,Bosch,AUTODOME 7100i IR (NDP-7804-Z12L),ptz,4K UHD,8,"1 inch CMOS, 8 MP (effective 5,544 x 3,694)",6.10–64.60,ir,IP66,IK10,true,2023
755-
bosch-nds-5703-f360,Bosch,FLEXIDOME panoramic 5100i (NDS-5703-F360),panoramic,6MP (4.5MP effective image circle),6,1/1.8-inch CMOS,182 (H) x 182 (V),none,none (indoor),IK08,,2021
755+
bosch-nds-5703-f360,Bosch,FLEXIDOME panoramic 5100i (NDS-5703-F360),panoramic,6MP (4.5MP effective image circle),6,1/1.8-inch CMOS,182 (H) x 182 (V),none,,IK08,,2021
756756
bosch-nds-5703-f360le,Bosch,FLEXIDOME panoramic 5100i 6MP (NDS-5703-F360LE),panoramic,6MP 360deg,6,"1/2.5"" CMOS",360,ir,IP66,IK10,false,
757757
bosch-nds-5704-f360,Bosch,FLEXIDOME panoramic 5100i (NDS-5704-F360),panoramic,12MP,12,1/2.3 inch CMOS,182,none,IP42,IK08,,2021
758758
bosch-nds-5704-f360le,Bosch,FLEXIDOME panoramic 5100i IR (NDS-5704-F360LE),panoramic,12MP,12,1/2.3-inch CMOS,182,ir,IP66,IK10,true,
@@ -791,7 +791,7 @@ bosch-nuv-3702-f06,Bosch,FLEXIDOME micro 3100i indoor (NUV-3702-F06),dome,HD 108
791791
bosch-nuv-3703-f02,Bosch,FLEXIDOME micro 3100i indoor (NUV-3703-F02),dome,5MP,5,1/2.7-inch CMOS,131,none,,IK08,,2023
792792
bosch-nuv-3703-f02h,Bosch,FLEXIDOME micro 3100i indoor (NUV-3703-F02H),dome,5MP,5,1/2.7 inch CMOS,"131.2 (horizontal), 91.4 (vertical)",none,,IK08,,2024
793793
bosch-nuv-3703-f04,Bosch,FLEXIDOME micro 3100i (NUV-3703-F04),dome,5MP,5,"1/2.7"" CMOS",100.8,none,,IK08,,2023
794-
bosch-nuv-3703-f06,Bosch,FLEXIDOME micro 3100i (NUV-3703-F06),dome,5MP,5,CMOS,"56.1 horizontal, 39.2 vertical",none,none (indoor),IK08,,2023
794+
bosch-nuv-3703-f06,Bosch,FLEXIDOME micro 3100i (NUV-3703-F06),dome,5MP,5,CMOS,"56.1 horizontal, 39.2 vertical",none,,IK08,,2023
795795
bosch-smart-home-outdoor-cam-2,Bosch Smart Home,Outdoor Camera II,dome,1080p HD,2,,130 diagonal,color,IP65,,true,2023
796796
camius-boltx-4k,Camius,Boltx 4K,bullet,4K UHD,8,"1/2.8"" CMOS",,ir,IP67,,false,2023
797797
camius-boltx-5mp,Camius,Boltx 5MP,bullet,5MP Super HD,5,"1/2.7"" CMOS",,ir,IP67,,false,2022

‎data/cameras.json‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74408,7 +74408,6 @@
7440874408
"rtsp",
7440974409
"http"
7441074410
],
74411-
"ip_rating": "none (indoor)",
7441274411
"ik_rating": "IK08",
7441374412
"audio": {
7441474413
"microphone": true
@@ -78782,7 +78781,6 @@
7878278781
"rtsp",
7878378782
"http"
7878478783
],
78785-
"ip_rating": "none (indoor)",
7878678784
"ik_rating": "IK08",
7878778785
"features": [
7878878786
"High Dynamic Range (HDR) 120 dB",

‎docs/cameras.json‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74408,7 +74408,6 @@
7440874408
"rtsp",
7440974409
"http"
7441074410
],
74411-
"ip_rating": "none (indoor)",
7441274411
"ik_rating": "IK08",
7441374412
"audio": {
7441474413
"microphone": true
@@ -78782,7 +78781,6 @@
7878278781
"rtsp",
7878378782
"http"
7878478783
],
78785-
"ip_rating": "none (indoor)",
7878678784
"ik_rating": "IK08",
7878778785
"features": [
7878878786
"High Dynamic Range (HDR) 120 dB",

0 commit comments

Comments
 (0)