Skip to content

Commit 5120a90

Browse files
mfahlandtmfahlandt
authored
feat: CNCF license policy as default, exception handling improvements (#10)
* Add CI Jobs for labeler and releases / pre releases * feat: CNCF license policy as default, exception handling improvements, deployment examples - Set CNCF Allowed Third-Party License Policy as default (18 permissive, 21 copyleft) - Promote "All CNCF Projects" exceptions to blanket exceptions automatically - Split compound license expressions (OR, AND, comma-separated) into individual SPDX IDs - Add substring matching for CNCF-style short package names in exception lookup - Parsing worker: fallback to PVC path for license exceptions loading - Add examples/ directory with Kind and Kubernetes deployment configs - Production example uses seed job instead of git-sync (git-sync fails for large repos) - Document all 3 SBOM ingestion methods (seed job, git-sync, manual PVC) - Add Makefile targets: kind-build, kind-deploy, kind-reingest - Add License Policy section to README with customisation instructions - Bump version to 0.1.2 across Helm chart, values, examples, and docs - Add 8 new tests for CNCF exception handling (87+ total) * feat: CNCF license policy as default, exception handling improvements, deployment examples - Set CNCF Allowed Third-Party License Policy as default (18 permissive, 21 copyleft) - Promote "All CNCF Projects" exceptions to blanket exceptions automatically - Split compound license expressions (OR, AND, comma-separated) into individual SPDX IDs - Add substring matching for CNCF-style short package names in exception lookup - Parsing worker: fallback to PVC path for license exceptions loading - Add examples/ directory with Kind and Kubernetes deployment configs - Production example uses seed job instead of git-sync (git-sync fails for large repos) - Document all 3 SBOM ingestion methods (seed job, git-sync, manual PVC) - Add Makefile targets: kind-build, kind-deploy, kind-reingest - Add License Policy section to README with customisation instructions - Bump version to 0.1.2 across Helm chart, values, examples, and docs - Add 8 new tests for CNCF exception handling (87+ total)
1 parent 48d52d0 commit 5120a90

50 files changed

Lines changed: 1860 additions & 90 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/labeler.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: Auto-Label PRs
22

33
on:
4-
pull_request:
4+
pull_request_target:
55
types: [opened, synchronize, reopened]
66

77
permissions:

‎.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,12 @@
88
*.so
99
*.dylib
1010

11+
# Compiled Go binaries (built by `go build`)
12+
backend/api-gateway
13+
backend/parsing-worker
14+
backend/ingestion-watcher
15+
backend/cve-refresher
16+
1117
# Test binary, built with `go test -c`
1218
*.test
1319

@@ -27,6 +33,9 @@ go.work.sum
2733
# env file
2834
.env
2935

36+
# Local kind deployment (secrets, overrides)
37+
local/
38+
3039
# Editor/IDE
3140
# .idea/
3241
# .vscode/

‎Makefile‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
.PHONY: ingest worker api
66
.PHONY: images images-push
77
.PHONY: sync-labels
8+
.PHONY: kind-up kind-down kind-reingest kind-build kind-deploy
89

910
SHELL := /bin/bash
1011
REGISTRY ?= ghcr.io
@@ -160,3 +161,41 @@ images-push: images ## Build and push all images to GHCR (TAG=dev)
160161
# ─── GitHub ──────────────────────────────────────────────────────────────────
161162
sync-labels: ## Sync GitHub labels from .github/labels.yml (requires gh + yq)
162163
.github/scripts/sync-labels.sh
164+
165+
# ─── Kind (local Kubernetes) ─────────────────────────────────────────────────
166+
kind-up: ## Deploy SeeBOM to a local Kind cluster (see local/secrets.env)
167+
./local/setup.sh
168+
169+
kind-down: ## Destroy the local Kind cluster
170+
./local/teardown.sh
171+
172+
kind-reingest: ## Re-ingest all SBOMs in Kind (no re-download, truncates data + re-queues)
173+
@echo "🗑️ Truncating data tables..."
174+
@source local/secrets.env 2>/dev/null; \
175+
kubectl exec -n seebom chi-seebom-clickhouse-seebom-cluster-0-0-0 -c clickhouse -- \
176+
clickhouse-client --database=seebom --password="$${CLICKHOUSE_PASSWORD:-seebom}" --multiquery \
177+
--query "TRUNCATE TABLE ingestion_queue; TRUNCATE TABLE license_compliance; TRUNCATE TABLE vulnerabilities; TRUNCATE TABLE sbom_packages; TRUNCATE TABLE sboms; TRUNCATE TABLE vex_statements;"
178+
@echo "♻️ Triggering ingestion watcher..."
179+
@kubectl create job --from=cronjob/seebom-ingestion-watcher seebom-reingest-$$(date +%s) -n seebom
180+
@echo "✅ Re-ingest started. Workers will re-process all SBOMs from the PVC."
181+
@echo " Monitor: curl -s http://localhost:8080/api/v1/stats/dashboard | jq .total_sboms"
182+
183+
kind-build: images ## Build dev images and load them into the Kind cluster
184+
@echo "📦 Loading images into Kind cluster..."
185+
kind load docker-image $(REGISTRY)/$(REPO)/ingestion-watcher:$(TAG) --name seebom
186+
kind load docker-image $(REGISTRY)/$(REPO)/parsing-worker:$(TAG) --name seebom
187+
kind load docker-image $(REGISTRY)/$(REPO)/api-gateway:$(TAG) --name seebom
188+
kind load docker-image $(REGISTRY)/$(REPO)/cve-refresher:$(TAG) --name seebom
189+
kind load docker-image $(REGISTRY)/$(REPO)/ui:$(TAG) --name seebom
190+
@echo "✅ Loaded 5 images into Kind (tag: $(TAG))"
191+
192+
kind-deploy: kind-build ## Build images, load into Kind, and upgrade Helm release
193+
@source local/secrets.env 2>/dev/null; \
194+
helm upgrade seebom deploy/helm/seebom/ \
195+
-n seebom \
196+
-f local/values-local.yaml \
197+
--set clickhouse.password="$${CLICKHOUSE_PASSWORD:-seebom}" \
198+
--set github.token="$${GITHUB_TOKEN:-}"
199+
@kubectl rollout restart deployment/seebom-api-gateway deployment/seebom-parsing-worker -n seebom
200+
@echo "✅ Deployed. Pods restarting with new images."
201+

‎README.md‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,24 @@ docker compose up -d --force-recreate api-gateway parsing-worker
130130

131131
See [docs/DEPLOYMENT_GUIDE.md](docs/DEPLOYMENT_GUIDE.md) for Kubernetes deployment instructions.
132132

133-
### Option B: Local Development (Hot Reload)
133+
### Option B: Local Kubernetes (Kind)
134+
135+
Deploy the full stack to a local [Kind](https://kind.sigs.k8s.io/) cluster, including ClickHouse Operator, CNCF SBOM ingestion, and the Angular UI:
136+
137+
```bash
138+
# 1. Copy secrets template and fill in your values
139+
cp examples/kind/secrets.env.example local/secrets.env
140+
vi local/secrets.env
141+
142+
# 2. Deploy
143+
make kind-up
144+
145+
# UI: http://localhost:8090 API: http://localhost:8080/healthz
146+
```
147+
148+
See [`examples/`](examples/) for Kind and production Kubernetes deployment configs.
149+
150+
### Option C: Local Development (Hot Reload)
134151

135152
Use this when you want to iterate on code quickly:
136153

@@ -218,18 +235,28 @@ See [docs/TESTING.md](docs/TESTING.md) for writing and running tests.
218235

219236
| Command | Description |
220237
|---------|-------------|
238+
| **Docker Compose** | |
221239
| `make dev` | Start full stack via Docker Compose |
222240
| `make dev-down` | Stop all containers |
223241
| `make dev-restart` | Restart with new `.env` values (keeps data) |
224242
| `make dev-logs` | Follow all container logs |
225243
| `make dev-reset` | Destroy data volumes and restart fresh |
244+
| `make dev-status` | Show container status and ingestion progress |
226245
| `make re-ingest` | Re-trigger the Ingestion Watcher (scans for new files) |
227246
| `make re-scan` | Wipe all data and re-process everything (e.g. after enabling OSV) |
228247
| `make cve-refresh` | Check all known PURLs for new CVEs (without re-scanning SBOMs) |
229248
| `make migrate` | Run all pending database migrations |
249+
| **Kind (Local Kubernetes)** | |
250+
| `make kind-up` | Create Kind cluster and deploy SeeBOM via Helm |
251+
| `make kind-down` | Destroy the Kind cluster |
252+
| `make kind-build` | Build all container images and load them into Kind |
253+
| `make kind-deploy` | Build images, Helm upgrade, and restart pods |
254+
| `make kind-reingest` | Re-ingest all SBOMs (truncate data, re-queue, no re-download) |
255+
| **ClickHouse** | |
230256
| `make ch-only` | Start only ClickHouse (for local dev) |
231257
| `make ch-migrate` | Run SQL migrations against ClickHouse |
232258
| `make ch-shell` | Open ClickHouse CLI |
259+
| **Local Dev** | |
233260
| `make api` | Run API Gateway locally |
234261
| `make ingest` | Run Ingestion Watcher locally |
235262
| `make worker` | Run Parsing Worker locally |
@@ -238,6 +265,9 @@ See [docs/TESTING.md](docs/TESTING.md) for writing and running tests.
238265
| `make backend-test` | Run all Go tests |
239266
| `make backend-vet` | Run go vet + go fmt |
240267
| `make ui-build` | Build Angular for production |
268+
| **Images** | |
269+
| `make images` | Build all 5 container images locally (TAG=dev) |
270+
| `make images-push` | Build and push all images to GHCR |
241271

242272
---
243273

@@ -288,6 +318,41 @@ make dev-reset
288318

289319
---
290320

321+
## License Policy
322+
323+
By default, SeeBOM enforces the [CNCF Allowed Third-Party License Policy](https://github.com/cncf/foundation/blob/main/policies-guidance/allowed-third-party-license-policy.md):
324+
325+
- **Permissive (allowed):** Apache-2.0, MIT, MIT-0, 0BSD, BSD-2-Clause, BSD-3-Clause, ISC, PSF-2.0, Python-2.0, PostgreSQL, UPL-1.0, X11, Zlib, OpenSSL, and a few more (18 total)
326+
- **Copyleft (flagged):** GPL, LGPL, AGPL, MPL-2.0, EPL, EUPL, CPAL, and others (21 total)
327+
- **Unknown:** Any license not in either list is flagged for review
328+
329+
### CNCF Exception List
330+
331+
The [CNCF license exceptions](https://github.com/cncf/foundation/blob/main/license-exceptions/exceptions.json) are automatically downloaded and applied. Packages covered by a CNCF Governing Board exception are marked as exempted rather than non-compliant.
332+
333+
Exceptions with `"project": "All CNCF Projects"` are treated as blanket exceptions (apply to every SBOM).
334+
335+
### Customising the Policy
336+
337+
Override the default policy via Helm values:
338+
339+
```yaml
340+
licensePolicy:
341+
custom: |
342+
{
343+
"permissive": ["Apache-2.0", "MIT"],
344+
"copyleft": ["GPL-3.0-only", "AGPL-3.0-only"]
345+
}
346+
```
347+
348+
Or edit the ConfigMap directly:
349+
350+
```bash
351+
kubectl edit configmap seebom-license-policy -n seebom
352+
```
353+
354+
---
355+
291356
## Tech Stack
292357

293358
| Layer | Technology |

‎backend/cmd/api-gateway/main.go‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ func main() {
2626
defer chClient.Close()
2727

2828
exceptionsPath := cfg.ExceptionsFile
29+
sbomDirExceptionsPath := cfg.SBOMDir + "/license-exceptions.json"
2930

3031
// Load license policy (permissive/copyleft classification).
3132
if perm, copy, err := license.LoadPolicy(cfg.LicensePolicyFile); err == nil {
@@ -160,11 +161,8 @@ func main() {
160161

161162
// Projects with license violations (filtered by exceptions).
162163
mux.HandleFunc("GET /api/v1/projects/license-violations", func(w http.ResponseWriter, r *http.Request) {
163-
// Load current exceptions for filtering.
164-
var excIdx *license.ExceptionIndex
165-
if idx, err := license.LoadExceptions(exceptionsPath); err == nil {
166-
excIdx = idx
167-
}
164+
// Load current exceptions for filtering (try config path, then SBOM dir).
165+
excIdx, _ := license.LoadExceptionsWithFallback(exceptionsPath, sbomDirExceptionsPath)
168166
violations, err := chClient.QueryProjectsWithLicenseViolations(r.Context(), excIdx)
169167
if err != nil {
170168
log.Printf("ERROR: license violations: %v", err)
@@ -216,10 +214,10 @@ func main() {
216214
writeJSON(w, http.StatusOK, resp)
217215
})
218216

219-
// ── License Exceptions (read-only from config file) ────────────────
217+
// ── License Exceptions (read-only from config file or SBOM dir) ────
220218
mux.HandleFunc("GET /api/v1/license-exceptions", func(w http.ResponseWriter, r *http.Request) {
221-
idx, err := license.LoadExceptions(exceptionsPath)
222-
if err != nil {
219+
idx, err := license.LoadExceptionsWithFallback(exceptionsPath, sbomDirExceptionsPath)
220+
if err != nil || idx == nil {
223221
writeJSON(w, http.StatusOK, license.ExceptionsFile{
224222
Version: "1.0.0",
225223
BlanketExceptions: []license.BlanketException{},

‎backend/cmd/parsing-worker/main.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,15 @@ func main() {
4545
log.Printf("Using default license policy (tried %s): %v", cfg.LicensePolicyFile, err)
4646
}
4747

48-
// Load license exceptions if available.
48+
// Load license exceptions if available (try config path, then SBOM dir fallback).
4949
var exceptionsIndex *license.ExceptionIndex
50-
if idx, err := license.LoadExceptions(cfg.ExceptionsFile); err == nil {
50+
sbomDirExceptionsPath := cfg.SBOMDir + "/license-exceptions.json"
51+
if idx, err := license.LoadExceptionsWithFallback(cfg.ExceptionsFile, sbomDirExceptionsPath); err == nil {
5152
exceptionsIndex = idx
52-
log.Printf("Loaded license exceptions from %s (%d blanket, %d specific)",
53-
cfg.ExceptionsFile, len(idx.Raw.BlanketExceptions), len(idx.Raw.Exceptions))
53+
log.Printf("Loaded license exceptions (%d blanket, %d specific)",
54+
len(idx.Raw.BlanketExceptions), len(idx.Raw.Exceptions))
5455
} else {
55-
log.Printf("No license exceptions loaded (tried %s): %v", cfg.ExceptionsFile, err)
56+
log.Printf("No license exceptions loaded (tried %s, %s): %v", cfg.ExceptionsFile, sbomDirExceptionsPath, err)
5657
}
5758

5859
// Initialize GitHub license resolver for unknown licenses.

‎backend/internal/clickhouse/queries_github_cache.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,11 @@ func (c *Client) QueryArchivedPackages(ctx context.Context) ([]ArchivedPackageIn
114114
FROM sbom_packages p FINAL
115115
ARRAY JOIN p.package_names AS pkg_name, p.package_purls AS pkg_purl
116116
JOIN sboms s FINAL ON p.sbom_id = s.sbom_id
117-
JOIN github_repo_metadata m FINAL ON m.archived = true
117+
CROSS JOIN (
118+
SELECT repo, pushed_at, stargazers
119+
FROM github_repo_metadata FINAL
120+
WHERE archived = true
121+
) m
118122
WHERE lower(pkg_purl) LIKE concat('%', m.repo, '%')
119123
ORDER BY project_name, s.source_file
120124
`)

‎backend/internal/license/checker.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,19 @@ type Policy struct {
3131
copyleft map[string]bool
3232
}
3333

34-
// built-in defaults, used when no policy file is provided.
34+
// built-in defaults based on the CNCF Allowed Third-Party License Policy:
35+
// https://github.com/cncf/foundation/blob/main/policies-guidance/allowed-third-party-license-policy.md
36+
// Apache-2.0 (CNCF project license) + the CNCF Allowlist are permissive.
37+
// Everything else requires a CNCF Governing Board exception.
3538
var defaultPermissive = []string{
36-
"MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC",
37-
"Unlicense", "0BSD", "CC0-1.0", "Zlib", "BSL-1.0", "PSF-2.0",
39+
"Apache-2.0",
40+
"MIT", "MIT-0",
41+
"0BSD", "BSD-2-Clause", "BSD-2-Clause-FreeBSD", "BSD-3-Clause",
42+
"ISC",
43+
"PSF-2.0", "Python-2.0", "Python-2.0.1",
44+
"PostgreSQL",
45+
"UPL-1.0", "X11", "Zlib",
46+
"OpenSSL", "OpenSSL-standalone", "SSLeay-standalone",
3847
}
3948

4049
var defaultCopyleft = []string{

0 commit comments

Comments
 (0)