diff --git a/grype/presenter/jsonl/presenter.go b/grype/presenter/jsonl/presenter.go new file mode 100644 index 00000000000..f350f9d5f20 --- /dev/null +++ b/grype/presenter/jsonl/presenter.go @@ -0,0 +1,47 @@ +// Package jsonl provides a presenter that emits one JSON object per line +// (the JSON Lines / ndjson convention). Each line is a single match record, +// suitable for streaming to tools like jq, xargs, or downstream processors +// that accept newline-delimited JSON on stdin. +// +// See https://github.com/anchore/grype/issues/1159 for the motivating use case. +package jsonl + +import ( + "encoding/json" + "io" + + "github.com/anchore/grype/grype/presenter/models" +) + +// Presenter writes the matches contained in the document one per line as +// JSON objects. Document-level metadata (descriptor, source, distro, +// ignoredMatches, alertsByPackage) is intentionally omitted — JSON Lines is +// a flat record stream by design. Consumers that need that metadata should +// use the standard `json` format. +type Presenter struct { + document models.Document +} + +// NewPresenter returns a new JSON Lines presenter. +func NewPresenter(pb models.PresenterConfig) *Presenter { + return &Presenter{ + document: pb.Document, + } +} + +// Present writes one JSON-encoded match per line, terminated by a newline. +// When there are no matches, no output is written — this is the standard +// jsonl convention (an empty file is a valid empty stream). +func (p *Presenter) Present(output io.Writer) error { + enc := json.NewEncoder(output) + // match the json presenter's behavior so values aren't HTML-escaped + enc.SetEscapeHTML(false) + // json.Encoder.Encode appends a newline after every value, so the output + // is naturally newline-delimited. + for _, m := range p.document.Matches { + if err := enc.Encode(m); err != nil { + return err + } + } + return nil +} diff --git a/grype/presenter/jsonl/presenter_test.go b/grype/presenter/jsonl/presenter_test.go new file mode 100644 index 00000000000..c2f6c63309b --- /dev/null +++ b/grype/presenter/jsonl/presenter_test.go @@ -0,0 +1,117 @@ +package jsonl + +import ( + "bytes" + "encoding/json" + "flag" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/clio" + "github.com/anchore/grype/grype/distro" + "github.com/anchore/grype/grype/match" + "github.com/anchore/grype/grype/pkg" + "github.com/anchore/grype/grype/presenter/internal" + "github.com/anchore/grype/grype/presenter/models" + "github.com/anchore/grype/internal/testutils" + "github.com/anchore/syft/syft/source" +) + +var update = flag.Bool("update", false, "update the *.golden files for jsonl presenter") + +func TestJSONLImgsPresenter(t *testing.T) { + var buffer bytes.Buffer + + pb := internal.GeneratePresenterConfig(t, internal.ImageSource) + pres := NewPresenter(pb) + + require.NoError(t, pres.Present(&buffer)) + actual := buffer.Bytes() + + if *update { + testutils.UpdateGoldenFileContents(t, actual) + } + expected := testutils.GetGoldenFileContents(t) + + if d := cmp.Diff(string(expected), string(actual)); d != "" { + t.Fatalf("diff: %s", d) + } + + // every emitted line must independently parse as JSON + for i, line := range nonEmptyLines(actual) { + var v map[string]any + require.NoErrorf(t, json.Unmarshal(line, &v), "line %d is not valid JSON: %q", i, string(line)) + } +} + +func TestJSONLDirsPresenter(t *testing.T) { + var buffer bytes.Buffer + + pb := internal.GeneratePresenterConfig(t, internal.DirectorySource) + pres := NewPresenter(pb) + + require.NoError(t, pres.Present(&buffer)) + actual := buffer.Bytes() + + if *update { + testutils.UpdateGoldenFileContents(t, actual) + } + expected := testutils.GetGoldenFileContents(t) + + if d := cmp.Diff(string(expected), string(actual)); d != "" { + t.Fatalf("diff: %s", d) + } +} + +func TestEmptyJSONLPresenter(t *testing.T) { + // no matches → no output (empty stream is a valid jsonl document). + var buffer bytes.Buffer + + ctx := pkg.Context{ + Source: &source.Description{}, + Distro: &distro.Distro{ + Type: "centos", + IDLike: []string{"rhel"}, + Version: "8.0", + }, + } + + doc, err := models.NewDocument(clio.Identification{Name: "grype", Version: "[not provided]"}, nil, ctx, match.NewMatches(), nil, models.NewMetadataMock(), nil, nil, models.SortByPackage, true, nil) + require.NoError(t, err) + + pb := models.PresenterConfig{ + ID: clio.Identification{Name: "grype", Version: "[not provided]"}, + Document: doc, + } + + pres := NewPresenter(pb) + require.NoError(t, pres.Present(&buffer)) + + assert.Equal(t, 0, buffer.Len(), "empty match set should produce no output") +} + +func TestJSONLLineCountMatchesDocument(t *testing.T) { + var buffer bytes.Buffer + + pb := internal.GeneratePresenterConfig(t, internal.ImageSource) + pres := NewPresenter(pb) + require.NoError(t, pres.Present(&buffer)) + + got := len(nonEmptyLines(buffer.Bytes())) + want := len(pb.Document.Matches) + assert.Equal(t, want, got, "expected one jsonl line per match in the document") +} + +func nonEmptyLines(b []byte) [][]byte { + var out [][]byte + for _, line := range bytes.Split(b, []byte("\n")) { + if len(strings.TrimSpace(string(line))) > 0 { + out = append(out, line) + } + } + return out +} diff --git a/grype/presenter/jsonl/testdata/snapshot/TestJSONLDirsPresenter.golden b/grype/presenter/jsonl/testdata/snapshot/TestJSONLDirsPresenter.golden new file mode 100644 index 00000000000..c9e66bd2e6d --- /dev/null +++ b/grype/presenter/jsonl/testdata/snapshot/TestJSONLDirsPresenter.golden @@ -0,0 +1,2 @@ +{"vulnerability":{"id":"CVE-1999-0001","dataSource":"","severity":"Low","urls":[],"cvss":[{"source":"nvd","type":"CVSS","version":"3.1","vector":"CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:C/C:L/I:L/A:H","metrics":{"baseScore":8.2},"vendorMetadata":{}}],"epss":[{"cve":"CVE-1999-0001","epss":0.03,"percentile":0.42,"date":"0001-01-01"}],"fix":{"versions":["1.2.1","2.1.3","3.4.0"],"state":"fixed"},"advisories":[],"risk":1.68},"relatedVulnerabilities":[],"matchDetails":[{"type":"exact-direct-match","matcher":"dpkg-matcher","searchedBy":{"distro":{"type":"ubuntu","version":"20.04"}},"found":{"constraint":">= 20"},"fix":{"suggestedVersion":"1.2.1"}}],"artifact":{"id":"bbb0ba712c2b94ea","name":"package-1","version":"1.1.1","type":"rpm","locations":[{"path":"/foo/bar/somefile-1.txt","accessPath":"somefile-1.txt"}],"language":"","licenses":[],"cpes":["cpe:2.3:a:anchore\\:oss:anchore\\/engine:0.9.2:*:*:en:*:*:*:*"],"purl":"","upstreams":[],"metadataType":"RpmMetadata","metadata":{"epoch":2,"modularityLabel":null}}} +{"vulnerability":{"id":"CVE-1999-0002","dataSource":"","severity":"Critical","urls":[],"cvss":[{"source":"nvd","type":"CVSS","version":"3.1","vector":"CVSS:3.1/AV:N/AC:H/PR:L/UI:N/S:C/C:H/I:H/A:H","metrics":{"baseScore":8.5},"vendorMetadata":{}}],"knownExploited":[{"cve":"CVE-1999-0002","knownRansomwareCampaignUse":"Known"}],"epss":[{"cve":"CVE-1999-0002","epss":0.08,"percentile":0.53,"date":"0001-01-01"}],"fix":{"versions":[],"state":""},"advisories":[],"risk":96.25000000000001},"relatedVulnerabilities":[],"matchDetails":[{"type":"exact-indirect-match","matcher":"dpkg-matcher","searchedBy":{"cpe":"somecpe"},"found":{"constraint":"somecpe"}}],"artifact":{"id":"74378afe15713625","name":"package-2","version":"2.2.2","type":"deb","locations":[{"path":"/foo/bar/somefile-2.txt","accessPath":"somefile-2.txt"}],"language":"","licenses":["Apache-2.0","MIT"],"cpes":["cpe:2.3:a:anchore:engine:2.2.2:*:*:en:*:*:*:*"],"purl":"pkg:deb/package-2@2.2.2","upstreams":[]}} diff --git a/grype/presenter/jsonl/testdata/snapshot/TestJSONLImgsPresenter.golden b/grype/presenter/jsonl/testdata/snapshot/TestJSONLImgsPresenter.golden new file mode 100644 index 00000000000..c9e66bd2e6d --- /dev/null +++ b/grype/presenter/jsonl/testdata/snapshot/TestJSONLImgsPresenter.golden @@ -0,0 +1,2 @@ +{"vulnerability":{"id":"CVE-1999-0001","dataSource":"","severity":"Low","urls":[],"cvss":[{"source":"nvd","type":"CVSS","version":"3.1","vector":"CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:C/C:L/I:L/A:H","metrics":{"baseScore":8.2},"vendorMetadata":{}}],"epss":[{"cve":"CVE-1999-0001","epss":0.03,"percentile":0.42,"date":"0001-01-01"}],"fix":{"versions":["1.2.1","2.1.3","3.4.0"],"state":"fixed"},"advisories":[],"risk":1.68},"relatedVulnerabilities":[],"matchDetails":[{"type":"exact-direct-match","matcher":"dpkg-matcher","searchedBy":{"distro":{"type":"ubuntu","version":"20.04"}},"found":{"constraint":">= 20"},"fix":{"suggestedVersion":"1.2.1"}}],"artifact":{"id":"bbb0ba712c2b94ea","name":"package-1","version":"1.1.1","type":"rpm","locations":[{"path":"/foo/bar/somefile-1.txt","accessPath":"somefile-1.txt"}],"language":"","licenses":[],"cpes":["cpe:2.3:a:anchore\\:oss:anchore\\/engine:0.9.2:*:*:en:*:*:*:*"],"purl":"","upstreams":[],"metadataType":"RpmMetadata","metadata":{"epoch":2,"modularityLabel":null}}} +{"vulnerability":{"id":"CVE-1999-0002","dataSource":"","severity":"Critical","urls":[],"cvss":[{"source":"nvd","type":"CVSS","version":"3.1","vector":"CVSS:3.1/AV:N/AC:H/PR:L/UI:N/S:C/C:H/I:H/A:H","metrics":{"baseScore":8.5},"vendorMetadata":{}}],"knownExploited":[{"cve":"CVE-1999-0002","knownRansomwareCampaignUse":"Known"}],"epss":[{"cve":"CVE-1999-0002","epss":0.08,"percentile":0.53,"date":"0001-01-01"}],"fix":{"versions":[],"state":""},"advisories":[],"risk":96.25000000000001},"relatedVulnerabilities":[],"matchDetails":[{"type":"exact-indirect-match","matcher":"dpkg-matcher","searchedBy":{"cpe":"somecpe"},"found":{"constraint":"somecpe"}}],"artifact":{"id":"74378afe15713625","name":"package-2","version":"2.2.2","type":"deb","locations":[{"path":"/foo/bar/somefile-2.txt","accessPath":"somefile-2.txt"}],"language":"","licenses":["Apache-2.0","MIT"],"cpes":["cpe:2.3:a:anchore:engine:2.2.2:*:*:en:*:*:*:*"],"purl":"pkg:deb/package-2@2.2.2","upstreams":[]}} diff --git a/internal/format/format.go b/internal/format/format.go index f6c099b346b..ec6230f5b9d 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -7,6 +7,7 @@ import ( const ( UnknownFormat Format = "unknown" JSONFormat Format = "json" + JSONLinesFormat Format = "jsonl" TableFormat Format = "table" CycloneDXFormat Format = "cyclonedx" CycloneDXJSON Format = "cyclonedx-json" @@ -33,6 +34,8 @@ func Parse(userInput string) Format { return TableFormat case strings.ToLower(JSONFormat.String()): return JSONFormat + case strings.ToLower(JSONLinesFormat.String()), "ndjson": + return JSONLinesFormat case strings.ToLower(TableFormat.String()): return TableFormat case strings.ToLower(SarifFormat.String()): @@ -57,6 +60,7 @@ func Parse(userInput string) Format { // AvailableFormats is a list of presenter format options available to users. var AvailableFormats = []Format{ JSONFormat, + JSONLinesFormat, TableFormat, CycloneDXFormat, CycloneDXJSON, diff --git a/internal/format/format_test.go b/internal/format/format_test.go index 665b442b749..0223eb9197d 100644 --- a/internal/format/format_test.go +++ b/internal/format/format_test.go @@ -23,6 +23,19 @@ func TestParse(t *testing.T) { "jSOn", JSONFormat, }, + { + "jsonl", + JSONLinesFormat, + }, + { + "JSONL", + JSONLinesFormat, + }, + { + // ndjson is a common alias for JSON lines and is accepted as an alternate spelling. + "ndjson", + JSONLinesFormat, + }, { "booboodepoopoo", UnknownFormat, diff --git a/internal/format/presenter.go b/internal/format/presenter.go index 71892d005b3..38850e9839e 100644 --- a/internal/format/presenter.go +++ b/internal/format/presenter.go @@ -5,6 +5,7 @@ import ( "github.com/anchore/grype/grype/presenter/cyclonedx" "github.com/anchore/grype/grype/presenter/json" + "github.com/anchore/grype/grype/presenter/jsonl" "github.com/anchore/grype/grype/presenter/models" "github.com/anchore/grype/grype/presenter/sarif" "github.com/anchore/grype/grype/presenter/table" @@ -23,6 +24,8 @@ func GetPresenter(format Format, c PresentationConfig, pb models.PresenterConfig switch format { case JSONFormat: return json.NewPresenter(pb) + case JSONLinesFormat: + return jsonl.NewPresenter(pb) case TableFormat: return table.NewPresenter(pb, c.ShowSuppressed)