feat(suse): add cvrf CVE feed

.github/workflows/update.yml

@@ -71,6 +71,10 @@ jobs:
         name: SUSE CVRF
         run: ./scripts/update.sh suse-cvrf "SUSE CVRF"
 
+      - if: always()
+        name: SUSE CVE CVRF
+        run: ./scripts/update.sh suse-cvrf-cve "SUSE CVE CVRF"
+
       - if: always()
         name: GitLab Advisory Database
         run: ./scripts/update.sh glad "GitLab Advisory Database"

README.md

@@ -20,7 +20,7 @@ https://github.com/aquasecurity/vuln-list/
 $ vuln-list-update -h
 Usage of vuln-list-update:
   -target string
-        update target (nvd, alpine, alpine-unfixed, redhat, redhat-oval, debian, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, ghsa, glad, cwe, osv, mariner, kevc, wolfi, chainguard, azure, openeuler, echo, minimos, rootio, seal)
+        update target (nvd, alpine, alpine-unfixed, redhat, redhat-oval, debian, ubuntu, amazon, oracle-oval, suse-cvrf, suse-cvrf-cve, photon, arch-linux, ghsa, glad, cwe, osv, mariner, kevc, wolfi, chainguard, azure, openeuler, echo, minimos, rootio, seal)
   -target-branch string
     	alternative repository branch (only glad)
   -target-uri string

main.go

@@ -32,14 +32,15 @@ import (
 	"github.com/aquasecurity/vuln-list-update/rootio"
 	"github.com/aquasecurity/vuln-list-update/seal"
 	susecvrf "github.com/aquasecurity/vuln-list-update/suse/cvrf"
+	susecvrfcve "github.com/aquasecurity/vuln-list-update/suse/cvrfcve"
 	"github.com/aquasecurity/vuln-list-update/ubuntu"
 	"github.com/aquasecurity/vuln-list-update/utils"
 	"github.com/aquasecurity/vuln-list-update/wolfi"
 )
 
 var (
 	target = flag.String("target", "", "update target (nvd, alpine, alpine-unfixed, redhat, redhat-oval, "+
-		"redhat-csaf-vex, debian, ubuntu, amazon, oracle-oval, suse-cvrf, photon, arch-linux, glad, cwe, osvdev, mariner, kevc, wolfi, "+
+		"redhat-csaf-vex, debian, ubuntu, amazon, oracle-oval, suse-cvrf, suse-cvrf-cve, photon, arch-linux, glad, cwe, osvdev, mariner, kevc, wolfi, "+
 		"chainguard, azure, openeuler, echo, minimos, eoldates, rootio)")
 	vulnListDir  = flag.String("vuln-list-dir", "", "vuln-list dir")
 	targetUri    = flag.String("target-uri", "", "alternative repository URI (only glad)")
@@ -113,6 +114,11 @@ func run() error {
 		if err := sc.Update(); err != nil {
 			return xerrors.Errorf("SUSE CVRF update error: %w", err)
 		}
+	case "suse-cvrf-cve":
+		sc := susecvrfcve.NewConfig()
+		if err := sc.Update(); err != nil {
+			return xerrors.Errorf("SUSE CVE CVRF update error: %w", err)
+		}
 	case "photon":
 		pc := photon.NewConfig()
 		if err := pc.Update(); err != nil {

suse/cvrf/cvrf.go

@@ -1,23 +1,17 @@
 package cvrf
 
 import (
-	"archive/tar"
-	"bytes"
-	"compress/bzip2"
-	"compress/gzip"
 	"encoding/xml"
-	"errors"
 	"fmt"
-	"io"
 	"log"
 	"path/filepath"
 	"regexp"
 	"strings"
-	"unicode/utf8"
 
 	"github.com/spf13/afero"
 	"golang.org/x/xerrors"
 
+	"github.com/aquasecurity/vuln-list-update/suse/cvrfarchive"
 	"github.com/aquasecurity/vuln-list-update/utils"
 )
 
@@ -47,81 +41,21 @@ func NewConfig() Config {
 func (c Config) Update() error {
 	log.Print("Fetching SUSE CVRF archive...")
 
-	// The SUSE server is sometimes unstable, so download the whole archive into
-	// memory before processing. Streaming directly from the HTTP response would
-	// make it hard to distinguish a mid-transfer disconnection (which surfaces
-	// as a truncated tar) from a legitimate parse error. The archive is only a
-	// few hundred MB, which fits comfortably in memory on CI runners.
-	body, err := utils.FetchURL(c.URL, "", retries)
-	if err != nil {
-		return xerrors.Errorf("failed to download CVRF archive: %w", err)
-	}
-
-	var decompressed io.Reader
-	switch {
-	case strings.HasSuffix(c.URL, ".tar.bz2"):
-		// The upstream archive is .tar.bz2, which is the only format used in production.
-		decompressed = bzip2.NewReader(bytes.NewReader(body))
-	case strings.HasSuffix(c.URL, ".tar.gz"):
-		// Go's compress/bzip2 lacks a Writer, so tests use .tar.gz instead.
-		gr, err := gzip.NewReader(bytes.NewReader(body))
-		if err != nil {
-			return xerrors.Errorf("failed to decompress gzip: %w", err)
-		}
-		defer gr.Close()
-		decompressed = gr
-	default:
-		return xerrors.Errorf("unsupported archive format: %s", c.URL)
-	}
-	tr := tar.NewReader(decompressed)
-
-	for {
-		hdr, err := tr.Next()
-		switch {
-		case errors.Is(err, io.EOF):
-			return nil
-		case err != nil:
-			return xerrors.Errorf("failed to read tar entry: %w", err)
-		case hdr.Typeflag != tar.TypeReg:
-			continue
-		}
-
-		filename := filepath.Base(hdr.Name)
-		// archive contains non-XML files (e.g. LICENSE), so skip them
-		if !strings.HasSuffix(filename, ".xml") {
-			continue
-		}
-		match := fileRegexp.FindStringSubmatch(filename)
-		if match == nil {
-			continue
-		}
+	return cvrfarchive.Walk(c.URL, retries, fileRegexp, func(e cvrfarchive.Entry) error {
+		match := fileRegexp.FindStringSubmatch(e.Filename)
 		osName := match[1]
 
-		data, err := io.ReadAll(tr)
-		if err != nil {
-			return xerrors.Errorf("failed to read tar entry data: %w", err)
-		}
-
-		if len(data) == 0 {
-			log.Printf("empty CVRF xml: %s", filename)
-			continue
-		}
-
-		if !utf8.Valid(data) {
-			log.Printf("invalid UTF-8: %s", filename)
-			data = []byte(strings.ToValidUTF8(string(data), ""))
-		}
-
 		var cv Cvrf
-		if err = xml.Unmarshal(data, &cv); err != nil {
-			return xerrors.Errorf("failed to decode SUSE XML (%s): %w", filename, err)
+		if err := xml.Unmarshal(e.Data, &cv); err != nil {
+			return xerrors.Errorf("failed to decode SUSE XML (%s): %w", e.Filename, err)
 		}
 
 		dir := filepath.Join(cvrfDir, suseDir, osName)
-		if err = c.saveCvrfPerYear(dir, cv.Tracking.ID, cv); err != nil {
+		if err := c.saveCvrfPerYear(dir, cv.Tracking.ID, cv); err != nil {
 			return xerrors.Errorf("failed to save CVRF: %w", err)
 		}
-	}
+		return nil
+	})
 }
 
 func (c Config) saveCvrfPerYear(dirName string, cvrfID string, data Cvrf) error {

suse/cvrfarchive/archive.go

@@ -0,0 +1,103 @@
+// Package cvrfarchive walks SUSE CVRF tar archives published at
+// http://ftp.suse.com/pub/projects/security/. It hides the
+// download/decompress/tar plumbing so feed-specific code only needs
+// to handle XML decoding and persistence.
+package cvrfarchive
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
+	"errors"
+	"io"
+	"log"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/xerrors"
+
+	"github.com/aquasecurity/vuln-list-update/utils"
+)
+
+// Entry is a single XML document extracted from a SUSE CVRF archive.
+// Data is guaranteed to be valid UTF-8 (invalid bytes are stripped).
+type Entry struct {
+	Filename string
+	Data     []byte
+}
+
+// Walk downloads a SUSE CVRF archive from url, decompresses it
+// (bzip2 or gzip, detected by the URL suffix) and invokes handler
+// for every .xml entry whose base name matches nameRegexp.
+// Non-regular tar entries, empty files and non-XML files are skipped;
+// invalid UTF-8 byte sequences are stripped from the data.
+func Walk(url string, retries int, nameRegexp *regexp.Regexp, handler func(Entry) error) error {
+	// The SUSE server is sometimes unstable, so download the whole archive into
+	// memory before processing. Streaming directly from the HTTP response would
+	// make it hard to distinguish a mid-transfer disconnection (which surfaces
+	// as a truncated tar) from a legitimate parse error. The archive is only a
+	// few hundred MB, which fits comfortably in memory on CI runners.
+	body, err := utils.FetchURL(url, "", retries)
+	if err != nil {
+		return xerrors.Errorf("failed to download archive: %w", err)
+	}
+
+	decompressed, err := decompress(url, body)
+	if err != nil {
+		return err
+	}
+
+	tr := tar.NewReader(decompressed)
+	for {
+		hdr, err := tr.Next()
+		switch {
+		case errors.Is(err, io.EOF):
+			return nil
+		case err != nil:
+			return xerrors.Errorf("failed to read tar entry: %w", err)
+		case hdr.Typeflag != tar.TypeReg:
+			continue
+		}
+
+		filename := filepath.Base(hdr.Name)
+		if !strings.HasSuffix(filename, ".xml") {
+			continue
+		}
+		if nameRegexp != nil && !nameRegexp.MatchString(filename) {
+			continue
+		}
+
+		data, err := io.ReadAll(tr)
+		if err != nil {
+			return xerrors.Errorf("failed to read tar entry data: %w", err)
+		}
+		if len(data) == 0 {
+			log.Printf("empty xml: %s", filename)
+			continue
+		}
+		if !utf8.Valid(data) {
+			log.Printf("invalid UTF-8: %s", filename)
+			data = []byte(strings.ToValidUTF8(string(data), ""))
+		}
+
+		if err := handler(Entry{Filename: filename, Data: data}); err != nil {
+			return err
+		}
+	}
+}
+
+func decompress(url string, body []byte) (io.Reader, error) {
+	switch {
+	case strings.HasSuffix(url, ".tar.bz2"):
+		// The upstream archive is .tar.bz2, which is the only format used in production.
+		return bzip2.NewReader(bytes.NewReader(body)), nil
+	case strings.HasSuffix(url, ".tar.gz"):
+		// Go's compress/bzip2 lacks a Writer, so tests use .tar.gz instead.
+		return gzip.NewReader(bytes.NewReader(body))
+	default:
+		return nil, xerrors.Errorf("unsupported archive format: %s", url)
+	}
+}

suse/cvrfcve/cvrfcve.go

@@ -0,0 +1,65 @@
+package cvrfcve
+
+import (
+	"encoding/xml"
+	"fmt"
+	"log"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+
+	"github.com/aquasecurity/vuln-list-update/suse/cvrfarchive"
+	"github.com/aquasecurity/vuln-list-update/utils"
+)
+
+const (
+	cvrfCVEArchiveURL = "http://ftp.suse.com/pub/projects/security/cvrf-cve.tar.bz2"
+	cvrfDir           = "cvrf"
+	suseCVEDir        = "suse-cves"
+	retries           = 5
+)
+
+var fileRegexp = regexp.MustCompile(`^cvrf-(CVE-\d{4}-\d+)\.xml$`)
+
+type Config struct {
+	VulnListDir string
+	URL         string
+	AppFs       afero.Fs
+}
+
+func NewConfig() Config {
+	return Config{
+		VulnListDir: utils.VulnListDir(),
+		URL:         cvrfCVEArchiveURL,
+		AppFs:       afero.NewOsFs(),
+	}
+}
+
+func (c Config) Update() error {
+	log.Print("Fetching SUSE CVE CVRF archive...")
+
+	return cvrfarchive.Walk(c.URL, retries, fileRegexp, func(e cvrfarchive.Entry) error {
+		// CVE ID is taken from the file name, already validated by fileRegexp.
+		cveID := fileRegexp.FindStringSubmatch(e.Filename)[1]
+
+		var cv Cvrf
+		if err := xml.Unmarshal(e.Data, &cv); err != nil {
+			return xerrors.Errorf("failed to decode SUSE CVE CVRF XML (%s): %w", e.Filename, err)
+		}
+
+		return c.saveCVEPerYear(cveID, cv)
+	})
+}
+
+func (c Config) saveCVEPerYear(cveID string, data Cvrf) error {
+	year := strings.Split(cveID, "-")[1]
+	yearDir := filepath.Join(c.VulnListDir, cvrfDir, suseCVEDir, year)
+	fileName := fmt.Sprintf("%s.json", cveID)
+	if err := utils.WriteJSON(c.AppFs, yearDir, fileName, data); err != nil {
+		return xerrors.Errorf("failed to write file: %w", err)
+	}
+	return nil
+}