Feature: Add apt origin (Debian support) (#275)

Author: Zweih

internal/origins/apt/constants.go

@@ -0,0 +1,36 @@
+package apt
+
+const (
+	fieldInstalledSize = "Installed-Size"
+	fieldPackage       = "Package"
+	fieldVersion       = "Version"
+	fieldArchitecture  = "Architecture"
+	fieldDescription   = "Description"
+	fieldHomepage      = "Homepage"
+	fieldMaintainer    = "Maintainer"
+	fieldConflicts     = "Conflicts"
+	fieldBreaks        = "Breaks"
+	fieldReplaces      = "Replaces"
+	fieldDepends       = "Depends"
+	fieldPreDepends    = "Pre-Depends"
+	fieldReccommends   = "Reccommends"
+	fieldSuggests      = "Suggests"
+	fieldProvides      = "Provides"
+	fieldPriority      = "Priority"
+	fieldEssential     = "Essential"
+
+	dpkgPath = "/var/lib/dpkg/status"
+
+	installReasonPath = "/var/lib/apt/extended_states"
+	packagePrefix     = "Package:"
+	autoInstallPrefix = "Auto-Installed:"
+
+	licensePath     = "/usr/share/doc"
+	licenseFileName = "copyright"
+	filesPrefix     = "Files:"
+	licensePrefix   = "License:"
+
+	pkgModRoot = "/var/lib/dpkg/info"
+	listExt    = ".list"
+	md5SumsExt = ".md5sums"
+)

internal/origins/apt/driver.go

@@ -0,0 +1,47 @@
+package apt
+
+import (
+	"fmt"
+	"os"
+	"qp/internal/pkgdata"
+)
+
+type AptDriver struct{}
+
+func (d *AptDriver) Name() string {
+	return "apt"
+}
+
+func (d *AptDriver) Detect() bool {
+	_, err := os.Stat(dpkgPath)
+	return err == nil
+}
+
+func (d *AptDriver) Load() ([]*pkgdata.PkgInfo, error) {
+	return fetchPackages(d.Name())
+}
+
+func (d *AptDriver) ResolveDeps(pkgs []*pkgdata.PkgInfo) ([]*pkgdata.PkgInfo, error) {
+	return pkgdata.ResolveDependencyGraph(pkgs, nil)
+}
+
+func (d *AptDriver) LoadCache(path string, modTime int64) ([]*pkgdata.PkgInfo, error) {
+	return pkgdata.LoadProtoCache(path, modTime)
+}
+
+func (d *AptDriver) SaveCache(
+	path string,
+	pkgs []*pkgdata.PkgInfo,
+	modTime int64,
+) error {
+	return pkgdata.SaveProtoCache(pkgs, path, modTime)
+}
+
+func (d *AptDriver) SourceModified() (int64, error) {
+	dirInfo, err := os.Stat(dpkgPath)
+	if err != nil {
+		return 0, fmt.Errorf("failed to read %s DB mod time: %v", d.Name(), err)
+	}
+
+	return dirInfo.ModTime().Unix(), nil
+}

internal/origins/apt/fetch.go

@@ -0,0 +1,23 @@
+package apt
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"qp/internal/pkgdata"
+)
+
+func fetchPackages(origin string) ([]*pkgdata.PkgInfo, error) {
+	file, err := os.Open(dpkgPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open status file: %w", err)
+	}
+	defer file.Close()
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read status file: %w", err)
+	}
+
+	return parseStatusFile(data, origin)
+}

internal/origins/apt/license.go

@@ -0,0 +1,98 @@
+package apt
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"qp/internal/pkgdata"
+	"strings"
+)
+
+var licenseHints = []struct {
+	phrase  string
+	license string
+	weight  int
+}{
+	{"most of the gnu c library", "LGPL-2.1+", 10},
+	{"gnu c library is free software", "LGPL-2.1+", 10},
+	{"gnu lesser general public license", "LGPL-2.1+", 8},
+	{"gnu general public license", "GPL-2+", 6},
+	{"distributed under the bsd license", "BSD", 5},
+	{"bsd license", "BSD", 5},
+	{"bsd-4-clause", "BSD-4-Clause", 4},
+	{"mit license", "MIT", 3},
+	{"expat", "Expat", 3},
+	{"isc license", "ISC", 3},
+	{"public domain", "public-domain", 2},
+	{"lgpl", "LGPL", 2},
+}
+
+func extractLicense(pkg *pkgdata.PkgInfo) error {
+	path := filepath.Join(licensePath, pkg.Name, licenseFileName)
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("failed to read license file for %s: %w", pkg.Name, err)
+	}
+	defer file.Close()
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		return fmt.Errorf("failed to load license file for %s: %w", pkg.Name, err)
+	}
+
+	var fallbackLicense string
+
+	for block := range bytes.SplitSeq(data, []byte("\n\n")) {
+		var isFilesAll bool
+		var license string
+
+		for byteLine := range bytes.SplitSeq(block, []byte("\n")) {
+			line := string(byteLine)
+			if strings.HasPrefix(line, filesPrefix) {
+				isFilesAll = strings.TrimSpace(strings.TrimPrefix(line, filesPrefix)) == "*"
+			}
+
+			if strings.HasPrefix(line, licensePrefix) {
+				license = strings.TrimSpace(strings.TrimPrefix(line, licensePrefix))
+				if fallbackLicense == "" {
+					fallbackLicense = license
+				}
+			}
+		}
+
+		if isFilesAll && license != "" {
+			pkg.License = license
+			return nil
+		}
+	}
+
+	if match, ok := matchLicenseText(data); ok {
+		pkg.License = match
+		return nil
+	}
+
+	if fallbackLicense != "" {
+		pkg.License = fallbackLicense
+		return nil
+	}
+
+	return fmt.Errorf("no license found for %s", pkg.Name)
+}
+
+func matchLicenseText(data []byte) (string, bool) {
+	text := strings.ToLower(string(data))
+
+	bestMatch := ""
+	bestWeight := 0
+
+	for _, hint := range licenseHints {
+		if strings.Contains(text, hint.phrase) && hint.weight > bestWeight {
+			bestMatch = hint.license
+			bestWeight = hint.weight
+		}
+	}
+
+	return bestMatch, bestMatch != ""
+}

internal/origins/apt/parser.go

@@ -0,0 +1,139 @@
+package apt
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"qp/internal/consts"
+	"qp/internal/pkgdata"
+	"strconv"
+	"strings"
+)
+
+func parseStatusFile(data []byte, origin string) ([]*pkgdata.PkgInfo, error) {
+	reasonMap, err := loadInstallReasons()
+	if err != nil {
+		return []*pkgdata.PkgInfo{}, err
+	}
+
+	var collectedErrors []error
+	pkgs := []*pkgdata.PkgInfo{}
+
+	for block := range bytes.SplitSeq(data, []byte("\n\n")) {
+		if len(block) < 1 {
+			continue
+		}
+
+		pkg, err := parseStatusBlock(block, reasonMap, origin)
+		if err != nil {
+			collectedErrors = append(collectedErrors, err)
+		}
+
+		pkgs = append(pkgs, pkg)
+	}
+
+	if len(collectedErrors) > 0 {
+		return pkgs, errors.Join(collectedErrors...)
+	}
+
+	return pkgs, nil
+}
+
+func parseStatusFields(block []byte) map[string]string {
+	fields := make(map[string]string)
+
+	for line := range bytes.SplitSeq(block, []byte("\n")) {
+		parts := bytes.SplitN(line, []byte(":"), 2)
+		if len(parts) != 2 {
+			continue
+		}
+
+		key := strings.TrimSpace(string(parts[0]))
+		value := strings.TrimSpace(string(parts[1]))
+		fields[key] = value
+	}
+
+	return fields
+}
+
+func parseStatusBlock(block []byte, reasonMap map[string]string, origin string) (*pkgdata.PkgInfo, error) {
+	fields := parseStatusFields(block)
+	var collected []error
+	pkg := &pkgdata.PkgInfo{}
+	meta := map[string]string{}
+
+	for key, value := range fields {
+		switch key {
+		case fieldInstalledSize:
+			size, err := strconv.Atoi(value)
+			if err != nil {
+				collected = append(collected, fmt.Errorf("invalid install size for %s: %v", pkg.Name, err))
+				continue
+			}
+			pkg.Size = consts.KB * int64(size)
+
+		case fieldPackage:
+			pkg.Name = value
+
+		case fieldVersion:
+			pkg.Version = value
+
+		case fieldArchitecture:
+			pkg.Arch = value
+
+		case fieldDescription:
+			pkg.Description = value
+
+		case fieldHomepage:
+			pkg.Url = value
+
+		case fieldMaintainer:
+			pkg.Packager = value
+
+		case fieldConflicts, fieldBreaks:
+			pkg.Conflicts = append(pkg.Conflicts, parseRelations(value)...)
+
+		case fieldReplaces:
+			pkg.Replaces = parseRelations(value)
+
+		case fieldDepends, fieldPreDepends:
+			pkg.Depends = append(pkg.Depends, parseRelations(value)...)
+
+		case fieldReccommends, fieldSuggests:
+			pkg.OptDepends = append(pkg.OptDepends, parseRelations(value)...)
+
+		case fieldProvides:
+			pkg.Provides = parseRelations(value)
+
+		case fieldPriority, fieldEssential:
+			meta[key] = value
+		}
+	}
+
+	if err := getInstallTime(pkg); err != nil {
+		collected = append(collected, err)
+	}
+	if err := extractLicense(pkg); err != nil {
+		collected = append(collected, err)
+	}
+
+	pkg.Origin = origin
+
+	// TODO: for dpkg-only systems, perhaps return "unknown" for non-system packages
+	if isSystem(meta) {
+		pkg.Reason = "system"
+	} else if reasonMap[pkg.Name] == "dependency" {
+		pkg.Reason = "dependency"
+	} else {
+		pkg.Reason = "explicit"
+	}
+
+	return pkg, errors.Join(collected...)
+}
+
+func isSystem(meta map[string]string) bool {
+	priority := strings.ToLower(meta[fieldPriority])
+	essential := strings.ToLower(meta[fieldEssential])
+
+	return priority == "required" || essential == "yes"
+}

internal/origins/apt/reasons.go

@@ -0,0 +1,44 @@
+package apt
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+)
+
+// only works for APT systems, we'll need to add more heuristics to get reasons for dpkg only systems
+func loadInstallReasons() (map[string]string, error) {
+	file, err := os.Open(installReasonPath)
+	if err != nil {
+		return map[string]string{}, nil
+	}
+	defer file.Close()
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read install reason file: %w", err)
+	}
+
+	reasonMap := make(map[string]string)
+
+	for block := range bytes.SplitSeq(data, []byte("\n\n")) {
+		var name string
+		var isAuto bool
+
+		for line := range bytes.SplitSeq(block, []byte("\n")) {
+			if bytes.HasPrefix(line, []byte(packagePrefix)) {
+				name = string(bytes.TrimSpace(line[len(packagePrefix):]))
+			} else if bytes.HasPrefix(line, []byte(autoInstallPrefix)) {
+				val := string(bytes.TrimSpace(line[len(autoInstallPrefix):]))
+				isAuto = val == "1"
+			}
+		}
+
+		if name != "" && isAuto {
+			reasonMap[name] = "dependency"
+		}
+	}
+
+	return reasonMap, nil
+}