Merge pull request #158 from projectdiscovery/shubh/headless-system-chrome

fix: use system chromium for httpx/nuclei headless when present

Author: ShubhamRasal

.github/workflows/build-test.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - '**.go'
       - '**.mod'
+      - 'Dockerfile'
   workflow_dispatch:
 
 permissions:
@@ -46,4 +47,28 @@ jobs:
 
       - name: Race Condition Tests
         run: go build -race .
-        working-directory: cmd/pd-agent/
+        working-directory: cmd/pd-agent/
+
+  build-image:
+    name: Build Image
+    needs: [lint]
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build image (no push)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: false
+          build-args: |
+            VERSION=pr-${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/codeql-analysis.yml.old

@@ -1,10 +1,12 @@
-name: 🎉 Release Binary
+name: 🎉 Release
 
 on:
   push:
     tags:
       - v*
-  workflow_dispatch:
+
+permissions:
+  contents: write
 
 jobs:
   build-mac:
@@ -81,4 +83,36 @@ jobs:
           version: latest
           args: release -f .goreleaser/windows.yml --clean
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  publish-image:
+    runs-on: ubuntu-latest-16-cores
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          build-args: |
+            VERSION=${{ github.ref_name }}
+          tags: ghcr.io/projectdiscovery/pd-agent:latest,ghcr.io/projectdiscovery/pd-agent:${{ github.ref_name }}

.github/workflows/ghcr-push.yml

@@ -9,33 +9,6 @@ cmd/fleetx/fleetx
 cmd/pd-agent/pd-agent
 output/*
 
-# Local runtime and scratch
-.claude/
-REVIEW.md
-chrome-ps-aux.txt
-ubisoft.txt
-
-# SQLite runtime files
-pd-agent-*.db
-pd-agent-*.db-shm
-pd-agent-*.db-wal
-
-# Local binaries at repo root
-/pd-agent-linux-amd64
-/pd-agent-linux-arm64
-/pd-agent-windows-amd64.exe
-/interactsh-debug
-
-# Scan logs
-scan-*.log
-
-# Local helper scripts at repo root
-/bench-create.sh
-/bench-create-multi.sh
-/cidr-ip.sh
-/start.sh
-/start-mac.sh
-/ssh-config
 
 # Local build artifacts
 /build/
@@ -48,3 +21,6 @@ scan-*.log
 !/docs/configuration.md
 /pd-agent-deployment.yaml
 /pd-agent-deployment-prod.yaml
+
+local/
+.claude/

.github/workflows/release-test.yml

@@ -3,6 +3,7 @@ module github.com/projectdiscovery/pd-agent
 go 1.25.7
 
 require (
+	github.com/go-rod/rod v0.116.2
 	github.com/json-iterator/go v1.1.12
 	github.com/klauspost/compress v1.18.4
 	github.com/miekg/dns v1.1.68
@@ -175,7 +176,6 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.26.0 // indirect
-	github.com/go-rod/rod v0.116.2 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
 	github.com/go-sql-driver/mysql v1.9.3 // indirect
 	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect

.github/workflows/release-binary.yml .github/workflows/release.yaml.github/workflows/release-binary.yml renamed to .github/workflows/release.yaml

@@ -37,6 +37,12 @@ func EnsureAll() (failed []string) {
 func warmupBrowser() error {
 	slog.Info("prereq: validating browser (embedded httpx screenshot probe)...")
 
+	if path, ok := runtools.SystemChromePath(); ok {
+		slog.Info("prereq: using system browser", "path", path)
+	} else {
+		slog.Info("prereq: no system browser found, go-rod will download Chrome (amd64 only)")
+	}
+
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.Header().Set("Content-Type", "text/html")
 		_, _ = w.Write([]byte(`<html><body><h1>pd-agent warmup</h1></body></html>`))
@@ -70,15 +76,24 @@ func warmupBrowser() error {
 	})
 	if runErr != nil {
 		msg := runErr.Error()
-		if strings.Contains(msg, "cannot open shared object") ||
+		switch {
+		case strings.Contains(msg, "cannot open shared object") ||
 			strings.Contains(msg, "Failed to launch the browser") ||
-			strings.Contains(msg, "error while loading shared libraries") {
+			strings.Contains(msg, "error while loading shared libraries"):
 			slog.Error("prereq: browser validation failed, missing Chrome dependencies",
 				"error", msg,
 				"hint", "install Chrome deps: sudo apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libxdamage1 libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2 libnspr4 libnss3 libxcomposite1 libxfixes3 libxshmfence1 libxkbcommon0")
 			return fmt.Errorf("missing Chrome shared libraries")
+		case strings.Contains(msg, "browser is not installed") ||
+			strings.Contains(msg, "find a browser binary") ||
+			strings.Contains(msg, "find a valid URL to download"):
+			slog.Error("prereq: browser validation failed, no chrome/chromium found",
+				"error", msg,
+				"hint", "install a system browser (e.g. sudo apt-get install -y chromium); pd-agent only auto-downloads Chrome on amd64, so arm64 hosts must have one installed")
+			return fmt.Errorf("no chrome/chromium browser available")
+		default:
+			slog.Warn("prereq: browser validation httpx probe had non-fatal error", "error", msg)
 		}
-		slog.Warn("prereq: browser validation httpx probe had non-fatal error", "error", msg)
 	}
 
 	if shotPath := readFirstScreenshotPath(tmpPath); shotPath != "" {
@@ -89,7 +104,7 @@ func warmupBrowser() error {
 	}
 
 	slog.Error("prereq: browser validation failed, screenshot was not written",
-		"hint", "Chrome may need more time on first launch (it downloads ~150MB); rerun, or pre-install Chrome via go-rod's manager")
+		"hint", "ensure a system chromium/chrome is installed and on PATH (the docker image ships /usr/bin/chromium)")
 	return fmt.Errorf("screenshot not produced")
 }
 

.gitignore

@@ -0,0 +1,33 @@
+package runtools
+
+import (
+	"sync"
+
+	"github.com/go-rod/rod/lib/launcher"
+)
+
+type chromeInfo struct {
+	path string
+	ok   bool
+}
+
+// resolveSystemChrome runs go-rod's LookPath once. Matching the check
+// httpx/nuclei run internally means we never force "use installed" on a host
+// that has none, which would hard-error instead of falling back to download.
+var resolveSystemChrome = sync.OnceValue(func() chromeInfo {
+	path, ok := launcher.LookPath()
+	return chromeInfo{path: path, ok: ok}
+})
+
+// systemChromeAvailable reports whether a chrome/chromium binary is installed
+// on this host. httpx and nuclei must be told to use it (go-rod ships no
+// arm64-linux chrome to download); when absent, go-rod downloads its own,
+// which only works on amd64.
+func systemChromeAvailable() bool { return resolveSystemChrome().ok }
+
+// SystemChromePath returns the resolved system browser path and whether one was
+// found, so callers can log which browser the embedded scanners will use.
+func SystemChromePath() (string, bool) {
+	c := resolveSystemChrome()
+	return c.path, c.ok
+}

go.mod

@@ -18,8 +18,7 @@ type HttpxOptions struct {
 	Concurrency int
 	Timeout     time.Duration
 	Screenshot  bool
-	// ScreenshotTimeout bounds each headless screenshot. First-time runs need
-	// longer because Chrome has to download.
+	// ScreenshotTimeout bounds each headless screenshot.
 	ScreenshotTimeout time.Duration
 	// DisableStdout suppresses httpx's per-result stdout writes.
 	DisableStdout    bool
@@ -56,6 +55,7 @@ func RunHttpx(ctx context.Context, targets []string, opts HttpxOptions) (string,
 		Timeout:            int(opts.Timeout.Seconds()),
 		Screenshot:         opts.Screenshot,
 		ScreenshotTimeout:  opts.ScreenshotTimeout,
+		UseInstalledChrome: systemChromeAvailable(),
 		DisableStdout:      opts.DisableStdout,
 		StoreResponseDir:   opts.StoreResponseDir,
 		StatusCode:         true,