fix(observability): probe only when observability is installed (#2482)

We should not train people or machines to ignore red ERROR messages.
With this change and #2479, we have zero ERROR log entries in a
successful build.

Previously RetrieveObservabilityEvents always sent three `test -f`
SSH commands to probe for the observability events file, even when the
hook was never installed. Each probe exits non-zero (file not found),
causing sendSSHCommand to log ERROR three times for every build to the
console.

During CPIO generation, scan the base initramfs for the hook's
sentinel file (etc/tetragon/tetragon.tp.d/network-monitor.yaml) and
record the result in cfg.ObservabilityHook. This is accurate regardless
of how the package got into the image — QEMU_ADDITIONAL_PACKAGES,
QEMU_BASE_INITRAMFS, or any other mechanism. RetrieveObservabilityEvents
returns immediately when ObservabilityHook is false, and treats a
missing events file as an error when it is true.

We can now also correctly ERROR when there _was_ a observability hook
installed rather than just assuming it was not there.

Store the result of that scan in a sidecar (<cpio>.observability) so we
do not have to scan on cached initramfs.  The sidecar is invalidated
automatically when the CPIO is newer (fresh build,
QEMU_ADDITIONAL_PACKAGES change, etc.).

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: smoser

pkg/build/build.go

@@ -774,7 +774,7 @@ func (b *Build) BuildPackage(ctx context.Context) error {
 	// is installed. Only applicable to QEMU builds which run in a full VM.
 	if b.Runner.Name() == container.QemuName {
 		if obsEvents, err := container.RetrieveObservabilityEvents(ctx, cfg); err != nil {
-			log.Warnf("failed to retrieve observability events: %v", err)
+			log.Errorf("failed to retrieve observability events: %v", err)
 		} else if obsEvents != nil {
 			container.LogObservabilityEvents(ctx, obsEvents)
 		}

pkg/container/config.go

@@ -81,4 +81,9 @@ type Config struct {
 	VirtiofsEnabled     bool   // Whether virtiofs is enabled for cache
 	VirtiofsdPID        int    // PID of virtiofsd daemon for cleanup
 	VirtiofsdSocketPath string // Path to Unix socket for virtiofsd
+
+	// ObservabilityHook is true when the observability hook's sentinel file
+	// was found in the initramfs CPIO during VM setup. When false,
+	// RetrieveObservabilityEvents returns immediately without probing the VM.
+	ObservabilityHook bool
 }

pkg/container/observability.go

@@ -31,6 +31,12 @@ var observabilityEventPaths = []string{
 	"/tmp/observability/events.log",
 }
 
+// observabilityHookSentinel is a file installed exclusively by the
+// observability hook package. Its presence in the initramfs CPIO confirms
+// the hook is installed, regardless of how it was included.
+// CPIO record names are stored without a leading slash.
+const observabilityHookSentinel = "etc/tetragon/tetragon.tp.d/network-monitor.yaml"
+
 // ObservabilityEvents holds parsed event data retrieved from the build VM.
 type ObservabilityEvents struct {
 	// RawData is the raw NDJSON event data.
@@ -60,18 +66,19 @@ type NetworkConnection struct {
 // via the SSHControlClient (port 2223, unchrooted root access). This should
 // be called after the build completes but before TerminatePod.
 //
-// Returns nil with no error if the observability hook is not installed or no
-// events were generated. This makes the feature fully optional — default
-// builds without the hook are completely unaffected.
+// If cfg.ObservabilityHook is false the function returns immediately without
+// probing the VM. If true, a missing events file is treated as an error.
 func RetrieveObservabilityEvents(ctx context.Context, cfg *Config) (*ObservabilityEvents, error) {
+	if !cfg.ObservabilityHook {
+		return nil, nil
+	}
 	if cfg.SSHControlClient == nil {
 		return nil, nil
 	}
 
 	log := clog.FromContext(ctx)
 
-	// Probe known event file locations. If none exist, the observability
-	// hook is not installed and we silently return nil.
+	// Probe known event file locations.
 	eventsPath := ""
 	for _, path := range observabilityEventPaths {
 		err := sendSSHCommand(ctx, cfg.SSHControlClient, cfg, nil, nil, nil, false,
@@ -83,9 +90,7 @@ func RetrieveObservabilityEvents(ctx context.Context, cfg *Config) (*Observabili
 	}
 
 	if eventsPath == "" {
-		// No events file found — observability hook is not installed.
-		// This is the normal case for default builds; return silently.
-		return nil, nil
+		return nil, fmt.Errorf("observability hook is installed but no events file found at any known path: %v", observabilityEventPaths)
 	}
 
 	log.Infof("qemu: found observability events at %s", eventsPath)

pkg/container/qemu_runner.go

@@ -2231,6 +2231,11 @@ func generateCpio(ctx context.Context, cfg *Config) (string, error) {
 		}
 	}
 
+	// Detect whether the observability hook is present. We cache the result in
+	// a sidecar file (<cpio>.observability) so we only scan the archive once
+	// per cached initramfs rather than on every melange invocation.
+	cfg.ObservabilityHook = observabilityHookPresent(ctx, baseInitramfs)
+
 	// Inject SSH host keys and modules
 	return injectRuntimeData(ctx, cfg, os.Getenv("QEMU_KERNEL_MODULES"), baseInitramfs)
 }
@@ -2317,6 +2322,68 @@ func GenerateBaseInitramfs(ctx context.Context, arch apko_types.Architecture, cf
 	return nil
 }
 
+// cpioContainsPath reports whether target exists as a record name in the CPIO
+// archive at cpioFile. CPIO record names are stored without a leading slash.
+func cpioContainsPath(cpioFile, target string) (bool, error) {
+	f, err := os.Open(cpioFile)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	rr, err := cpio.Newc.NewFileReader(f)
+	if err != nil {
+		return false, err
+	}
+
+	errFound := errors.New("found")
+	err = cpio.ForEachRecord(rr, func(r cpio.Record) error {
+		if r.Name == target {
+			return errFound
+		}
+		return nil
+	})
+	if errors.Is(err, errFound) {
+		return true, nil
+	}
+	return false, err
+}
+
+// observabilityHookPresent reports whether the observability hook is present in
+// the CPIO archive at cpioFile. The result is cached in a sidecar file
+// (<cpioFile>.observability) so the archive is only scanned once per cached
+// initramfs. Subsequent calls just read the tiny sidecar.
+func observabilityHookPresent(ctx context.Context, cpioFile string) bool {
+	sidecar := cpioFile + ".observability"
+
+	// Use the cached result when the sidecar is at least as new as the CPIO.
+	cpioInfo, cpioErr := os.Stat(cpioFile)
+	sidecarInfo, sidecarErr := os.Stat(sidecar)
+	if cpioErr == nil && sidecarErr == nil && !sidecarInfo.ModTime().Before(cpioInfo.ModTime()) {
+		data, err := os.ReadFile(sidecar)
+		if err == nil {
+			return strings.TrimSpace(string(data)) == "true"
+		}
+	}
+
+	// Sidecar missing or stale — scan the archive.
+	present, err := cpioContainsPath(cpioFile, observabilityHookSentinel)
+	if err != nil {
+		clog.FromContext(ctx).Debugf("qemu: could not inspect initramfs for observability hook: %v", err)
+		return false
+	}
+
+	// Write the sidecar so future invocations skip the scan.
+	val := "false"
+	if present {
+		val = "true"
+	}
+	if err := os.WriteFile(sidecar, []byte(val+"\n"), 0o600); err != nil {
+		clog.FromContext(ctx).Debugf("qemu: could not write observability sidecar: %v", err)
+	}
+	return present
+}
+
 // getAdditionalPackages parses and validates the QEMU_ADDITIONAL_PACKAGES environment variable.
 // Returns a list of package names to add to the initramfs, or empty slice if none/invalid.
 func getAdditionalPackages(ctx context.Context) []string {