internal/test: run tests iff npx exists and DONT_USE_NETWORK is not set

Signed-off-by: Xe Iaso <me@xeiaso.net>

Author: Xe

cmd/anubis/main.go

@@ -8,6 +8,8 @@ import (
 	"log/slog"
 	"net"
 	"net/http"
+	"net/http/httputil"
+	"net/url"
 	"os"
 	"os/signal"
 	"strconv"
@@ -18,11 +20,9 @@ import (
 
 	"github.com/TecharoHQ/anubis"
 	"github.com/TecharoHQ/anubis/internal"
-	"github.com/TecharoHQ/anubis/lib"
+	libanubis "github.com/TecharoHQ/anubis/lib"
 	"github.com/TecharoHQ/anubis/lib/policy/config"
 	"github.com/TecharoHQ/anubis/web"
-	"github.com/TecharoHQ/anubis/xess"
-	"github.com/a-h/templ"
 	"github.com/facebookgo/flagenv"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
@@ -90,6 +90,34 @@ func setupListener(network string, address string) (net.Listener, string) {
 	return listener, formattedAddress
 }
 
+func makeReverseProxy(target string) (http.Handler, error) {
+	u, err := url.Parse(target)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse target URL: %w", err)
+	}
+
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+
+	// https://github.com/oauth2-proxy/oauth2-proxy/blob/4e2100a2879ef06aea1411790327019c1a09217c/pkg/upstream/http.go#L124
+	if u.Scheme == "unix" {
+		// clean path up so we don't use the socket path in proxied requests
+		addr := u.Path
+		u.Path = ""
+		// tell transport how to dial unix sockets
+		transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			dialer := net.Dialer{}
+			return dialer.DialContext(ctx, "unix", addr)
+		}
+		// tell transport how to handle the unix url scheme
+		transport.RegisterProtocol("unix", libanubis.UnixRoundTripper{Transport: transport})
+	}
+
+	rp := httputil.NewSingleHostReverseProxy(u)
+	rp.Transport = transport
+
+	return rp, nil
+}
+
 func main() {
 	flagenv.Parse()
 	flag.Parse()
@@ -103,13 +131,18 @@ func main() {
 		return
 	}
 
-	s, err := lib.New(*target, *policyFname, *challengeDifficulty)
+	rp, err := makeReverseProxy(*target)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("can't make reverse proxy: %v", err)
+	}
+
+	policy, err := libanubis.LoadPoliciesOrDefault(*policyFname, *challengeDifficulty)
+	if err != nil {
+		log.Fatalf("can't parse policy file: %v", err)
 	}
 
 	fmt.Println("Rule error IDs:")
-	for _, rule := range s.Policy.Bots {
+	for _, rule := range policy.Bots {
 		if rule.Action != config.RuleDeny {
 			continue
 		}
@@ -123,25 +156,13 @@ func main() {
 	}
 	fmt.Println()
 
-	mux := http.NewServeMux()
-	xess.Mount(mux)
-
-	mux.Handle(anubis.StaticPath, internal.UnchangingCache(http.StripPrefix(anubis.StaticPath, http.FileServerFS(web.Static))))
-
-	// mux.HandleFunc("GET /.within.website/x/cmd/anubis/static/js/main.mjs", serveMainJSWithBestEncoding)
-
-	mux.HandleFunc("POST /.within.website/x/cmd/anubis/api/make-challenge", s.MakeChallenge)
-	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/pass-challenge", s.PassChallenge)
-	mux.HandleFunc("GET /.within.website/x/cmd/anubis/api/test-error", s.TestError)
-
-	if *robotsTxt {
-		mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, web.Static, "static/robots.txt")
-		})
-
-		mux.HandleFunc("/.well-known/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			http.ServeFileFS(w, r, web.Static, "static/robots.txt")
-		})
+	s, err := libanubis.New(libanubis.Options{
+		Next:           rp,
+		Policy:         policy,
+		ServeRobotsTXT: *robotsTxt,
+	})
+	if err != nil {
+		log.Fatalf("can't construct libanubis.Server: %v", err)
 	}
 
 	wg := new(sync.WaitGroup)
@@ -154,10 +175,8 @@ func main() {
 		go metricsServer(ctx, wg.Done)
 	}
 
-	mux.HandleFunc("/", s.MaybeReverseProxy)
-
 	var h http.Handler
-	h = mux
+	h = s
 	h = internal.DefaultXRealIP(*debugXRealIPDefault, h)
 	h = internal.XForwardedForToXRealIP(h)
 
@@ -212,11 +231,6 @@ func metricsServer(ctx context.Context, done func()) {
 	}
 }
 
-func ohNoes(w http.ResponseWriter, r *http.Request, err error) {
-	slog.Error("super fatal error", "err", err)
-	templ.Handler(web.Base("Oh noes!", web.ErrorPage("An internal server error happened")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r)
-}
-
 func serveMainJSWithBestEncoding(w http.ResponseWriter, r *http.Request) {
 	priorityList := []string{"zstd", "br", "gzip"}
 	enc2ext := map[string]string{

internal/test/playwright_test.go

@@ -1,5 +1,3 @@
-//go:build integration
-
 // Integration tests for Anubis, using Playwright.
 //
 // These tests require an already running Anubis and Playwright server.
@@ -16,31 +14,60 @@
 package test
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"os"
+	"os/exec"
 	"testing"
 	"time"
 
+	"github.com/TecharoHQ/anubis"
+	libanubis "github.com/TecharoHQ/anubis/lib"
 	"github.com/playwright-community/playwright-go"
 )
 
 var (
-	anubisServer          = flag.String("anubis", "http://localhost:8923", "Anubis server URL")
 	serverBindAddr        = flag.String("bind", "localhost:3923", "test server bind address")
+	playwrightPort        = flag.Int("playwright-port", 3000, "Playwright port")
 	playwrightServer      = flag.String("playwright", "ws://localhost:3000", "Playwright server URL")
 	playwrightMaxTime     = flag.Duration("playwright-max-time", 5*time.Second, "maximum time for Playwright requests")
 	playwrightMaxHardTime = flag.Duration("playwright-max-hard-time", 5*time.Minute, "maximum time for hard Playwright requests")
 
 	testCases = []testCase{
-		{name: "firefox", action: actionChallenge, realIP: placeholderIP, userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0"},
-		{name: "headlessChrome", action: actionDeny, realIP: placeholderIP, userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.6099.28 Safari/537.36"},
-		{name: "kagiBadIP", action: actionChallenge, isHard: true, realIP: placeholderIP, userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)"},
-		{name: "kagiGoodIP", action: actionAllow, realIP: "216.18.205.234", userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)"},
-		{name: "unknownAgent", action: actionAllow, realIP: placeholderIP, userAgent: "AnubisTest/0"},
+		{
+			name:      "firefox",
+			action:    actionChallenge,
+			realIP:    placeholderIP,
+			userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:136.0) Gecko/20100101 Firefox/136.0",
+		},
+		{
+			name:      "headlessChrome",
+			action:    actionDeny,
+			realIP:    placeholderIP,
+			userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.6099.28 Safari/537.36",
+		},
+		{
+			name:      "kagiBadIP",
+			action:    actionChallenge,
+			isHard:    true,
+			realIP:    placeholderIP,
+			userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)",
+		},
+		{
+			name:      "kagiGoodIP",
+			action:    actionAllow,
+			realIP:    "216.18.205.234",
+			userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)",
+		},
+		{
+			name:      "unknownAgent",
+			action:    actionAllow,
+			realIP:    placeholderIP,
+			userAgent: "AnubisTest/0",
+		},
 	}
 )
 
@@ -49,7 +76,8 @@ const (
 	actionDeny      action = "DENY"
 	actionChallenge action = "CHALLENGE"
 
-	placeholderIP = "fd11:5ee:bad:c0de::"
+	placeholderIP     = "fd11:5ee:bad:c0de::"
+	playwrightVersion = "1.50.1"
 )
 
 type action string
@@ -61,9 +89,86 @@ type testCase struct {
 	realIP, userAgent string
 }
 
+func doesNPXExist(t *testing.T) {
+	t.Helper()
+
+	if _, err := exec.LookPath("npx"); err != nil {
+		t.Skipf("npx not found in PATH, skipping integration smoke testing: %v", err)
+	}
+}
+
+func run(t *testing.T, command string) string {
+	t.Helper()
+
+	shPath, err := exec.LookPath("sh")
+	if err != nil {
+		t.Fatalf("[unexpected] %v", err)
+	}
+
+	t.Logf("running command: %s", command)
+
+	cmd := exec.Command(shPath, "-c", command)
+	cmd.Stdin = nil
+	cmd.Stderr = os.Stderr
+	output, err := cmd.Output()
+	if err != nil {
+		t.Fatalf("can't run command: %v", err)
+	}
+
+	return string(output)
+}
+
+func daemonize(t *testing.T, command string) {
+	t.Helper()
+
+	shPath, err := exec.LookPath("sh")
+	if err != nil {
+		t.Fatalf("[unexpected] %v", err)
+	}
+
+	t.Logf("daemonizing command: %s", command)
+
+	cmd := exec.Command(shPath, "-c", command)
+	cmd.Stdin = nil
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("can't daemonize command: %v", err)
+	}
+
+	t.Cleanup(func() {
+		cmd.Process.Kill()
+	})
+}
+
+func startPlaywright(t *testing.T) {
+	t.Helper()
+
+	run(t, fmt.Sprintf("npx --yes playwright@%s install", playwrightVersion))
+	daemonize(t, fmt.Sprintf("npx --yes playwright@%s run-server --port %d", playwrightVersion, *playwrightPort))
+
+	for true {
+		if _, err := http.Get(fmt.Sprintf("http://localhost:%d", *playwrightPort)); err != nil {
+			time.Sleep(250 * time.Millisecond)
+			continue
+		}
+		break
+	}
+}
+
 func TestPlaywrightBrowser(t *testing.T) {
+	if os.Getenv("DONT_USE_NETWORK") != "" {
+		t.Skip("test requires network egress")
+		return
+	}
+
+	doesNPXExist(t)
+	startPlaywright(t)
+
 	pw := setupPlaywright(t)
-	spawnTestServer(t)
+	anubisURL := spawnAnubis(t)
+
 	browsers := []playwright.BrowserType{pw.Chromium, pw.Firefox, pw.WebKit}
 
 	for _, typ := range browsers {
@@ -75,7 +180,7 @@ func TestPlaywrightBrowser(t *testing.T) {
 					t.Skip("skipping hard challenge with deadline")
 				}
 
-				perfomedAction := executeTestCase(t, tc, typ)
+				perfomedAction := executeTestCase(t, tc, typ, anubisURL)
 
 				if perfomedAction != tc.action {
 					t.Errorf("unexpected test result, expected %s, got %s", tc.action, perfomedAction)
@@ -97,7 +202,7 @@ func buildBrowserConnect(name string) string {
 	return u.String()
 }
 
-func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType) action {
+func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType, anubisURL string) action {
 	deadline, _ := t.Deadline()
 
 	browser, err := typ.Connect(buildBrowserConnect(typ.Name()), playwright.BrowserTypeConnectOptions{
@@ -129,7 +234,7 @@ func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType) acti
 	// Attempt challenge.
 
 	start := time.Now()
-	_, err = page.Goto(*anubisServer, playwright.PageGotoOptions{
+	_, err = page.Goto(anubisURL, playwright.PageGotoOptions{
 		Timeout: pwTimeout(tc, deadline),
 	})
 	if err != nil {
@@ -252,25 +357,34 @@ func setupPlaywright(t *testing.T) *playwright.Playwright {
 	return pw
 }
 
-func spawnTestServer(t *testing.T) {
+func spawnAnubis(t *testing.T) string {
 	t.Helper()
 
-	s := new(http.Server)
-	s.Addr = *serverBindAddr
-	s.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("Content-Type", "text/html")
 		fmt.Fprintf(w, "<html><body><span id=anubis-test>%d</span></body></html>", time.Now().Unix())
 	})
 
-	go func() {
-		if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			t.Logf("test HTTP server terminated unexpectedly: %v", err)
-		}
-	}()
+	policy, err := libanubis.LoadPoliciesOrDefault("", anubis.DefaultDifficulty)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := libanubis.New(libanubis.Options{
+		Next:           h,
+		Policy:         policy,
+		ServeRobotsTXT: true,
+	})
+	if err != nil {
+		t.Fatalf("can't construct libanubis.Server: %v", err)
+	}
+
+	ts := httptest.NewServer(s)
+	t.Log(ts.URL)
 
 	t.Cleanup(func() {
-		if err := s.Shutdown(context.Background()); err != nil {
-			t.Fatalf("could not shutdown test server: %v", err)
-		}
+		ts.Close()
 	})
+
+	return ts.URL
 }