Expand versioned E2E coverage

versioned/Makefile

@@ -1,4 +1,4 @@
-.PHONY: build test e2e testapp build-docker docker-push release
+.PHONY: build test e2e e2e-clean-results e2e-build e2e-run e2e-report testapp build-docker docker-push release
 
 VERSION ?= $(shell git describe --always)
 SET_LATEST ?= 0
@@ -33,8 +33,31 @@ testapp:
 	mkdir -p build
 	go build -o build/testapp ./e2e/testapp
 
-e2e: build testapp
-	docker compose -f e2e/docker-compose.yml up --build --abort-on-container-exit --exit-code-from tests
+e2e: e2e-clean-results e2e-build e2e-run e2e-report
+
+e2e-clean-results:
+	rm -rf build/test-results
+	mkdir -p build/test-results
+
+e2e-build:
+	docker compose -f e2e/docker-compose.yml build
+
+e2e-run:
+	docker compose -f e2e/docker-compose.yml up --no-build --abort-on-container-exit --exit-code-from tests
+	docker compose -f e2e/docker-compose.yml down -v
+	docker compose -f e2e/docker-compose.yml up --no-build -d oracle versiond
+	docker compose -f e2e/docker-compose.yml stop versiond
+	docker compose -f e2e/docker-compose.yml run --rm --no-deps -e REGISTER_STARTUP_VERSION=1 tests gotestsum --format testname --junitfile /app/test-results/reconcile-register-junit.xml -- -tags e2e ./e2e -v -timeout 300s -run '^TestRegisterStartupVersion$$'
+	docker compose -f e2e/docker-compose.yml start versiond
+	docker compose -f e2e/docker-compose.yml run --rm --no-deps -e EXPECT_INITIAL_V1=1 tests gotestsum --format testname --junitfile /app/test-results/reconcile-on-start-junit.xml -- -tags e2e ./e2e -v -timeout 300s -run '^TestStartupFromExistingOracleState$$'
+	docker compose -f e2e/docker-compose.yml down -v
+
+e2e-report:
+	uvx junitparser merge --glob 'build/test-results/*-junit.xml' build/test-results/e2e-junit.xml
+	@status=0; \
+	uvx junitparser verify build/test-results/e2e-junit.xml || status=$$?; \
+	uvx junit2html --summary-matrix build/test-results/e2e-junit.xml; \
+	exit $$status
 
 build-docker:
 	$(eval PLATFORM=linux/amd64)

versioned/e2e/Dockerfile.tests

@@ -3,6 +3,6 @@ WORKDIR /app
 COPY go.mod go.sum* ./
 RUN go mod download
 COPY . .
+RUN go install gotest.tools/gotestsum@v1.12.3
 RUN CGO_ENABLED=0 go build -o /app/build/testapp ./e2e/testapp
-RUN CGO_ENABLED=0 go test -c -tags e2e -o /e2e-tests ./e2e/
-CMD ["/e2e-tests", "-test.v", "-test.timeout", "300s"]
+CMD ["gotestsum", "--format", "testname", "--junitfile", "/app/test-results/run-junit.xml", "--", "-tags", "e2e", "./e2e", "-v", "-timeout", "300s", "-skip", "^(TestRegisterStartupVersion|TestStartupFromExistingOracleState)$"]

versioned/e2e/docker-compose.yml

@@ -1,5 +1,6 @@
 services:
   oracle:
+    image: versioned-e2e-oracle:latest
     build:
       context: ..
       dockerfile: e2e/Dockerfile.mockoracle
@@ -10,6 +11,7 @@ services:
       - oracle-data:/data
 
   versiond:
+    image: versioned-e2e-versiond:latest
     build:
       context: ..
       dockerfile: Dockerfile
@@ -23,13 +25,16 @@ services:
       - oracle
 
   tests:
+    image: versioned-e2e-tests:latest
     build:
       context: ..
       dockerfile: e2e/Dockerfile.tests
     environment:
       ORACLE_URL: http://oracle:8080
       VERSIOND_URL: http://versiond:8080
       TESTAPP_PATH: /app/build/testapp
+    volumes:
+      - ../build/test-results:/app/test-results
     depends_on:
       - oracle
       - versiond

versioned/e2e/e2e_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -34,6 +35,84 @@ func TestBasicFlow(t *testing.T) {
 	}
 }
 
+func TestChildProcessCrashRecovery(t *testing.T) {
+	zipData, hash := buildTestappZip(t)
+	version := "child-crash-recovery"
+
+	uploadBinary(t, version+".zip", zipData)
+	putVersion(t, version, fmt.Sprintf("%s/binaries/%s.zip", oracleURL, version), hash, 9005)
+	waitForVersion(t, version, 90*time.Second)
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/%s/exit", versiondURL, version), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("request child exit: %v", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		t.Fatalf("child exit status = %d, want 204", resp.StatusCode)
+	}
+
+	waitForVersionUnavailable(t, version, 10*time.Second)
+	waitForVersion(t, version, 90*time.Second)
+
+	var recovered map[string]string
+	getJSON(t, fmt.Sprintf("%s/%s/", versiondURL, version), &recovered)
+	if recovered["prefix"] != version {
+		t.Errorf("prefix = %q, want %q", recovered["prefix"], version)
+	}
+}
+
+func TestRegisterStartupVersion(t *testing.T) {
+	if os.Getenv("REGISTER_STARTUP_VERSION") == "" {
+		t.Skip("startup version registration is enabled only by the startup scenario")
+	}
+
+	zipData, hash := buildTestappZip(t)
+
+	uploadBinary(t, "startup-v1.zip", zipData)
+	putVersion(t, "v1", fmt.Sprintf("%s/binaries/startup-v1.zip", oracleURL), hash, 9001)
+}
+
+func TestStartupFromExistingOracleState(t *testing.T) {
+	if os.Getenv("EXPECT_INITIAL_V1") == "" {
+		t.Skip("startup oracle state scenario is enabled only after versiond restart")
+	}
+
+	waitForVersion(t, "v1", 90*time.Second)
+
+	var proxied map[string]string
+	getJSON(t, fmt.Sprintf("%s/v1/", versiondURL), &proxied)
+	if proxied["prefix"] != "v1" {
+		t.Errorf("prefix = %q, want %q", proxied["prefix"], "v1")
+	}
+
+	resp, err := http.Get(fmt.Sprintf("%s/healthz", versiondURL))
+	if err != nil {
+		t.Fatalf("GET healthz: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	var statuses []map[string]interface{}
+	if err := json.Unmarshal(body, &statuses); err != nil {
+		t.Fatalf("decode healthz: %v, body: %s", err, string(body))
+	}
+
+	for _, s := range statuses {
+		if s["name"] == "v1" {
+			if s["status"] != "running" {
+				t.Errorf("v1 status = %q, want running", s["status"])
+			}
+			return
+		}
+	}
+	t.Fatalf("v1 not found in healthz response: %s", string(body))
+}
+
 func TestAddVersion(t *testing.T) {
 	zipData, hash := buildTestappZip(t)
 
@@ -84,6 +163,71 @@ func TestRemoveVersion(t *testing.T) {
 	}
 }
 
+func TestOracleTemporaryFailureKeepsVersionsRunning(t *testing.T) {
+	setOracleFailure(t, false)
+	t.Cleanup(func() {
+		setOracleFailure(t, false)
+	})
+
+	zipData, hash := buildTestappZip(t)
+	version := "oracle-failure"
+
+	uploadBinary(t, version+".zip", zipData)
+	putVersion(t, version, fmt.Sprintf("%s/binaries/%s.zip", oracleURL, version), hash, 9004)
+	waitForVersion(t, version, 90*time.Second)
+
+	setOracleFailure(t, true)
+	waitForPollCycles(2)
+
+	var resp map[string]string
+	getJSON(t, fmt.Sprintf("%s/%s/", versiondURL, version), &resp)
+	if resp["prefix"] != version {
+		t.Errorf("prefix = %q, want %q", resp["prefix"], version)
+	}
+}
+
+func TestEmptyOracleResponseKeepsVersionsRunning(t *testing.T) {
+	setOracleFailure(t, false)
+	deleteAllVersions(t)
+
+	zipData, hash := buildTestappZip(t)
+
+	uploadBinary(t, "empty-oracle-v1.zip", zipData)
+	putVersion(t, "v1", fmt.Sprintf("%s/binaries/empty-oracle-v1.zip", oracleURL), hash, 9001)
+	waitForVersion(t, "v1", 90*time.Second)
+
+	deleteVersion(t, "v1")
+	waitForPollCycles(2)
+
+	var resp map[string]string
+	getJSON(t, fmt.Sprintf("%s/v1/", versiondURL), &resp)
+	if resp["prefix"] != "v1" {
+		t.Errorf("prefix = %q, want %q", resp["prefix"], "v1")
+	}
+}
+
+func TestFailedSameVersionUpdateKeepsOldChildRunning(t *testing.T) {
+	setOracleFailure(t, false)
+
+	zipData, hash := buildTestappZip(t)
+
+	uploadBinary(t, "failed-update-v1.zip", zipData)
+	putVersion(t, "v1", fmt.Sprintf("%s/binaries/failed-update-v1.zip", oracleURL), hash, 9001)
+	waitForVersion(t, "v1", 90*time.Second)
+
+	uploadBinary(t, "failed-update-v1-bad.zip", zipData)
+	putVersion(t, "v1", fmt.Sprintf("%s/binaries/failed-update-v1-bad.zip", oracleURL), "wrong_hash", 9001)
+
+	waitForPollCycles(2)
+
+	var resp map[string]string
+	getJSON(t, fmt.Sprintf("%s/v1/", versiondURL), &resp)
+	if resp["prefix"] != "v1" {
+		t.Errorf("prefix = %q, want %q", resp["prefix"], "v1")
+	}
+	assertHealthStatus(t, "v1", "running")
+}
+
 func TestHashMismatch(t *testing.T) {
 	zipData, _ := buildTestappZip(t)
 

versioned/e2e/mockoracle/main.go

@@ -27,6 +27,7 @@ type store struct {
 	mu       sync.RWMutex
 	versions []Version
 	binDir   string
+	fail     bool
 }
 
 func main() {
@@ -51,10 +52,30 @@ func main() {
 	}
 
 	http.HandleFunc("/versions", func(w http.ResponseWriter, r *http.Request) {
-		s.mu.RLock()
-		defer s.mu.RUnlock()
-		w.Header().Set("Content-Type", "application/json")
-		json.NewEncoder(w).Encode(VersionConfig{Versions: s.versions})
+		switch r.Method {
+		case http.MethodGet:
+			s.mu.RLock()
+			fail := s.fail
+			versions := append([]Version(nil), s.versions...)
+			s.mu.RUnlock()
+			if fail {
+				http.Error(w, "oracle failure enabled", http.StatusInternalServerError)
+				return
+			}
+			w.Header().Set("Content-Type", "application/json")
+			json.NewEncoder(w).Encode(VersionConfig{Versions: versions})
+		case http.MethodDelete:
+			if s.failureEnabled() {
+				http.Error(w, "oracle failure enabled", http.StatusInternalServerError)
+				return
+			}
+			s.mu.Lock()
+			s.versions = nil
+			s.mu.Unlock()
+			w.WriteHeader(http.StatusNoContent)
+		default:
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		}
 	})
 
 	http.HandleFunc("/versions/", func(w http.ResponseWriter, r *http.Request) {
@@ -65,6 +86,10 @@ func main() {
 		}
 		switch r.Method {
 		case http.MethodPut:
+			if s.failureEnabled() {
+				http.Error(w, "oracle failure enabled", http.StatusInternalServerError)
+				return
+			}
 			var v Version
 			if err := json.NewDecoder(r.Body).Decode(&v); err != nil {
 				http.Error(w, err.Error(), http.StatusBadRequest)
@@ -87,6 +112,10 @@ func main() {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(v)
 		case http.MethodDelete:
+			if s.failureEnabled() {
+				http.Error(w, "oracle failure enabled", http.StatusInternalServerError)
+				return
+			}
 			s.mu.Lock()
 			for i, v := range s.versions {
 				if v.Name == name {
@@ -101,6 +130,18 @@ func main() {
 		}
 	})
 
+	http.HandleFunc("/fail", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+		enabled := r.URL.Query().Get("enabled") == "true"
+		s.mu.Lock()
+		s.fail = enabled
+		s.mu.Unlock()
+		w.WriteHeader(http.StatusNoContent)
+	})
+
 	http.HandleFunc("/binaries/", func(w http.ResponseWriter, r *http.Request) {
 		name := strings.TrimPrefix(r.URL.Path, "/binaries/")
 		if strings.Contains(name, "..") {
@@ -129,3 +170,9 @@ func main() {
 	log.Printf("mock oracle listening on %s", addr)
 	log.Fatal(http.ListenAndServe(addr, nil))
 }
+
+func (s *store) failureEnabled() bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.fail
+}