fix(kt-devnet): skip fileserver deployment when possible (#14413)

This is a temporary measure, until we can integrate a proofs-only
fileserver with the op-challenger deployment.

But in the meantime, this unblocks a decent amount of idempotent
kurtosis deployments, by avoiding kurtosis invalidating the execution
cache needlessly.

Author: sigma

Diff for: kurtosis-devnet/fileserver/main.star

@@ -12,18 +12,22 @@ def get_used_ports():
     return used_ports
 
 
-def run(plan, source_path):
+def run(plan, source_path, server_image=FILESERVER_IMAGE):
     service_name = "fileserver"
     config = get_fileserver_config(
-        plan,
-        service_name,
-        source_path,
+        plan = plan,
+        service_name = service_name,
+        source_path = source_path,
+        server_image = server_image,
+    )
+    plan.add_service(
+        name = service_name,
+        config = config,
     )
-    service = plan.add_service(service_name, config)
     return service_name
 
 
-def get_fileserver_config(plan, service_name, source_path):
+def get_fileserver_config(plan, service_name, source_path, server_image):
     files = {}
 
     # Upload content to container
@@ -42,7 +46,7 @@ def get_fileserver_config(plan, service_name, source_path):
 
     ports = get_used_ports()
     return ServiceConfig(
-        image=FILESERVER_IMAGE,
+        image=server_image,
         ports=ports,
         cmd=["nginx", "-g", "daemon off;"],
         files=files,

Diff for: kurtosis-devnet/pkg/deploy/deploy.go

@@ -240,12 +240,14 @@ func (d *Deployer) Deploy(ctx context.Context, r io.Reader) (*kurtosis.KurtosisE
 		deployer: d.ktDeployer,
 	}
 
+	ch := srv.getState(ctx)
+
 	buf, err := d.renderTemplate(tmpDir, srv.URL)
 	if err != nil {
 		return nil, fmt.Errorf("error rendering template: %w", err)
 	}
 
-	if err := srv.Deploy(ctx, tmpDir); err != nil {
+	if err := srv.Deploy(ctx, tmpDir, ch); err != nil {
 		return nil, fmt.Errorf("error deploying fileserver: %w", err)
 	}
 

Diff for: kurtosis-devnet/pkg/deploy/fileserver.go

@@ -3,11 +3,17 @@ package deploy
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
+	ktfs "github.com/ethereum-optimism/optimism/devnet-sdk/kt/fs"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/util"
 )
@@ -25,17 +31,43 @@ func (f *FileServer) URL(path ...string) string {
 	return fmt.Sprintf("http://%s/%s", FILESERVER_PACKAGE, strings.Join(path, "/"))
 }
 
-func (f *FileServer) Deploy(ctx context.Context, sourceDir string) error {
+func (f *FileServer) Deploy(ctx context.Context, sourceDir string, stateCh <-chan *fileserverState) error {
+	// Check if source directory is empty. If it is, then ie means we don't have
+	// anything to serve, so we might as well not deploy the fileserver.
+	entries, err := os.ReadDir(sourceDir)
+	if err != nil {
+		return fmt.Errorf("error reading source directory: %w", err)
+	}
+	if len(entries) == 0 {
+		return nil
+	}
+
+	srcHash, err := calculateDirHash(sourceDir)
+	if err != nil {
+		return fmt.Errorf("error calculating source directory hash: %w", err)
+	}
 	// Create a temp dir in the fileserver package
 	baseDir := filepath.Join(f.baseDir, FILESERVER_PACKAGE)
 	if err := os.MkdirAll(baseDir, 0755); err != nil {
 		return fmt.Errorf("error creating base directory: %w", err)
 	}
+
+	configHash, err := calculateDirHash(filepath.Join(baseDir, "static_files", "nginx"))
+	if err != nil {
+		return fmt.Errorf("error calculating base directory hash: %w", err)
+	}
+
+	refState := <-stateCh
+	if refState.contentHash == srcHash && refState.configHash == configHash {
+		log.Println("No changes to fileserver, skipping deployment")
+		return nil
+	}
+
 	// Can't use MkdirTemp here because the directory name needs to always be the same
 	// in order for kurtosis file artifact upload to be idempotent.
 	// (i.e. the file upload and all its downstream dependencies can be SKIPPED on re-runs)
 	tempDir := filepath.Join(baseDir, "upload-content")
-	err := os.Mkdir(tempDir, 0755)
+	err = os.Mkdir(tempDir, 0755)
 	if err != nil {
 		return fmt.Errorf("error creating temporary directory: %w", err)
 	}
@@ -68,3 +100,145 @@ func (f *FileServer) Deploy(ctx context.Context, sourceDir string) error {
 
 	return nil
 }
+
+type fileserverState struct {
+	contentHash string
+	configHash  string
+}
+
+// downloadAndHashArtifact downloads an artifact and calculates its hash
+func downloadAndHashArtifact(ctx context.Context, enclave, artifactName string) (string, error) {
+	fs, err := ktfs.NewEnclaveFS(ctx, enclave)
+	if err != nil {
+		return "", fmt.Errorf("failed to create enclave fs: %w", err)
+	}
+
+	// Create temp dir
+	tempDir, err := os.MkdirTemp("", artifactName+"-*")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp dir: %w", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	// Download artifact
+	artifact, err := fs.GetArtifact(ctx, artifactName)
+	if err != nil {
+		return "", fmt.Errorf("failed to get artifact: %w", err)
+	}
+
+	// Ensure parent directories exist before extracting
+	if err := os.MkdirAll(tempDir, 0755); err != nil {
+		return "", fmt.Errorf("failed to create temp dir structure: %w", err)
+	}
+
+	// Extract to temp dir
+	if err := artifact.Download(tempDir); err != nil {
+		return "", fmt.Errorf("failed to download artifact: %w", err)
+	}
+
+	// Calculate hash
+	hash, err := calculateDirHash(tempDir)
+	if err != nil {
+		return "", fmt.Errorf("failed to calculate hash: %w", err)
+	}
+
+	return hash, nil
+}
+
+func (f *FileServer) getState(ctx context.Context) <-chan *fileserverState {
+	stateCh := make(chan *fileserverState)
+
+	go func(ctx context.Context) {
+		st := &fileserverState{}
+		var wg sync.WaitGroup
+
+		type artifactInfo struct {
+			name string
+			dest *string
+		}
+
+		artifacts := []artifactInfo{
+			{"fileserver-content", &st.contentHash},
+			{"fileserver-nginx-conf", &st.configHash},
+		}
+
+		for _, art := range artifacts {
+			wg.Add(1)
+			go func(art artifactInfo) {
+				defer wg.Done()
+				hash, err := downloadAndHashArtifact(ctx, f.enclave, art.name)
+				if err == nil {
+					*art.dest = hash
+				}
+			}(art)
+		}
+
+		wg.Wait()
+		stateCh <- st
+	}(ctx)
+
+	return stateCh
+}
+
+type entry struct {
+	RelPath string `json:"rel_path"`
+	Size    int64  `json:"size"`
+	Mode    string `json:"mode"`
+	Content []byte `json:"content"`
+}
+
+// calculateDirHash returns a SHA256 hash of the directory contents
+// It walks through the directory, hashing file names and contents
+func calculateDirHash(dir string) (string, error) {
+	hash := sha256.New()
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Get path relative to root dir
+		relPath, err := filepath.Rel(dir, path)
+		if err != nil {
+			return err
+		}
+
+		// Skip the root directory
+		if relPath == "." {
+			return nil
+		}
+
+		// Add the relative path and file info to hash
+		entry := entry{
+			RelPath: relPath,
+			Size:    info.Size(),
+			Mode:    info.Mode().String(),
+		}
+
+		// If it's a regular file, add its contents to hash
+		if !info.IsDir() {
+			content, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			entry.Content = content
+		}
+
+		jsonBytes, err := json.Marshal(entry)
+		if err != nil {
+			return err
+		}
+		_, err = hash.Write(jsonBytes)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("error walking directory: %w", err)
+	}
+
+	hashStr := hex.EncodeToString(hash.Sum(nil))
+	return hashStr, nil
+}

Diff for: kurtosis-devnet/pkg/deploy/fileserver_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis/sources/spec"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -20,36 +21,111 @@ func TestDeployFileserver(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(tmpDir)
 
+	// Create test files
+	sourceDir := filepath.Join(tmpDir, "fileserver")
+	require.NoError(t, os.MkdirAll(sourceDir, 0755))
+
+	// Create required directory structure
+	nginxDir := filepath.Join(sourceDir, "static_files", "nginx")
+	require.NoError(t, os.MkdirAll(nginxDir, 0755))
+
 	// Create a mock deployer function
 	mockDeployerFunc := func(opts ...kurtosis.KurtosisDeployerOptions) (deployer, error) {
 		return &mockDeployer{}, nil
 	}
 
 	testCases := []struct {
-		name        string
-		fs          *FileServer
-		shouldError bool
+		name         string
+		setup        func(t *testing.T, sourceDir, nginxDir string, state *fileserverState)
+		state        *fileserverState
+		shouldError  bool
+		shouldDeploy bool
 	}{
 		{
-			name: "successful deployment",
-			fs: &FileServer{
-				baseDir:  tmpDir,
-				enclave:  "test-enclave",
-				dryRun:   true,
-				deployer: mockDeployerFunc,
+			name: "empty source directory - no deployment needed",
+			setup: func(t *testing.T, sourceDir, nginxDir string, state *fileserverState) {
+				// No files to create
 			},
-			shouldError: false,
+			state:        &fileserverState{},
+			shouldError:  false,
+			shouldDeploy: false,
+		},
+		{
+			name: "new files to deploy",
+			setup: func(t *testing.T, sourceDir, nginxDir string, state *fileserverState) {
+				require.NoError(t, os.WriteFile(
+					filepath.Join(sourceDir, "test.txt"),
+					[]byte("test content"),
+					0644,
+				))
+			},
+			state:        &fileserverState{},
+			shouldError:  false,
+			shouldDeploy: true,
+		},
+		{
+			name: "no changes - deployment skipped",
+			setup: func(t *testing.T, sourceDir, nginxDir string, state *fileserverState) {
+				require.NoError(t, os.WriteFile(
+					filepath.Join(sourceDir, "test.txt"),
+					[]byte("test content"),
+					0644,
+				))
+
+				// Calculate actual hash for the test file
+				hash, err := calculateDirHash(sourceDir)
+				require.NoError(t, err)
+
+				// Calculate nginx config hash
+				configHash, err := calculateDirHash(nginxDir)
+				require.NoError(t, err)
+
+				// Update state with actual hashes
+				state.contentHash = hash
+				state.configHash = configHash
+			},
+			state:        &fileserverState{},
+			shouldError:  false,
+			shouldDeploy: false,
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.fs.Deploy(ctx, filepath.Join(tmpDir, "fileserver"))
+			// Clean up and recreate source directory for each test
+			require.NoError(t, os.RemoveAll(sourceDir))
+			require.NoError(t, os.MkdirAll(sourceDir, 0755))
+
+			// Recreate nginx directory
+			require.NoError(t, os.MkdirAll(nginxDir, 0755))
+
+			// Setup test files
+			tc.setup(t, sourceDir, nginxDir, tc.state)
+
+			fs := &FileServer{
+				baseDir:  tmpDir,
+				enclave:  "test-enclave",
+				dryRun:   true,
+				deployer: mockDeployerFunc,
+			}
+
+			// Create state channel and send test state
+			ch := make(chan *fileserverState, 1)
+			ch <- tc.state
+			close(ch)
+
+			err := fs.Deploy(ctx, sourceDir, ch)
 			if tc.shouldError {
 				require.Error(t, err)
 			} else {
 				require.NoError(t, err)
 			}
+
+			// Verify deployment directory was created only if deployment was needed
+			deployDir := filepath.Join(tmpDir, FILESERVER_PACKAGE)
+			if tc.shouldDeploy {
+				assert.DirExists(t, deployDir)
+			}
 		})
 	}
 }