Merge pull request #3433 from ntnn/test-integration-2

Rework test/integration framework

Author: kcp-ci-bot

sdk/testing/server/config.go

@@ -16,7 +16,13 @@ limitations under the License.
 
 package server
 
-import "path/filepath"
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+)
 
 // Config qualify a kcp server to start
 //
@@ -33,6 +39,43 @@ type Config struct {
 	RunInProcess bool
 }
 
+func (c Config) KubeconfigPath() string {
+	return filepath.Join(c.DataDir, "admin.kubeconfig")
+}
+
+func (c Config) BuildArgs(t TestingT) ([]string, error) {
+	kcpListenPort, err := GetFreePort(t)
+	if err != nil {
+		return nil, err
+	}
+	etcdClientPort, err := GetFreePort(t)
+	if err != nil {
+		return nil, err
+	}
+	etcdPeerPort, err := GetFreePort(t)
+	if err != nil {
+		return nil, err
+	}
+
+	args := []string{
+		"--root-directory", c.DataDir,
+		"--secure-port=" + kcpListenPort,
+		"--embedded-etcd-client-port=" + etcdClientPort,
+		"--embedded-etcd-peer-port=" + etcdPeerPort,
+		"--embedded-etcd-wal-size-bytes=" + strconv.Itoa(5*1000), // 5KB
+		"--kubeconfig-path=" + c.KubeconfigPath(),
+		"--feature-gates=" + fmt.Sprintf("%s", utilfeature.DefaultFeatureGate),
+		"--audit-log-path", filepath.Join(c.ArtifactDir, "kcp.audit"),
+		"--v=4",
+	}
+
+	if c.BindAddress != "" {
+		args = append(args, "--bind-address="+c.BindAddress)
+	}
+
+	return append(args, c.Args...), nil
+}
+
 // Option a function that wish to modify a given kcp configuration.
 type Option func(*Config)
 

sdk/testing/server/external.go

@@ -18,12 +18,14 @@ package server
 
 import (
 	"bytes"
+	"context"
 	cryptorand "crypto/rand"
 	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -34,6 +36,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@@ -48,14 +51,14 @@ import (
 // the configuration can be loaded synchronously and no locking is
 // required to subsequently access it.
 func NewExternalKCPServer(name, kubeconfigPath string, shardKubeconfigPaths map[string]string, clientCADir string) (RunningServer, error) {
-	cfg, err := loadKubeConfig(kubeconfigPath, "base")
+	cfg, err := LoadKubeConfig(kubeconfigPath, "base")
 	if err != nil {
 		return nil, err
 	}
 
 	shardCfgs := map[string]clientcmd.ClientConfig{}
 	for shard, path := range shardKubeconfigPaths {
-		shardCfg, err := loadKubeConfig(path, "base")
+		shardCfg, err := LoadKubeConfig(path, "base")
 		if err != nil {
 			return nil, err
 		}
@@ -169,17 +172,19 @@ func (s *externalKCPServer) Stopped() bool {
 	return false
 }
 
+var ErrEmptyKubeConfig = fmt.Errorf("kubeconfig is empty")
+
 // LoadKubeConfig loads a kubeconfig from disk. This method is
 // intended to be common between fixture for servers whose lifecycle
 // is test-managed and fixture for servers whose lifecycle is managed
 // separately from a test run.
-func loadKubeConfig(kubeconfigPath, contextName string) (clientcmd.ClientConfig, error) {
+func LoadKubeConfig(kubeconfigPath, contextName string) (clientcmd.ClientConfig, error) {
 	fs, err := os.Stat(kubeconfigPath)
 	if err != nil {
 		return nil, err
 	}
 	if fs.Size() == 0 {
-		return nil, fmt.Errorf("%s points to an empty file", kubeconfigPath)
+		return nil, ErrEmptyKubeConfig
 	}
 
 	rawConfig, err := clientcmd.LoadFromFile(kubeconfigPath)
@@ -190,6 +195,29 @@ func loadKubeConfig(kubeconfigPath, contextName string) (clientcmd.ClientConfig,
 	return clientcmd.NewNonInteractiveClientConfig(*rawConfig, contextName, nil, nil), nil
 }
 
+// WaitLoadKubeConfig wraps LoadKubeConfig and waits until the context
+// is cancelled, two minutes have passed, or the kubeconfig file is
+// loaded without error.
+func WaitLoadKubeConfig(ctx context.Context, kubeconfigPath, contextName string) (clientcmd.ClientConfig, error) {
+	var config clientcmd.ClientConfig
+	if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 2*time.Minute, true,
+		func(ctx context.Context) (bool, error) {
+			loaded, err := LoadKubeConfig(kubeconfigPath, contextName)
+			if err != nil {
+				if os.IsNotExist(err) || errors.Is(ErrEmptyKubeConfig, err) {
+					return false, nil
+				}
+				return false, err
+			}
+			config = loaded
+			return true, nil
+		},
+	); err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
 // clientCAUserConfig returns a config based on a dynamically created client certificate.
 // The returned client CA is signed by "test/e2e/framework/client-ca.crt".
 func clientCAUserConfig(t TestingT, cfg *rest.Config, clientCAConfigDirectory, username string, groups ...string) *rest.Config {

sdk/testing/server/fixture.go

@@ -27,7 +27,6 @@ import (
 	"path"
 	"path/filepath"
 	"runtime/debug"
-	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -41,7 +40,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	kubernetesscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -190,37 +188,6 @@ func newKcpServer(t TestingT, cfg Config) (*kcpServer, error) {
 		return nil, fmt.Errorf("could not create data dir: %w", err)
 	}
 
-	kcpListenPort, err := GetFreePort(t)
-	if err != nil {
-		return nil, err
-	}
-	etcdClientPort, err := GetFreePort(t)
-	if err != nil {
-		return nil, err
-	}
-	etcdPeerPort, err := GetFreePort(t)
-	if err != nil {
-		return nil, err
-	}
-
-	args := []string{
-		"--root-directory", s.cfg.DataDir,
-		"--secure-port=" + kcpListenPort,
-		"--embedded-etcd-client-port=" + etcdClientPort,
-		"--embedded-etcd-peer-port=" + etcdPeerPort,
-		"--embedded-etcd-wal-size-bytes=" + strconv.Itoa(5*1000), // 5KB
-		"--kubeconfig-path=" + s.KubeconfigPath(),
-		"--feature-gates=" + fmt.Sprintf("%s", utilfeature.DefaultFeatureGate),
-		"--audit-log-path", filepath.Join(s.cfg.ArtifactDir, "kcp.audit"),
-		"--v=4",
-	}
-
-	if cfg.BindAddress != "" {
-		args = append(args, "--bind-address="+cfg.BindAddress)
-	}
-
-	s.cfg.Args = append(args, s.cfg.Args...)
-
 	return s, nil
 }
 
@@ -278,7 +245,11 @@ func (c *kcpServer) Run(t TestingT) error {
 			runner = func(ctx context.Context, t TestingT, cfg Config) (<-chan struct{}, error) {
 				t.Log("RunInProcessFunc is deprecated, please migrate to ContextRunInProcessFunc")
 				t.Log("RunInProcessFunc is deprecated, stopping the server will not work")
-				return RunInProcessFunc(t, cfg.DataDir, cfg.Args)
+				args, err := cfg.BuildArgs(t)
+				if err != nil {
+					return nil, err
+				}
+				return RunInProcessFunc(t, cfg.DataDir, args)
 			}
 		}
 	}
@@ -325,7 +296,12 @@ func (c *kcpServer) Stopped() bool {
 }
 
 func runExternal(ctx context.Context, t TestingT, cfg Config) (<-chan struct{}, error) {
-	commandLine := append(StartKcpCommand("KCP"), cfg.Args...)
+	args, err := cfg.BuildArgs(t)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build kcp args: %w", err)
+	}
+
+	commandLine := append(StartKcpCommand("KCP"), args...)
 
 	t.Logf("running: %v", strings.Join(commandLine, " "))
 
@@ -438,7 +414,7 @@ func (c *kcpServer) Name() string {
 
 // KubeconfigPath exposes the path of the kubeconfig file of this kcp server.
 func (c *kcpServer) KubeconfigPath() string {
-	return filepath.Join(c.cfg.DataDir, "admin.kubeconfig")
+	return c.cfg.KubeconfigPath()
 }
 
 // Config exposes a copy of the base client config for this server. Client-side throttling is disabled (QPS=-1).
@@ -515,35 +491,13 @@ func (c *kcpServer) RawConfig() (clientcmdapi.Config, error) {
 }
 
 func (c *kcpServer) loadCfg(ctx context.Context) error {
-	var lastError error
-	if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
-		if c.Stopped() {
-			return false, fmt.Errorf("failed to load admin kubeconfig: server has stopped")
-		}
-
-		config, err := loadKubeConfig(c.KubeconfigPath(), "base")
-		if err != nil {
-			// A missing file is likely caused by the server not
-			// having started up yet. Ignore these errors for the
-			// purposes of logging.
-			if !os.IsNotExist(err) {
-				lastError = err
-			}
-
-			return false, nil
-		}
-
-		c.lock.Lock()
-		c.clientCfg = config
-		c.lock.Unlock()
-
-		return true, nil
-	}); err != nil && lastError != nil {
-		return fmt.Errorf("failed to load admin kubeconfig: %w", lastError)
-	} else if err != nil {
-		// should never happen
-		return fmt.Errorf("failed to load admin kubeconfig: %w", err)
+	config, err := WaitLoadKubeConfig(ctx, c.KubeconfigPath(), "base")
+	if err != nil {
+		return err
 	}
+	c.lock.Lock()
+	c.clientCfg = config
+	c.lock.Unlock()
 	return nil
 }