Timeout directory opening

Author: melgenek

pkg/drivers/fdb/fdb.go

@@ -81,28 +81,25 @@ func (f *FDB) Start(ctx context.Context) error {
 	}
 	f.db = db
 
-	etcd, err := directory.CreateOrOpen(db, []string{f.dirName}, nil)
-	if err != nil {
+	if err = f.openDirectory(); err != nil {
 		return err
 	}
-	f.dir = etcd
 
 	if CleanDirOnStart {
-		_, err = db.Transact(func(tr fdb.Transaction) (ret interface{}, e error) {
-			tr.ClearRange(etcd)
-			return
+		_, err = transact(db, 0, func(tr fdb.Transaction) (interface{}, error) {
+			tr.ClearRange(f.dir)
+			return 0, nil
 		})
+		if err != nil {
+			return err
+		}
 	}
 
-	if err != nil {
-		return err
-	}
-
-	f.byRevision = CreateByRevisionSubspace(etcd)
-	f.byKeyAndRevision = CreateByKeyRevisionSubspace(etcd)
-	f.watch = CreateWatchSubspace(etcd)
-	f.compactRev = CreateCompactRevisionSubspace(etcd)
-	f.rev = CreateRevisionSubspace(etcd)
+	f.byRevision = CreateByRevisionSubspace(f.dir)
+	f.byKeyAndRevision = CreateByKeyRevisionSubspace(f.dir)
+	f.watch = CreateWatchSubspace(f.dir)
+	f.compactRev = CreateCompactRevisionSubspace(f.dir)
+	f.rev = CreateRevisionSubspace(f.dir)
 
 	// https://github.com/kubernetes/kubernetes/blob/442a69c3bdf6fe8e525b05887e57d89db1e2f3a5/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go#L97
 	if !APITest {
@@ -118,6 +115,28 @@ func (f *FDB) Start(ctx context.Context) error {
 	return nil
 }
 
+// The FDB operations intentionally retry forever and there is no explicit way to timeout directory opening.
+// https://forums.foundationdb.org/t/golang-fdb-mustopendefault-does-not-fail-when-fdb-cluster-content-points-to-invalid-host/715/2
+// Wrapping directory opening in a goroutine.
+func (f *FDB) openDirectory() error {
+	errCh := make(chan error, 1)
+	go func() {
+		dir, err := directory.CreateOrOpen(f.db, []string{f.dirName}, nil)
+		f.dir = dir
+		errCh <- err
+	}()
+
+	select {
+	case err := <-errCh:
+		if err != nil {
+			return err
+		}
+	case <-time.After(transactionTimeout):
+		return errors.New("directory creation timed out")
+	}
+	return nil
+}
+
 // https://apple.github.io/foundationdb/tls.html#configuring-tls
 func (f *FDB) setTLSConfig() error {
 	if f.tlsConfig.CertFile != "" {

pkg/drivers/fdb/fdb_test.go

@@ -517,6 +517,17 @@ func TestExceedSizeLarge(t *testing.T) {
 	require.ErrorIs(t, err, rpctypes.ErrRequestTooLarge)
 }
 
+func TestFailUnavailableServer(t *testing.T) {
+	transactionTimeout = 1 * time.Second
+
+	f := NewFdbStructured("any:any@1.2.3.4:9999", tls.Config{}, "dir1")
+	ctx, cancelCtx := context.WithTimeout(context.Background(), time.Duration(3)*time.Second)
+	defer cancelCtx()
+
+	err := f.Start(ctx)
+	require.Error(t, err)
+}
+
 func createRecords(t *testing.T, f server.Backend, ctx context.Context, recordCount int, recordSize int) map[string][]byte {
 	g := errgroup.Group{}
 	g.SetLimit(50)

pkg/drivers/fdb/fdb_util.go

@@ -13,10 +13,12 @@ const (
 	notCommittedErrorCode = 1020 // Transaction not committed due to conflict with another transaction
 
 	splitRangeAfterDuration  = 1 * time.Second
-	transactionTimeout       = 10 * time.Second
 	transactionMaxRetryCount = 1000
 )
 
+// var for testing
+var transactionTimeout = 10 * time.Second
+
 var forceRetryTransaction = func(i int) bool { return false }
 
 type Processor[T any] interface {
@@ -60,9 +62,6 @@ func processBatch(db fdb.Database, selector fdb.SelectorRange, collector Process
 
 	res, err := transact(db, batchResult{}, func(tr fdb.Transaction) (batchResult, error) {
 		res := batchResult{collectorNeedsMore: true}
-		if err := tr.Options().SetTimeout(transactionTimeout.Milliseconds()); err != nil {
-			return res, fmt.Errorf("failed to set timeout limit: %w", err)
-		}
 
 		start := time.Now()
 		// Snapshot read does not add read conflict ranges
@@ -104,6 +103,10 @@ func transact[T any](d fdb.Database, defaultValue T, f func(fdb.Transaction) (T,
 	wrapped := func() (T, error) {
 		defer panicToError(&e)
 
+		if err := tr.Options().SetTimeout(transactionTimeout.Milliseconds()); err != nil {
+			return defaultValue, fmt.Errorf("failed to set timeout limit: %w", err)
+		}
+
 		// https://forums.foundationdb.org/t/defaults-for-transaction-timeouts-and-retries/315/2
 		e = tr.Options().SetRetryLimit(transactionMaxRetryCount)
 		if e != nil {