Network: Get fresh copy of global config when retrying network startup (#14540)

This way a fresh copy of state can be retrieved on each iteration when
trying to start networks that couldn't be started on the first pass.

This helps in scenarios where the network couldn't be started due to
a problem with the server/cluster's global config that is rectified
and then the go routine running the retry mechanism needs to get an
updated copy of the global config before retrying starting the network.

Reorganises this function so that the scope of the initial setup
attempt's
state variable is limited.

Author: tomponline

lxd/api_cluster.go

@@ -423,7 +423,7 @@ func clusterPutBootstrap(d *Daemon, r *http.Request, req api.ClusterPut) respons
 		}
 
 		// Restart the networks (to pickup forkdns and the like).
-		err = networkStartup(s)
+		err = networkStartup(d.State)
 		if err != nil {
 			return err
 		}
@@ -849,7 +849,7 @@ func clusterPutJoin(d *Daemon, r *http.Request, req api.ClusterPut) response.Res
 
 		// Start up networks so any post-join changes can be applied now that we have a Node ID.
 		logger.Debug("Starting networks after cluster join")
-		err = networkStartup(s)
+		err = networkStartup(d.State)
 		if err != nil {
 			logger.Errorf("Failed starting networks: %v", err)
 		}
@@ -3471,7 +3471,7 @@ func restoreClusterMember(d *Daemon, r *http.Request) response.Response {
 		metadata := make(map[string]any)
 
 		// Restart the networks.
-		err = networkStartup(d.State())
+		err = networkStartup(d.State)
 		if err != nil {
 			return err
 		}

lxd/daemon.go

@@ -1653,7 +1653,7 @@ func (d *Daemon) init() error {
 	if !d.db.Cluster.LocalNodeIsEvacuated() {
 		logger.Infof("Initializing networks")
 
-		err = networkStartup(d.State())
+		err = networkStartup(d.State)
 		if err != nil {
 			return err
 		}

lxd/networks.go

@@ -1548,20 +1548,9 @@ func networkLeasesGet(d *Daemon, r *http.Request) response.Response {
 	return response.SyncResponse(true, leases)
 }
 
-func networkStartup(s *state.State) error {
+func networkStartup(stateFunc func() *state.State) error {
 	var err error
 
-	// Get a list of projects.
-	var projectNames []string
-
-	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
-		projectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx())
-		return err
-	})
-	if err != nil {
-		return fmt.Errorf("Failed to load projects: %w", err)
-	}
-
 	// Build a list of networks to initialise, keyed by project and network name.
 	const networkPriorityStandalone = 0 // Start networks not dependent on any other network first.
 	const networkPriorityPhysical = 1   // Start networks dependent on physical interfaces second.
@@ -1572,38 +1561,14 @@ func networkStartup(s *state.State) error {
 		networkPriorityLogical:    make(map[network.ProjectNetwork]struct{}),
 	}
 
-	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
-		for _, projectName := range projectNames {
-			networkNames, err := tx.GetCreatedNetworkNamesByProject(ctx, projectName)
-			if err != nil {
-				return fmt.Errorf("Failed to load networks for project %q: %w", projectName, err)
-			}
-
-			for _, networkName := range networkNames {
-				pn := network.ProjectNetwork{
-					ProjectName: projectName,
-					NetworkName: networkName,
-				}
-
-				// Assume all networks are networkPriorityStandalone initially.
-				initNetworks[networkPriorityStandalone][pn] = struct{}{}
-			}
-		}
-
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-
 	loadedNetworks := make(map[network.ProjectNetwork]network.Network)
 
-	initNetwork := func(n network.Network, priority int) error {
+	initNetwork := func(s *state.State, n network.Network, priority int) error {
 		err = n.Start()
 		if err != nil {
 			err = fmt.Errorf("Failed starting: %w", err)
 
-			_ = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
+			_ = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error {
 				return tx.UpsertWarningLocalNode(ctx, n.Project(), entity.TypeNetwork, int(n.ID()), warningtype.NetworkUnvailable, err.Error())
 			})
 
@@ -1625,7 +1590,7 @@ func networkStartup(s *state.State) error {
 		return nil
 	}
 
-	loadAndInitNetwork := func(pn network.ProjectNetwork, priority int, firstPass bool) error {
+	loadAndInitNetwork := func(s *state.State, pn network.ProjectNetwork, priority int, firstPass bool) error {
 		var err error
 		var n network.Network
 
@@ -1670,39 +1635,78 @@ func networkStartup(s *state.State) error {
 			return nil
 		}
 
-		err = initNetwork(n, priority)
+		err = initNetwork(s, n, priority)
 		if err != nil {
 			return err
 		}
 
 		return nil
 	}
 
-	// Try initializing networks in priority order.
-	for priority := range initNetworks {
-		for pn := range initNetworks[priority] {
-			err := loadAndInitNetwork(pn, priority, true)
+	remainingNetworksCount := func() int {
+		remainingNetworks := 0
+		for _, projectNetworks := range initNetworks {
+			remainingNetworks += len(projectNetworks)
+		}
+
+		return remainingNetworks
+	}
+
+	{
+		// Peform first pass to start networks.
+		// Local scope for state variable during initial pass of setting up networks.
+		s := stateFunc()
+		err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error {
+			projectNames, err := dbCluster.GetProjectNames(ctx, tx.Tx())
 			if err != nil {
-				logger.Error("Failed initializing network", logger.Ctx{"project": pn.ProjectName, "network": pn.NetworkName, "err": err})
+				return fmt.Errorf("Failed to load projects: %w", err)
+			}
 
-				continue
+			for _, projectName := range projectNames {
+				networkNames, err := tx.GetCreatedNetworkNamesByProject(ctx, projectName)
+				if err != nil {
+					return fmt.Errorf("Failed to load networks for project %q: %w", projectName, err)
+				}
+
+				for _, networkName := range networkNames {
+					pn := network.ProjectNetwork{
+						ProjectName: projectName,
+						NetworkName: networkName,
+					}
+
+					// Assume all networks are networkPriorityStandalone initially.
+					initNetworks[networkPriorityStandalone][pn] = struct{}{}
+				}
 			}
+
+			return nil
+		})
+		if err != nil {
+			return err
 		}
-	}
 
-	loadedNetworks = nil // Don't store loaded networks after first pass.
+		// Try initializing networks in priority order.
+		for priority := range initNetworks {
+			for pn := range initNetworks[priority] {
+				err := loadAndInitNetwork(s, pn, priority, true)
+				if err != nil {
+					logger.Error("Failed initializing network", logger.Ctx{"project": pn.ProjectName, "network": pn.NetworkName, "err": err})
 
-	remainingNetworks := 0
-	for _, networks := range initNetworks {
-		remainingNetworks += len(networks)
+					continue
+				}
+			}
+		}
+
+		loadedNetworks = nil // Don't store loaded networks after first pass.
 	}
 
 	// For any remaining networks that were not successfully initialised, we now start a go routine to
 	// periodically try to initialize them again in the background.
-	if remainingNetworks > 0 {
+	if remainingNetworksCount() > 0 {
 		go func() {
 			for {
 				t := time.NewTimer(time.Duration(time.Minute))
+				s := stateFunc() // Get fresh state in case global config has been updated.
 
 				select {
 				case <-s.ShutdownCtx.Done():
@@ -1716,7 +1720,7 @@ func networkStartup(s *state.State) error {
 					// Try initializing networks in priority order.
 					for priority := range initNetworks {
 						for pn := range initNetworks[priority] {
-							err := loadAndInitNetwork(pn, priority, false)
+							err := loadAndInitNetwork(s, pn, priority, false)
 							if err != nil {
 								logger.Error("Failed initializing network", logger.Ctx{"project": pn.ProjectName, "network": pn.NetworkName, "err": err})
 
@@ -1727,11 +1731,7 @@ func networkStartup(s *state.State) error {
 						}
 					}
 
-					remainingNetworks := 0
-					for _, networks := range initNetworks {
-						remainingNetworks += len(networks)
-					}
-
+					remainingNetworks := remainingNetworksCount()
 					if remainingNetworks <= 0 {
 						logger.Info("All networks initialized")
 					}