fix: host compat with ovs in container (#23944)

Co-authored-by: Qiu Jian <[email protected]>

Author: swordqiu

pkg/hostman/hostinfo/hostbridge/ovs.go

@@ -17,6 +17,7 @@ package hostbridge
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
@@ -281,8 +282,29 @@ func (o *SOVSBridgeDriver) WarmupConfig() error {
 
 func OVSPrepare() error {
 	ovs := system_service.GetService("openvswitch")
-	if !ovs.IsInstalled() {
-		return fmt.Errorf("Service openvswitch not installed!")
+	if !ovs.IsInstalled() || !ovs.IsActive() {
+		// no openvswitch service found, first try load openvswitch kernel modules, then try ovs-vsctl command, if success, return nil
+		err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", "openvswitch").Run()
+		if err != nil {
+			return errors.Wrap(err, "Failed to load openvswitch kernel modules")
+		}
+		// wait for the ovs-vswitchd to start
+		startProbe := time.Now()
+		const waitSeconds = 60 * 2 // wait ovs-vswitch running for 2 minutes
+		for time.Since(startProbe) < time.Duration(waitSeconds)*time.Second {
+			err := procutils.NewCommand("ovs-vsctl", "show").Run()
+			if err != nil {
+				time.Sleep(2 * time.Second)
+			} else {
+				return nil
+			}
+		}
+		// ovs service start timeout
+		if !ovs.IsInstalled() {
+			// ovs service not installed, return error
+			return fmt.Errorf("service openvswitch not installed and wait ovs service timeout, please check ovs service status")
+		}
+		// ovs service installed but not running, continue to start the service
 	}
 	if ovs.IsEnabled() {
 		err := ovs.Disable()

pkg/hostman/hostinfo/hostinfo.go

@@ -623,7 +623,6 @@ func (h *SHostInfo) detectHostInfo() error {
 	if err = h.GetNodeHugepages(); err != nil {
 		return errors.Wrap(err, "GetNodeHugepages")
 	}
-	system_service.Init()
 	if options.HostOptions.CheckSystemServices {
 		if err := h.checkSystemServices(); err != nil {
 			return err

pkg/hostman/system_service/system_service.go

@@ -17,6 +17,7 @@ package system_service
 import (
 	"fmt"
 	"strings"
+	"sync"
 
 	"yunion.io/x/log"
 
@@ -43,8 +44,15 @@ type ISystemService interface {
 type NewServiceFunc func()
 
 var serviceMap map[string]ISystemService
+var serviceMapLock *sync.Mutex = &sync.Mutex{}
 
-func Init() {
+func initServiceMap() {
+	serviceMapLock.Lock()
+	defer serviceMapLock.Unlock()
+
+	if serviceMap != nil {
+		return
+	}
 	serviceMap = map[string]ISystemService{
 		"ntpd":           NewNtpdService(),
 		"telegraf":       NewTelegrafService(),
@@ -60,6 +68,7 @@ func Init() {
 }
 
 func GetService(name string) ISystemService {
+	initServiceMap()
 	if service, ok := serviceMap[name]; ok {
 		return service
 	} else {

pkg/lbagent/haproxy.go

@@ -18,7 +18,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -65,9 +65,6 @@ func NewHaproxyHelper(opts *Options, lbagentId string) (*HaproxyHelper, error) {
 			return nil, fmt.Errorf("sysctl: %s", err)
 		}
 	}
-
-	system_service.Init()
-
 	return helper, nil
 }
 
@@ -199,7 +196,7 @@ func (h *HaproxyHelper) handleUseCorpusCmd(ctx context.Context, cmd *LbagentCmd)
 			if err == nil {
 				d := buf.Bytes()
 				p := filepath.Join(dir, "telegraf.conf")
-				err := ioutil.WriteFile(p, d, agentutils.FileModeFile)
+				err := os.WriteFile(p, d, agentutils.FileModeFile)
 				if err == nil {
 					err := h.reloadTelegraf(ctx, agentParams)
 					if err != nil {
@@ -590,16 +587,21 @@ func (h *HaproxyHelper) reloadKeepalived(ctx context.Context) error {
 		"--vrrp_pid", vrrpPidFile.Path,
 		"--checkers_pid", checkersPidFile.Path,
 		"--use-file", h.keepalivedConf(),
-		"-D",
-		"-d",
-		"-S",
-		"0",
+		"--log-detail",
+		"--no-syslog",
+		"--dump-conf",
+		"--log-console",
+		"--dont-fork",
+	}
+	err := h.runService(args)
+	if err != nil {
+		return errors.Wrapf(err, "run service %s", strings.Join(args, " "))
 	}
-	return h.runCmd(args)
+	return nil
 }
 
 func (h *HaproxyHelper) runCmd(args []string) error {
-	log.Debugf("run command %s", args)
+	log.Infof("run command %s", strings.Join(args, " "))
 
 	name := args[0]
 	args = args[1:]
@@ -619,6 +621,8 @@ func (h *HaproxyHelper) runCmd(args []string) error {
 }
 
 func (h *HaproxyHelper) startCmd(args []string) (*exec.Cmd, error) {
+	log.Infof("start command %s", strings.Join(args, " "))
+
 	name := args[0]
 	args = args[1:]
 	cmd := exec.Command(name, args...)
@@ -629,3 +633,51 @@ func (h *HaproxyHelper) startCmd(args []string) (*exec.Cmd, error) {
 	}
 	return cmd, nil
 }
+
+func (h *HaproxyHelper) runService(args []string) error {
+	log.Infof("run service %s", strings.Join(args, " "))
+
+	name := args[0]
+	args = args[1:]
+	cmd := exec.Command(name, args...)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		log.Errorf("service %s stdout pipe error: %s", cmd.String(), err)
+		return errors.Wrapf(err, "service %s stdout pipe error", cmd.String())
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		log.Errorf("service %s stderr pipe error: %s", cmd.String(), err)
+		return errors.Wrapf(err, "service %s stderr pipe error", cmd.String())
+	}
+	drain := func(out io.ReadCloser, isErr bool) {
+		defer out.Close()
+		buf := make([]byte, 1024)
+		for {
+			n, err := stdout.Read(buf)
+			if err != nil {
+				log.Errorf("read pipe error: %s", err)
+				return
+			}
+			if isErr {
+				log.Errorln(string(buf[:n]))
+			} else {
+				log.Infoln(string(buf[:n]))
+			}
+		}
+	}
+	err = cmd.Start()
+	if err != nil {
+		return errors.Wrapf(err, "start service %s", cmd.String())
+	}
+	go func(cmd *exec.Cmd) {
+		err := cmd.Wait()
+		if err != nil {
+			log.Errorf("service %s exited with error: %s", cmd.String(), err)
+		}
+	}(cmd)
+	go drain(stdout, false)
+	go drain(stderr, true)
+	return nil
+}

pkg/lbagent/hastate.go

@@ -38,6 +38,8 @@ const (
 	HA_STATE_SCRIPT_NAME    = "ha_state.sh"
 	HA_STATE_SCRIPT_CONTENT = `
 #!/bin/bash
+
+echo "keepalived notify $@" > /proc/1/fd/1
 echo "$@" >%s
 `
 	HA_STATE_FILENAME = "ha_state"

pkg/lbagent/models/haproxy.go

@@ -69,7 +69,7 @@ func (b *LoadbalancerCorpus) GenHaproxyConfigs(dir string, opts *AgentParams) (*
 				"",
 			}
 			s := strings.Join(lines, "\n")
-			err := ioutil.WriteFile(p, []byte(s), agentutils.FileModeFile)
+			err := os.WriteFile(p, []byte(s), agentutils.FileModeFile)
 			if err != nil {
 				return nil, fmt.Errorf("write 01-haproxy.cfg: %s", err)
 			}
@@ -82,7 +82,7 @@ func (b *LoadbalancerCorpus) GenHaproxyConfigs(dir string, opts *AgentParams) (*
 			d = append(d, []byte(lbcert.PrivateKey)...)
 			fn := fmt.Sprintf("%s.pem", lbcert.Id)
 			p := filepath.Join(certsBase, fn)
-			err := ioutil.WriteFile(p, d, agentutils.FileModeFileSensitive)
+			err := os.WriteFile(p, d, agentutils.FileModeFileSensitive)
 			if err != nil {
 				return nil, fmt.Errorf("write cert %s: %s", lbcert.Id, err)
 			}

pkg/lbagent/models/keepalived.go

@@ -16,7 +16,7 @@ package models
 
 import (
 	"bytes"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"text/template"
 
@@ -67,7 +67,7 @@ func (b *LoadbalancerCorpus) GenKeepalivedConfigs(dir string, opts *GenKeepalive
 		// write keepalived.conf
 		d := buf.Bytes()
 		p := filepath.Join(dir, "keepalived.conf")
-		err := ioutil.WriteFile(p, d, agentutils.FileModeFile)
+		err := os.WriteFile(p, d, agentutils.FileModeFile)
 		if err != nil {
 			return err
 		}

pkg/lbagent/service.go

@@ -28,6 +28,7 @@ import (
 
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/ovnutils"
 	"yunion.io/x/onecloud/pkg/util/procutils"
@@ -72,7 +73,11 @@ func StartService() {
 	}
 
 	if !opts.DisableLocalVpc {
-		err := ovnutils.InitOvn(opts.SOvnOptions)
+		err := hostbridge.OVSPrepare()
+		if err != nil {
+			log.Fatalf("ovs prepare fail: %s", err)
+		}
+		err = ovnutils.InitOvn(opts.SOvnOptions)
 		if err != nil {
 			log.Fatalf("ovn init fail: %s", err)
 		}

pkg/util/ovnutils/ovnutils.go

@@ -134,7 +134,6 @@ func InitOvn(opts SOvnOptions) (err error) {
 			err = panicVal.(error)
 		}
 	}()
-	system_service.Init()
 	mustPrepOvsdbConfig(opts)
 	configBridgeMtu(opts)
 	if _, ok := ovnContainerImageTag(); !ok {