add support command for automated firmware update (#55)

Author: umezawatakeshi

.github/workflows/main.yaml

@@ -25,6 +25,7 @@ jobs:
             /tmp/setup-hw
             /tmp/monitor-hw
             /tmp/collector
+            /tmp/setup-apply-firmware
           retention-days: 1
   build_image:
     name: Build images
@@ -40,6 +41,7 @@ jobs:
           cp setup-hw docker
           cp monitor-hw docker
           cp collector docker
+          cp setup-apply-firmware docker
       - run: |
           docker build -t quay.io/cybozu/setup-hw:latest --target stage1 docker
           docker build -t quay.io/cybozu/setup-hw-secret:latest docker

Makefile

@@ -1,4 +1,10 @@
-BIN_PKGS = ./pkg/idrac-passwd-hash ./pkg/setup-hw ./pkg/monitor-hw ./pkg/collector
+# binaries to be included in the image
+BINS_IMAGE = setup-hw monitor-hw collector setup-apply-firmware
+
+# binaries not to be included in the image
+BINS_NOIMAGE = idrac-passwd-hash
+
+BINS = $(BINS_IMAGE) $(BINS_NOIMAGE)
 GENERATED = redfish/rendered_rules.go
 GENERATE_SRC = $(shell find redfish/rules)
 
@@ -29,14 +35,14 @@ install: generate
 ifdef GOBIN
 	mkdir -p $(GOBIN)
 endif
-	GOBIN=$(GOBIN) go install $(BIN_PKGS)
+	GOBIN=$(GOBIN) go install $(foreach f, $(BINS), ./pkg/$(f))
 
 build-image: install
 ifdef GOBIN
 	mkdir -p $(GOBIN)
-	cp $(GOBIN)/setup-hw $(GOBIN)/monitor-hw $(GOBIN)/collector ./docker/
+	cp $(foreach f, $(BINS_IMAGE), $(GOBIN)/$(f)) ./docker/
 else
-	cp $(GOPATH)/bin/setup-hw $(GOPATH)/bin/monitor-hw $(GOPATH)/bin/collector ./docker/
+	cp $(foreach f, $(BINS_IMAGE), $(GOPATH)/bin/$(f)) ./docker/
 endif
 	cd docker && docker build -t quay.io/cybozu/setup-hw:dev .
 

docker/.gitignore

@@ -1,3 +1,4 @@
 setup-hw
 monitor-hw
 collector
+setup-apply-firmware

docker/Dockerfile

@@ -3,6 +3,7 @@ FROM centos:7 as stage1
 COPY setup-hw /usr/local/bin/setup-hw
 COPY monitor-hw /usr/local/sbin/monitor-hw
 COPY collector /usr/local/sbin/collector
+COPY setup-apply-firmware /usr/local/sbin/setup-apply-firmware
 
 CMD ["/usr/local/sbin/monitor-hw"]
 

docs/setup-apply-firmware.md

@@ -0,0 +1,26 @@
+Automated Firmware Update Support Tool
+======================================
+
+`setup-apply-firmware` is a tool to configure BMC to apply firmware update.
+
+Synopsis
+--------
+
+```console
+$ setup-apply-firmware UPDATER_URL...
+```
+
+`UPDATER_URL` should be those supported by `curl`.
+
+Description
+-----------
+
+`setup-apply-firmware` is a tool to configure BMC to apply firmware update.
+
+It downloads the firmware updaters specified by `UPDATER_URL`s and sends them to BMC.
+
+Caveat
+------
+
+- The order to send each updater is indefinite.
+- This tool does not initiate reboot.

pkg/setup-apply-firmware/dell.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/cybozu-go/log"
+	"github.com/cybozu-go/well"
+)
+
+func setupDell(ctx context.Context, files []string) error {
+	for _, f := range files {
+		cmd := well.CommandContext(ctx, "/opt/dell/srvadmin/bin/idracadm7", "update", "-f", f)
+		buf := bytes.Buffer{}
+		cmd.Stdout = &buf
+		cmd.Stderr = &buf
+		err := cmd.Run()
+		// we cannot use exit status to detect errors because `idracadm7 update` returns nonzero status even in case of successful update initiation.
+		var exitError *exec.ExitError
+		if err != nil && !errors.As(err, &exitError) {
+			return fmt.Errorf("racadm update failed at file %s: %w", f, err)
+		}
+		msg := buf.String()
+		if err = checkRacadmOutput(msg, f); err != nil {
+			return err
+		}
+		log.Info("racadm update succeeded", map[string]interface{}{
+			"file": f,
+		})
+
+		// if the next `idracadm7 update` is executed immediately after the previous one, it will fail.
+		time.Sleep(time.Second * 10)
+	}
+
+	return nil
+}
+
+func checkRacadmOutput(msg, f string) error {
+	if !strings.Contains(msg, "\nRAC987: ") {
+		return fmt.Errorf("racadm update failed at file %s: msg: %s", f, msg)
+	}
+	return nil
+}

pkg/setup-apply-firmware/dell_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestCheckRacadmOutput(t *testing.T) {
+	t.Parallel()
+
+	err := checkRacadmOutput(`Applying...
+RAC987: Update initiated.
+`, "filename.dat")
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = checkRacadmOutput(`ERROR: Invalid File Type
+`, "some_file_name.dat")
+	if err == nil || err.Error() != "racadm update failed at file some_file_name.dat: msg: ERROR: Invalid File Type\n" {
+		t.Fatal(err)
+	}
+
+	err = checkRacadmOutput(``, "some_file_name.dat")
+	if err == nil || err.Error() != "racadm update failed at file some_file_name.dat: msg: " {
+		t.Fatal(err)
+	}
+}

pkg/setup-apply-firmware/download.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/cybozu-go/well"
+)
+
+func downloadUpdaters(ctx context.Context, urls, files []string) error {
+	if len(urls) != len(files) {
+		return fmt.Errorf("length of urls and files are defferent")
+	}
+
+	for i, u := range urls {
+		f := files[i]
+		cmd := well.CommandContext(ctx, "/usr/bin/curl", "-sSfo", f, u)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err := cmd.Run()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/setup-apply-firmware/main.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/cybozu-go/log"
+	"github.com/cybozu-go/setup-hw/lib"
+	"github.com/cybozu-go/well"
+)
+
+func main() {
+	well.LogConfig{}.Apply()
+	ctx := context.Background()
+
+	vendor, err := lib.DetectVendor()
+	if err != nil {
+		log.ErrorExit(err)
+	}
+
+	var setup func(context.Context, []string) error
+	switch vendor {
+	case lib.QEMU:
+		setup = setupQEMU
+	case lib.Dell:
+		setup = setupDell
+	default:
+		log.ErrorExit(errors.New("unsupported vendor hardware"))
+	}
+
+	tmpdir, err := os.MkdirTemp("/tmp", "setup-apply-firmware-")
+	if err != nil {
+		log.ErrorExit(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	urls := os.Args[1:]
+	files := make([]string, len(urls))
+	for i, u := range urls {
+		parsedUrl, err := url.Parse(u)
+		if err != nil {
+			log.ErrorExit(err)
+		}
+		f := path.Base(parsedUrl.Path)
+		files[i] = filepath.Join(tmpdir, f)
+	}
+
+	err = downloadUpdaters(ctx, urls, files)
+	if err != nil {
+		log.ErrorExit(err)
+	}
+
+	err = setup(ctx, files)
+	if err != nil {
+		log.ErrorExit(err)
+	}
+}

pkg/setup-apply-firmware/qemu.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+)
+
+func setupQEMU(ctx context.Context, files []string) error {
+	// nothing to do... just check file existence
+	for _, f := range files {
+		info, err := os.Stat(f)
+		if err != nil {
+			return err
+		}
+		if !info.Mode().IsRegular() {
+			return fmt.Errorf("file %s is not a regular file", f)
+		}
+	}
+
+	return nil
+}