Migrate to go-plugin system for provisioners

Signed-off-by: s3rj1k <evasive.gyron@gmail.com>

Author: s3rj1k

Dockerfile

@@ -1,28 +1,50 @@
 # Support FROM override
 ARG BUILD_IMAGE=docker.io/golang:1.25.9@sha256:5ab234a9519e05043f4a97a505a59f21dc40eee172d6b17d411863d6bba599bb
-ARG BASE_IMAGE=gcr.io/distroless/static:nonroot@sha256:9ecc53c269509f63c69a266168e4a687c7eb8c0cfd753bd8bfcaa4f58a90876f
+ARG BASE_IMAGE=gcr.io/distroless/base-debian13:nonroot@sha256:fb282f8ed3057f71dbfe3ea0f5fa7e961415dafe4761c23948a9d4628c6166fe
 
-# Build the manager binary
-FROM $BUILD_IMAGE AS builder
+# Shared SDK stage: pinned Go toolchain, modules, and checked-out tree.
+# Downstream builder stages copy from here so we pay the `go mod download`
+# cost once. Third parties can also build custom provisioner plugins against
+# this image to guarantee toolchain and module-version parity with BMO.
+FROM $BUILD_IMAGE AS sdk
 
 WORKDIR /workspace
 
-# Bring in the go dependencies before anything else so we can take
-# advantage of caching these layers in future builds.
 COPY go.mod go.sum ./
 COPY apis/go.mod apis/go.sum apis/
 COPY hack/tools/go.mod hack/tools/go.sum hack/tools/
 COPY pkg/hardwareutils/go.mod pkg/hardwareutils/go.sum pkg/hardwareutils/
 RUN go mod download
-ARG LDFLAGS=-s -w -extldflags=-static
 
 COPY . .
+
+ENV CGO_ENABLED=1
+ENV GO111MODULE=on
+
+# Build the manager binary
+FROM sdk AS builder
+ARG ARCH=amd64
+ARG LDFLAGS=-s -w
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=${ARCH} GO111MODULE=on \
+    go build -a -ldflags "${LDFLAGS}" -o baremetal-operator main.go
+
+# Build the ironic provisioner plugin
+FROM sdk AS ironic-plugin-builder
+ARG ARCH=amd64
+ARG LDFLAGS=-s -w
+RUN GOOS=linux GOARCH=${ARCH} \
+    go build -buildmode=plugin -ldflags "${LDFLAGS}" \
+    -o ironic-provisioner.so ./pkg/provisioner/ironic/plugin/
+
+# Build the demo provisioner plugin
+FROM sdk AS demo-plugin-builder
 ARG ARCH=amd64
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} GO111MODULE=on go build -a -ldflags "${LDFLAGS}" -o baremetal-operator main.go
+ARG LDFLAGS=-s -w
+RUN GOOS=linux GOARCH=${ARCH} \
+    go build -buildmode=plugin -ldflags "${LDFLAGS}" \
+    -o demo-provisioner.so ./pkg/provisioner/demo/plugin/
 
-# Copy the controller-manager into a thin image
-# BMO has a dependency preventing us to use the static one,
-# using the base one instead
+# Runtime image. Uses distroless/base (not static) because Go plugins need glibc.
 FROM $BASE_IMAGE
 
 # image.version is set during image build by automation
@@ -36,5 +58,7 @@ LABEL org.opencontainers.image.vendor="Metal3-io"
 
 WORKDIR /
 COPY --from=builder /workspace/baremetal-operator .
+COPY --from=ironic-plugin-builder /workspace/ironic-provisioner.so /plugins/ironic-provisioner.so
+COPY --from=demo-plugin-builder /workspace/demo-provisioner.so /plugins/demo-provisioner.so
 USER nonroot:nonroot
 ENTRYPOINT ["/baremetal-operator"]

Makefile

@@ -321,6 +321,32 @@ docker-debug: generate manifests ## Build the docker image with debug info
 	--build-arg LDFLAGS="-extldflags=-static" \
 	. -t ${IMG}-$(ARCH):${IMG_TAG}
 
+## --------------------------------------
+## Plugin / SDK Targets
+## --------------------------------------
+
+IRONIC_PLUGIN_DIR = pkg/provisioner/ironic/plugin
+IRONIC_PLUGIN_SO = bin/ironic-provisioner.so
+
+.PHONY: ironic-plugin
+ironic-plugin: ## Build the ironic provisioner plugin .so locally
+	CGO_ENABLED=1 go build -buildmode=plugin -ldflags "$(LDFLAGS)" -o $(IRONIC_PLUGIN_SO) ./$(IRONIC_PLUGIN_DIR)/
+
+.PHONY: docker-build-sdk
+docker-build-sdk: ## Build the BMO SDK image for authoring custom provisioner plugins
+	$(CONTAINER_RUNTIME) build --platform=linux/$(ARCH) \
+	--build-arg ARCH=$(ARCH) \
+	--build-arg http_proxy=$(http_proxy) \
+	--build-arg https_proxy=$(https_proxy) \
+	--target sdk \
+	. -t ${IMG}-sdk-$(ARCH):${IMG_TAG}
+
+.PHONY: docker-build-sdk-all
+docker-build-sdk-all: $(addprefix docker-build-sdk-,$(ALL_ARCH)) ## Build the SDK image for all architectures
+
+docker-build-sdk-%:
+	$(MAKE) ARCH=$* docker-build-sdk
+
 ## --------------------------------------
 ## Docker — All ARCH
 ## --------------------------------------

README.md

@@ -32,6 +32,7 @@ components, see the [Metal³ docs](https://github.com/metal3-io/metal3-docs).
 - [Setup Development Environment](docs/dev-setup.md)
 - [Configuration](docs/configuration.md)
 - [Testing](docs/testing.md)
+- [Provisioner plugins](docs/plugin-provisioners.md)
 
 ## Integration tests
 

docs/plugin-provisioners.md

@@ -0,0 +1,73 @@
+# Provisioner plugins
+
+BMO loads its provisioner at runtime from a Go plugin `.so`. The manager binary
+is decoupled from provisioner-specific dependencies (e.g. the Ironic client).
+
+## What ships in the BMO image
+
+The container image bakes two plugins under `/plugins/`:
+
+| Path                                 | Provisioner      |
+|--------------------------------------|------------------|
+| `/plugins/ironic-provisioner.so`     | ironic (default) |
+| `/plugins/demo-provisioner.so`       | demo             |
+
+The `fixture` provisioner is compiled into the manager and selected with
+`--test-mode` for tests and offline runs.
+
+## Selecting a plugin
+
+```text
+--provisioner-plugin=/path/to/custom.so
+```
+
+Also reads `$PROVISIONER_PLUGIN`. Defaults to the ironic path when unset.
+
+## Writing a custom plugin
+
+A plugin is `package main` exporting two symbols: `PluginName() string` and
+`NewProvisionerFactory(provisioner.PluginConfig) (provisioner.Factory, error)`.
+See [`pkg/provisioner/demo/plugin/main.go`](../pkg/provisioner/demo/plugin/main.go)
+for a minimal reference.
+
+Build:
+
+```sh
+CGO_ENABLED=1 go build -buildmode=plugin -o myprov.so ./path/to/plugin
+```
+
+## Toolchain lock
+
+Plugins must be built with the **same Go toolchain and same build flags** as
+the BMO binary they load into, and every module shared with the host (directly
+or transitively, including stdlib) must resolve to the **exact same version**.
+Deps unique to the plugin can be added freely because they don't participate
+in the version check. Mismatches produce:
+
+```text
+plugin was built with a different version of package <pkg>
+```
+
+For an in-tree plugin you get this for free, since the root `go.mod` is shared.
+
+For an out-of-tree plugin maintained in its own repo:
+
+- Pin BMO in the plugin's `go.mod` at the release you target. Go's module
+  resolution will then pick versions of shared deps (`k8s.io/*`,
+  `controller-runtime`, `go-logr`, etc.) compatible with BMO's closure:
+
+  ```sh
+  go get github.com/metal3-io/baremetal-operator@vX.Y.Z
+  go mod tidy
+  ```
+
+- Build with the same Go toolchain (see `go-version` target in BMO's
+  Makefile):
+
+  ```sh
+  CGO_ENABLED=1 go build -buildmode=plugin -o myprov.so ./plugin
+  ```
+
+- If `plugin.Open` reports a mismatch on a specific package, align it in
+  your `go.mod` (`replace` or `go get <pkg>@<ver>`) to the version used by
+  the BMO release, then rebuild.

main.go

@@ -16,6 +16,7 @@ limitations under the License.
 package main
 
 import (
+	"cmp"
 	"crypto/tls"
 	"flag"
 	"fmt"
@@ -30,9 +31,7 @@ import (
 	webhooks "github.com/metal3-io/baremetal-operator/internal/webhooks/metal3.io/v1alpha1"
 	"github.com/metal3-io/baremetal-operator/pkg/imageprovider"
 	"github.com/metal3-io/baremetal-operator/pkg/provisioner"
-	"github.com/metal3-io/baremetal-operator/pkg/provisioner/demo"
 	"github.com/metal3-io/baremetal-operator/pkg/provisioner/fixture"
-	"github.com/metal3-io/baremetal-operator/pkg/provisioner/ironic"
 	"github.com/metal3-io/baremetal-operator/pkg/secretutils"
 	"github.com/metal3-io/baremetal-operator/pkg/version"
 	ironicv1alpha1 "github.com/metal3-io/ironic-standalone-operator/api/v1alpha1"
@@ -73,6 +72,9 @@ var (
 
 const leaderElectionID = "baremetal-operator"
 
+// defaultIronicPluginPath is where the BMO image bakes the ironic provisioner plugin.
+const defaultIronicPluginPath = "/plugins/ironic-provisioner.so"
+
 func init() {
 	_ = clientgoscheme.AddToScheme(scheme)
 
@@ -132,7 +134,7 @@ func main() {
 	var preprovImgEnable bool
 	var devLogging bool
 	var runInTestMode bool
-	var runInDemoMode bool
+	var provisionerPlugin string
 	var webhookPort int
 	var restConfigQPS float64
 	var restConfigBurst int
@@ -155,8 +157,8 @@ func main() {
 	flag.BoolVar(&preprovImgEnable, "build-preprov-image", false, "enable integration with the PreprovisioningImage API")
 	flag.BoolVar(&devLogging, "dev", false, "enable developer logging")
 	flag.BoolVar(&runInTestMode, "test-mode", false, "disable ironic communication")
-	flag.BoolVar(&runInDemoMode, "demo-mode", false,
-		"use the demo provisioner to set host states")
+	flag.StringVar(&provisionerPlugin, "provisioner-plugin", os.Getenv("PROVISIONER_PLUGIN"),
+		"Path to a Go plugin .so implementing the provisioner Factory. Defaults to "+defaultIronicPluginPath+".")
 	flag.StringVar(&healthAddr, "health-addr", ":9440",
 		"The address the health endpoint binds to.")
 	flag.IntVar(&webhookPort, "webhook-port", 9443, //nolint:mnd
@@ -317,22 +319,25 @@ func main() {
 	if runInTestMode {
 		ctrl.Log.Info("using test provisioner")
 		provisionerFactory = &fixture.Fixture{}
-	} else if runInDemoMode {
-		ctrl.Log.Info("using demo provisioner")
-		provisionerFactory = &demo.Demo{}
 	} else {
+		pluginPath := cmp.Or(provisionerPlugin, defaultIronicPluginPath)
 		provLog := zap.New(zap.UseFlagOptions(&logOpts)).WithName("provisioner")
-		// Check if we should use Ironic CR integration
-		if ironicName != "" && ironicNamespace != "" {
-			provisionerFactory, err = ironic.NewProvisionerFactoryWithClient(provLog, preprovImgEnable,
-				mgr.GetClient(), mgr.GetAPIReader(), ironicName, ironicNamespace)
-		} else {
-			provisionerFactory, err = ironic.NewProvisionerFactory(provLog, preprovImgEnable)
+		var hostFeatures []provisioner.HostFeature
+		if preprovImgEnable {
+			hostFeatures = append(hostFeatures, provisioner.FeaturePreprovImg)
 		}
+		var pluginName string
+		provisionerFactory, pluginName, err = provisioner.LoadProvisionerPlugin(pluginPath, provisioner.PluginConfig{
+			Logger:    provLog,
+			Features:  hostFeatures,
+			K8sClient: mgr.GetClient(),
+			APIReader: mgr.GetAPIReader(),
+		})
 		if err != nil {
-			setupLog.Error(err, "cannot start ironic provisioner")
+			setupLog.Error(err, "cannot load provisioner plugin", "path", pluginPath)
 			os.Exit(1)
 		}
+		ctrl.Log.Info("loaded provisioner plugin", "name", pluginName, "path", pluginPath)
 	}
 
 	maxConcurrency, err := getMaxConcurrentReconciles(controllerConcurrency)

pkg/provisioner/demo/plugin/main.go

@@ -0,0 +1,35 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/metal3-io/baremetal-operator/pkg/provisioner"
+	"github.com/metal3-io/baremetal-operator/pkg/provisioner/demo"
+)
+
+// PluginName is advertised to the host via plugin.Lookup.
+//
+//nolint:unused // resolved by plugin.Lookup; see pkg/provisioner/plugin.go
+func PluginName() string { return "demo" }
+
+// NewProvisionerFactory is the exported symbol that BMO looks up in the plugin.
+// It is resolved at runtime via plugin.Lookup from the host binary, so static
+// analysis cannot see the reference.
+//
+//nolint:unused // resolved by plugin.Lookup; see pkg/provisioner/plugin.go
+func NewProvisionerFactory(_ provisioner.PluginConfig) (provisioner.Factory, error) {
+	return &demo.Demo{}, nil
+}

pkg/provisioner/ironic/plugin/main.go

@@ -0,0 +1,54 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+
+	"github.com/metal3-io/baremetal-operator/pkg/provisioner"
+	"github.com/metal3-io/baremetal-operator/pkg/provisioner/ironic"
+)
+
+const pluginName = "ironic"
+
+// PluginName is advertised to the host via plugin.Lookup.
+//
+//nolint:unused // resolved by plugin.Lookup; see pkg/provisioner/plugin.go
+func PluginName() string { return pluginName }
+
+// NewProvisionerFactory is the exported symbol that BMO looks up in the plugin.
+// It is resolved at runtime via plugin.Lookup from the host binary, so static
+// analysis cannot see the reference.
+//
+//nolint:unused // resolved by plugin.Lookup; see pkg/provisioner/plugin.go
+func NewProvisionerFactory(config provisioner.PluginConfig) (provisioner.Factory, error) {
+	logger := config.Logger.WithName(pluginName)
+
+	ironicName := os.Getenv("IRONIC_NAME")
+	ironicNamespace := os.Getenv("IRONIC_NAMESPACE")
+
+	havePreprovImgBuilder := config.HasFeature(provisioner.FeaturePreprovImg)
+
+	if config.K8sClient != nil && ironicName != "" && ironicNamespace != "" {
+		return ironic.NewProvisionerFactoryWithClient(
+			logger, havePreprovImgBuilder,
+			config.K8sClient, config.APIReader,
+			ironicName, ironicNamespace,
+		)
+	}
+
+	return ironic.NewProvisionerFactory(logger, havePreprovImgBuilder)
+}