Add testdata generation for hybrid operator and sanity check to CI (#105)

* [tests] add testdata generation

This commit makes the following changes:
1. Add testdata generation logic
2. Move the helm charts for unit tests in their respective repository
3. Modify main.go to build a separate binary containing the specified plugins
   needed to scaffold a hybrid project.

* [Makefile] cleanup makefile

This commit cleans up makefile to remove previous test which clones and
generates hybrid sample.

* [ci] Add test-sanity to CI

This commit adds test-sanity to ci, so that we are sure
that any changes in scaffolding is picked up.

Signed-off-by: varshaprasad96 <[email protected]>

* [Makefile] Add makefile target to fix lint

Signed-off-by: varshaprasad96 <[email protected]>

Author: varshaprasad96

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: ~1.16
+        go-version: ~1.17
       id: go
 
     - name: Check out code into the Go module directory
@@ -40,6 +40,9 @@ jobs:
 
     - name: Test
       run: make test
+    
+    - name: Test Sanity
+      run: make test-sanity
 
     - uses: shogo82148/actions-goveralls@v1
       with:

.github/workflows/deploy.yml

@@ -28,7 +28,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: ~1.16
+        go-version: ~1.17
 
     - name: Create release
       run: |

Makefile

@@ -25,6 +25,14 @@ TOOLS_BIN_DIR=$(TOOLS_DIR)/bin
 SCRIPTS_DIR=$(TOOLS_DIR)/scripts
 export PATH := $(BUILD_DIR):$(TOOLS_BIN_DIR):$(SCRIPTS_DIR):$(PATH)
 
+##@ Development
+
+.PHONY: generate
+generate: build # Generate CLI docs and samples
+	rm -rf testdata/
+	go run ./hack/generate/samples/generate_testdata.go
+	go generate ./...
+
 .PHONY: all
 all: test lint build
 
@@ -38,37 +46,21 @@ all: test lint build
 ENVTEST_VERSION = $(shell go list -m k8s.io/client-go | cut -d" " -f2 | sed 's/^v0\.\([[:digit:]]\+\)\.[[:digit:]]\+$$/1.\1.x/')
 TESTPKG ?= ./...
 # TODO: Modify this to use setup-envtest binary
-test: test-hybrid-plugin
+test: build
 	go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 	source <(setup-envtest use -p env $(ENVTEST_VERSION)) && go test -race -covermode atomic -coverprofile cover.out $(TESTPKG)
 
-test-hybrid-plugin: remove-tmp build/operator-sdk
-	mkdir -p tmp/test-hybrid-operator
-	(cd tmp/test-hybrid-operator && \
-	 ../../build/operator-sdk init --plugins hybrid --domain example.com --repo example.com/example/example-operator && \
-	 ../../build/operator-sdk create api --plugins go.kubebuilder.io/v3 --group apps --version v1alpha1 --kind=MemcachedBackup --resource --controller && \
-	 ../../build/operator-sdk create api --plugins helm.sdk.operatorframework.io/v1 --helm-chart=memcached --helm-chart-repo=https://charts.bitnami.com/bitnami --group apps --version v1alpha1 --kind=Memcached && \
-	 make docker-build)
+.PHONY: test-sanity
+test-sanity: generate fix ## Test repo formatting, linting, etc.
+	go vet ./...
+	$(SCRIPTS_DIR)/fetch golangci-lint 1.31.0 && $(TOOLS_BIN_DIR)/golangci-lint run
+	git diff --exit-code # diff again to ensure other checks don't change repo
 
 # Build manager binary
 .PHONY: build
 build:
 	CGO_ENABLED=0 mkdir -p $(BUILD_DIR) && go build $(GO_BUILD_ARGS) -o $(BUILD_DIR) ./
 
-# Incredibly fragile nad hopefully temporary build step for the SDK binary
-build/operator-sdk:
-	mkdir -p tmp
-	git clone https://github.com/operator-framework/operator-sdk.git tmp/operator-sdk-hybrid-patched
-	cp hack/osdk.patch tmp/operator-sdk-hybrid-patched/
-	(cd tmp/operator-sdk-hybrid-patched && git apply osdk.patch)
-	sed -i".bak" "s|REPLACE_ME_WITH_PATH|$(shell pwd)|" tmp/operator-sdk-hybrid-patched/go.mod
-	(cd tmp/operator-sdk-hybrid-patched && go mod tidy && make build/operator-sdk)
-	mkdir -p build
-	cp tmp/operator-sdk-hybrid-patched/build/operator-sdk ./build/operator-sdk
-
-remove-tmp:
-	rm -rf tmp/
-
 # Run go fmt and go mod tidy, and check for clean git tree
 .PHONY: fix
 fix:

hack/generate/samples/generate_testdata.go

@@ -0,0 +1,55 @@
+// Copyright 2021 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+
+	"github.com/operator-framework/helm-operator-plugins/hack/generate/samples/internal/hybrid"
+	log "github.com/sirupsen/logrus"
+)
+
+func main() {
+	// binaryPath allow inform the binary that should be used.
+	// By default it is helm-operator-plugins
+	var binaryPath string
+
+	flag.StringVar(&binaryPath, "binaryPath", "bin/helm-operator-plugins", "Binary path that should be used")
+	flag.Parse()
+
+	// Make the binary path absolute if pathed, for reproducibility and debugging purposes.
+	if dir, _ := filepath.Split(binaryPath); dir != "" {
+		tmp, err := filepath.Abs(binaryPath)
+		if err != nil {
+			log.Fatalf("Failed to make binary path %q absolute: %v", binaryPath, err)
+		}
+		binaryPath = tmp
+	}
+
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// samplesPath is the path where all samples should be generated
+	samplesPath := filepath.Join(wd, "testdata")
+	log.Infof("writing sample directories under %s", samplesPath)
+
+	log.Infof("creating Hybrid Memcached Sample")
+	hybrid.GenerateMemcachedSamples(binaryPath, samplesPath)
+
+}

hack/generate/samples/internal/hybrid/generate.go

@@ -0,0 +1,21 @@
+// Copyright 2021 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hybrid
+
+import "path/filepath"
+
+func GenerateMemcachedSamples(binaryPath, rootPath string) {
+	GenerateMemcachedSample(binaryPath, filepath.Join(rootPath, "hybrid"))
+}

hack/generate/samples/internal/hybrid/memcached.go

@@ -0,0 +1,101 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hybrid
+
+import (
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/operator-framework/helm-operator-plugins/hack/generate/samples/internal/pkg"
+)
+
+// Memcached defines the Memcached Sample in Helm
+type Memcached struct {
+	ctx *pkg.SampleContext
+}
+
+// GenerateMemcachedSample will call all actions to create the directory and generate the sample
+// The Context to run the samples are not the same in the e2e test. In this way, note that it should NOT
+// be called in the e2e tests since it will call the Prepare() to set the sample context and generate the files
+// in the testdata directory. The e2e tests only ought to use the Run() method with the TestContext.
+func GenerateMemcachedSample(binaryPath, samplesPath string) {
+	ctx, err := pkg.NewSampleContext(binaryPath, filepath.Join(samplesPath, "memcached-operator"),
+		"GO111MODULE=on")
+	pkg.CheckError("generating Helm memcached context", err)
+
+	memcached := Memcached{&ctx}
+	memcached.Prepare()
+	memcached.Run()
+}
+
+// Prepare the Context for the Memcached Helm Sample
+// Note that sample directory will be re-created and the context data for the sample
+// will be set such as the domain and GVK.
+func (mh *Memcached) Prepare() {
+	log.Infof("destroying directory for memcached helm samples")
+	mh.ctx.Destroy()
+
+	log.Infof("creating directory")
+	err := mh.ctx.Prepare()
+	pkg.CheckError("creating directory", err)
+
+	log.Infof("setting domain and GVK")
+	mh.ctx.Version = "v1alpha1"
+	mh.ctx.Group = "cache"
+	mh.ctx.Kind = "Memcached"
+}
+
+func (mh *Memcached) Run() {
+	// When we scaffold Helm based projects, it tries to use the discovery API of a Kubernetes
+	// cluster to intelligently build the RBAC rules that the operator will require based on the
+	// content of the helm chart.
+	//
+	// Here, we intentionally set KUBECONFIG to a broken value to ensure that operator-sdk will be
+	// unable to reach a real cluster, and thus will generate a default RBAC rule set. This is
+	// required to make Helm project generation idempotent because contributors and CI environments
+	// can all have slightly different environments that can affect the content of the generated
+	// role and cause sanity testing to fail.
+	os.Setenv("KUBECONFIG", "broken_so_we_generate_static_default_rules")
+	log.Infof("using init command and scaffolding the project")
+
+	err := mh.ctx.Init(
+		"--plugins", "hybrid/v1-alpha",
+		"--repo", "github.com/example/memcached-operator",
+	)
+
+	pkg.CheckError("creating the project", err)
+
+	// TODO: Uncomment this code after helm plugin is migrated
+	// err = mh.ctx.CreateAPI(
+	// 	"--plugins", "helm.sdk.operatorframework.io/v1",
+	// 	"--group", mh.ctx.Group,
+	// 	"--version", mh.ctx.Version,
+	// 	"--kind", mh.ctx.Kind,
+	// )
+
+	// pkg.CheckError("creating helm api", err)
+
+	err = mh.ctx.CreateAPI(
+		"--plugins", "base.go.kubebuilder.io/v3",
+		"--group", mh.ctx.Group,
+		"--version", mh.ctx.Version,
+		"--kind", mh.ctx.Kind,
+		"--resource", "--controller",
+	)
+
+	pkg.CheckError("creating go api", err)
+}

hack/generate/samples/internal/pkg/context.go

@@ -0,0 +1,28 @@
+// Copyright 2021 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import "github.com/operator-framework/helm-operator-plugins/internal/testutils"
+
+// SampleContext represents the Context used to generate the samples
+type SampleContext struct {
+	testutils.TestContext
+}
+
+// NewSampleContext returns a SampleContext containing a new kubebuilder TestContext.
+func NewSampleContext(binary string, path string, env ...string) (s SampleContext, err error) {
+	s.TestContext, err = testutils.NewPartialTestContext(binary, path, env...)
+	return s, err
+}

hack/generate/samples/internal/pkg/utils.go

@@ -0,0 +1,29 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkg
+
+import (
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// CheckError will exit with exit code 1 when err is not nil.
+func CheckError(msg string, err error) {
+	if err != nil {
+		log.Errorf("error %s: %s", msg, err)
+		os.Exit(1)
+	}
+}