Merge pull request #409 from uselagoon/k8s-collector

feat: environment collection

Author: rocketeerbkw

cmd/collect_environment.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/uselagoon/build-deploy-tool/internal/collector"
+	"github.com/uselagoon/build-deploy-tool/internal/helpers"
+	"github.com/uselagoon/build-deploy-tool/internal/k8s"
+)
+
+func init() {
+	collectCmd.AddCommand(collectEnvironment)
+}
+
+var collectEnvironment = &cobra.Command{
+	Use:     "environment",
+	Aliases: []string{"e"},
+	Short:   "Collect seed information about the environment",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		// get a k8s client
+		client, err := k8s.NewClient()
+		if err != nil {
+			return err
+		}
+		// create a collector
+		col := collector.NewCollector(client)
+		namespace := helpers.GetEnv("NAMESPACE", "", false)
+		namespace, err = helpers.GetNamespace(namespace, "/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+		if err != nil {
+			return err
+		}
+		if namespace == "" {
+			return fmt.Errorf("unable to detect namespace")
+		}
+		// collect the environment
+		data, err := CollectEnvironment(col, namespace)
+		if err != nil {
+			return err
+		}
+		env, err := json.MarshalIndent(data, "", "  ")
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(env))
+		return nil
+	},
+}
+
+// CollectEnvironment .
+func CollectEnvironment(c *collector.Collector, namespace string) (*collector.LagoonEnvState, error) {
+	state, err := c.Collect(context.Background(), namespace)
+	if err != nil {
+		return nil, err
+	}
+	return state, nil
+}

cmd/collect_environment_test.go

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"encoding/json"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/andreyvit/diff"
+	"github.com/uselagoon/build-deploy-tool/internal/collector"
+	"github.com/uselagoon/build-deploy-tool/internal/k8s"
+
+	// changes the testing to source from root so paths to test resources must be defined from repo root
+	_ "github.com/uselagoon/build-deploy-tool/internal/testing"
+)
+
+func TestCollectEnvironment(t *testing.T) {
+	type args struct {
+		namespace string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		seedDir string
+		want    string
+		wantErr bool
+	}{
+
+		{
+			name: "list-environment",
+			args: args{
+				namespace: "example-project-main",
+			},
+			seedDir: "internal/collector/testdata/seed/seed-1",
+			want:    "internal/collector/testdata/json-result/result-1.json",
+			wantErr: false,
+		},
+
+		{
+			name: "list-environment-with-pvcs",
+			args: args{
+				namespace: "example-project-main",
+			},
+			seedDir: "internal/collector/testdata/seed/seed-2",
+			want:    "internal/collector/testdata/json-result/result-2.json",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			client, err := k8s.NewFakeClient(tt.args.namespace)
+			if err != nil {
+				t.Errorf("error creating fake client")
+			}
+			err = k8s.SeedFakeData(client, tt.args.namespace, tt.seedDir)
+			if err != nil {
+				t.Errorf("error seeding fake data: %v", err)
+			}
+			col := collector.NewCollector(client)
+			got, err := CollectEnvironment(col, tt.args.namespace)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("CollectEnvironment() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			results, err := os.ReadFile(tt.want)
+			if err != nil {
+				t.Errorf("couldn't read file %v: %v", tt.want, err)
+			}
+			env, err := json.MarshalIndent(got, "", "  ")
+			if err != nil {
+				t.Errorf("couldn't read file %v: %v", tt.want, err)
+			}
+			if !reflect.DeepEqual(string(results), string(env)) {
+				t.Errorf("Collect() = \n%v", diff.LineDiff(string(env), string(results)))
+			}
+		})
+	}
+}

cmd/root.go

@@ -60,6 +60,13 @@ var validateCmd = &cobra.Command{
 	Long:    `Validate resources for Lagoon builds`,
 }
 
+var collectCmd = &cobra.Command{
+	Use:     "collect",
+	Aliases: []string{"col", "c"},
+	Short:   "Collect resource information",
+	Long:    `Collect resource information for Lagoon builds`,
+}
+
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
@@ -99,6 +106,7 @@ func init() {
 	rootCmd.AddCommand(taskCmd)
 	rootCmd.AddCommand(identifyCmd)
 	rootCmd.AddCommand(validateCmd)
+	rootCmd.AddCommand(collectCmd)
 
 	rootCmd.PersistentFlags().StringP("lagoon-yml", "l", ".lagoon.yml",
 		"The .lagoon.yml file to read")

cmd/validate_lagoonyml_test.go

@@ -171,7 +171,7 @@ func TestValidateLagoonYml(t *testing.T) {
 
 			err = yaml.Unmarshal(wantsLYAMLString, wantsLYAML)
 			if err != nil {
-				t.Errorf(err.Error())
+				t.Errorf("couldn't unmarshal yaml: %v", err)
 				return
 			}
 

go.mod

@@ -27,6 +27,7 @@ require (
 	k8s.io/api v0.32.1
 	k8s.io/apimachinery v0.32.1
 	k8s.io/client-go v0.32.1
+	sigs.k8s.io/controller-runtime v0.20.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -86,7 +87,6 @@ require (
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect
 	k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
-	sigs.k8s.io/controller-runtime v0.20.0 // indirect
 	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect
 )

internal/collector/collector.go

@@ -0,0 +1,132 @@
+package collector
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
+	k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	networkv1 "k8s.io/api/networking/v1"
+
+	mariadbv1 "github.com/amazeeio/dbaas-operator/apis/mariadb/v1"
+	mongodbv1 "github.com/amazeeio/dbaas-operator/apis/mongodb/v1"
+	postgresv1 "github.com/amazeeio/dbaas-operator/apis/postgres/v1"
+	client "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type Collector struct {
+	Client client.Client
+}
+
+type LagoonEnvState struct {
+	Deployments           *appsv1.DeploymentList             `json:"deployments"`
+	Cronjobs              *batchv1.CronJobList               `json:"cronjobs"`
+	Ingress               *networkv1.IngressList             `json:"ingress"`
+	Services              *corev1.ServiceList                `json:"services"`
+	Secrets               *corev1.SecretList                 `json:"secrets"`
+	PVCs                  *corev1.PersistentVolumeClaimList  `json:"pvcs"`
+	SchedulesV1           *k8upv1.ScheduleList               `json:"schedulesv1"`
+	SchedulesV1Alpha1     *k8upv1alpha1.ScheduleList         `json:"schedulesv1alpha1"`
+	PreBackupPodsV1       *k8upv1.PreBackupPodList           `json:"prebackuppodsv1"`
+	PreBackupPodsV1Alpha1 *k8upv1alpha1.PreBackupPodList     `json:"prebackuppodsv1alpha1"`
+	MariaDBConsumers      *mariadbv1.MariaDBConsumerList     `json:"mariadbconsumers"`
+	MongoDBConsumers      *mongodbv1.MongoDBConsumerList     `json:"mongodbconsumers"`
+	PostgreSQLConsumers   *postgresv1.PostgreSQLConsumerList `json:"postgresqlconsumers"`
+}
+
+func NewCollector(client client.Client) *Collector {
+	return &Collector{
+		Client: client,
+	}
+}
+
+func (c *Collector) Collect(ctx context.Context, namespace string) (*LagoonEnvState, error) {
+	var state LagoonEnvState
+	var err error
+	state.Deployments, err = c.CollectDeployments(ctx, namespace)
+	if err != nil {
+		return nil, err
+	}
+	state.Cronjobs, err = c.CollectCronjobs(ctx, namespace)
+	if err != nil {
+		return nil, err
+	}
+	state.Ingress, err = c.CollectIngress(ctx, namespace)
+	if err != nil {
+		return nil, err
+	}
+	state.Services, err = c.CollectServices(ctx, namespace)
+	if err != nil {
+		return nil, err
+	}
+	state.Secrets, err = c.CollectSecrets(ctx, namespace)
+	if err != nil {
+		return nil, err
+	}
+	state.PVCs, err = c.CollectPVCs(ctx, namespace)
+	if err != nil {
+		return nil, err
+	}
+	state.MariaDBConsumers, err = c.CollectMariaDBConsumers(ctx, namespace)
+	if err != nil {
+		// handle if consumer crds not installed
+		if !strings.Contains(err.Error(), "no matches for kind") {
+			fmt.Fprintln(os.Stderr, err)
+			return nil, err
+		}
+	}
+	state.MongoDBConsumers, err = c.CollectMongoDBConsumers(ctx, namespace)
+	if err != nil {
+		// handle if consumer crds not installed
+		if !strings.Contains(err.Error(), "no matches for kind") {
+			fmt.Fprintln(os.Stderr, err)
+			return nil, err
+		}
+	}
+	state.PostgreSQLConsumers, err = c.CollectPostgreSQLConsumers(ctx, namespace)
+	if err != nil {
+		// handle if consumer crds not installed
+		if !strings.Contains(err.Error(), "no matches for kind") {
+			fmt.Fprintln(os.Stderr, err)
+			return nil, err
+		}
+	}
+	state.SchedulesV1, err = c.CollectSchedulesV1(ctx, namespace)
+	if err != nil {
+		// handle if k8up v1 crds not installed
+		if !strings.Contains(err.Error(), "no matches for kind") {
+			fmt.Fprintln(os.Stderr, err)
+			return nil, err
+		}
+	}
+	state.SchedulesV1Alpha1, err = c.CollectSchedulesV1Alpha1(ctx, namespace)
+	if err != nil {
+		// handle if k8up v1alpha1 crds not installed
+		if !strings.Contains(err.Error(), "no matches for kind") {
+			fmt.Fprintln(os.Stderr, err)
+			return nil, err
+		}
+	}
+	state.PreBackupPodsV1, err = c.CollectPreBackupPodsV1(ctx, namespace)
+	if err != nil {
+		// handle if k8up v1 crds not installed
+		if !strings.Contains(err.Error(), "no matches for kind") {
+			fmt.Fprintln(os.Stderr, err)
+			return nil, err
+		}
+	}
+	state.PreBackupPodsV1Alpha1, err = c.CollectPreBackupPodsV1Alpha1(ctx, namespace)
+	if err != nil {
+		// handle if k8up v1alpha1 crds not installed
+		if !strings.Contains(err.Error(), "no matches for kind") {
+			fmt.Fprintln(os.Stderr, err)
+			return nil, err
+		}
+	}
+	return &state, nil
+}