diff --git a/charts/opensearch-operator/files/opensearch.opster.io_opensearchclusters.yaml b/charts/opensearch-operator/files/opensearch.opster.io_opensearchclusters.yaml index 1ea3615cc..b329b1bc0 100644 --- a/charts/opensearch-operator/files/opensearch.opster.io_opensearchclusters.yaml +++ b/charts/opensearch-operator/files/opensearch.opster.io_opensearchclusters.yaml @@ -839,10 +839,35 @@ spec: type: object jvm: type: string + keystore: + items: + properties: + keyMappings: + additionalProperties: + type: string + description: Key mappings from secret to keystore keys + type: object + secret: + description: Secret containing key value pairs + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + type: array nodeSelector: additionalProperties: type: string type: object + pluginsList: + items: + type: string + type: array resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/docs/designs/crd.md b/docs/designs/crd.md index ceb49c0e1..52bb82b1e 100644 --- a/docs/designs/crd.md +++ b/docs/designs/crd.md @@ -6,7 +6,7 @@ A resource is an endpoint in the Kubernetes API that stores a collection of API A [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) is an extension of the Kubernetes API, many core Kubernetes functions are now built using custom resources, making Kubernetes more modular. Cluster admins can update custom resources independently of the cluster itself. Once a custom resource is installed, users can create and access its objects using kubectl, just as they do for built-in resources like Pods. -The CustomResourceDefinition API resource allows you to define custom resources. Defining a CRD object creates a new custom resource with a name and schema that you specify. The Kubernetes API serves and handles the storage of your custom resource. Every resource is build from `KGV` that stands for Group Version Resource and this is what drives the Kubernetes API Server structure. +The CustomResourceDefinition API resource allows you to define custom resources. Defining a CRD object creates a new custom resource with a name and schema that you specify. The Kubernetes API serves and handles the storage of your custom resource. Every resource is build from `KGV` that stands for Group Version Resource and this is what drives the Kubernetes API Server structure. The `OpensearchCLuster` CRD is representing an Opensearch cluster. @@ -115,7 +115,7 @@ ClusterSpec defines the desired state of OpensearchCluster GeneralConfig -GeneralConfig defines global Opensearch cluster configuration +GeneralConfig defines global Opensearch cluster configuration @@ -290,6 +290,18 @@ Bootstrap defines Opensearch bootstrap pod configuration + + + + + + + + + + + +
Added extra items to opensearch.yml in the bootstrap pod map[string]string general.additionalConfig
keystore[]opsterv1.KeystoreValueList of objects that define secret values that will populate the opensearch keystore in the bootstrap podfalse -
pluginsList[]stringList of plugins that should be installed for OpenSearch at startup in the boostrap podfalse []
@@ -432,7 +444,7 @@ Dashboards defines Opensearch-Dashboard configuration and deployment NodePools -Every NodePool is defining different Opensearch Nodes StatefulSet +Every NodePool is defining different Opensearch Nodes StatefulSet @@ -581,8 +593,8 @@ InitHelperConfig defines global Opensearch InitHelper image configuration - - + +
string Version of InitHelper (busybox) image to deploy false1.27.2-buildx
1.27.2-buildx

@@ -676,7 +688,7 @@ Monitoring TLS configuration options Keystore

-Every Keystore Value defines a secret to pull secrets from. +Every Keystore Value defines a secret to pull secrets from. diff --git a/docs/userguide/main.md b/docs/userguide/main.md index 7f55780e9..227c3f716 100644 --- a/docs/userguide/main.md +++ b/docs/userguide/main.md @@ -124,7 +124,7 @@ spec: nodePools: - component: masters replicas: 3 # The number of replicas - diskSize: "30Gi" # The disk size to use + diskSize: "30Gi" # The disk size to use resources: # The resource requests and limits for that nodepool requests: memory: "2Gi" @@ -221,7 +221,7 @@ If you provide your own node certificates you must also provide an admin cert th spec: security: config: - adminSecret: + adminSecret: name: my-first-cluster-admin-cert # The secret must have keys tls.crt and tls.key ``` @@ -278,6 +278,14 @@ To install a plugin for opensearch dashboards add it to the list under `dashboar - sample-plugin-name ``` +To install a plugin for the bootstrap pod add it to the list under `bootstrap.pluginsList`: + +```yaml + bootstrap: + pluginsList: ["repository-s3"] +``` + + Please note: * [Bundled plugins](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/plugins/#bundled-plugins) do not have to be added to the list, they are installed automatically @@ -323,6 +331,18 @@ If you only want to load some keys from a secret or rename the existing keys, yo Note that only provided keys will be loaded from the secret! Any keys not specified will be ignored. +To populate the keystore of the boostrap pod add the secrets under the `bootstrap.keystore` section: + +```yaml + bootstrap: + # ... + keystore: + - secret: + name: credentials + - secret: + name: some-other-secret +``` + ### SmartScaler What is SmartScaler? @@ -382,7 +402,7 @@ You can configure the snapshot repositories for the OpenSearch cluster through t ```yaml spec: general: - snapshotRepositories: + snapshotRepositories: - name: my_s3_repository_1 type: s3 settings: @@ -737,7 +757,7 @@ spec: projected: sources: serviceAccountToken: - path: "token" + path: "token" dashboards: additionalVolumes: - name: example-secret @@ -775,7 +795,7 @@ spec: env: - name: MY_ENV_VAR value: "myvalue" - # the other options are supported here as well + # the other options are supported here as well ``` ### Custom cluster domain name @@ -793,7 +813,7 @@ manager: During cluster initialization the operator uses init containers as helpers. For these containers a busybox image is used ( specifically `docker.io/busybox:latest`). In case you are working in an offline environment and the cluster cannot access the registry or you want to customize the image, you can override the image used by specifying the `initHelper` image in your cluster spec: ```yaml - spec: + spec: initHelper: # You can either only specify the version version: "1.27.2-buildcustom" @@ -1393,7 +1413,7 @@ metadata: spec: opensearchCluster: name: my-first-cluster - + name: logs_template # name of the index template - defaults to metadata.name. Can't be updated in-place indexPatterns: # required index patterns diff --git a/opensearch-operator/api/v1/opensearch_types.go b/opensearch-operator/api/v1/opensearch_types.go index 75e822abf..7da1e55a1 100644 --- a/opensearch-operator/api/v1/opensearch_types.go +++ b/opensearch-operator/api/v1/opensearch_types.go @@ -171,6 +171,8 @@ type BootstrapConfig struct { Jvm string `json:"jvm,omitempty"` // Extra items to add to the opensearch.yml, defaults to General.AdditionalConfig AdditionalConfig map[string]string `json:"additionalConfig,omitempty"` + PluginsList []string `json:"pluginsList,omitempty"` + Keystore []KeystoreValue `json:"keystore,omitempty"` } type DashboardsServiceSpec struct { diff --git a/opensearch-operator/api/v1/zz_generated.deepcopy.go b/opensearch-operator/api/v1/zz_generated.deepcopy.go index 055616a08..fb0068d7f 100644 --- a/opensearch-operator/api/v1/zz_generated.deepcopy.go +++ b/opensearch-operator/api/v1/zz_generated.deepcopy.go @@ -295,6 +295,18 @@ func (in *BootstrapConfig) DeepCopyInto(out *BootstrapConfig) { (*out)[key] = val } } + if in.PluginsList != nil { + in, out := &in.PluginsList, &out.PluginsList + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Keystore != nil { + in, out := &in.Keystore, &out.Keystore + *out = make([]KeystoreValue, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapConfig. diff --git a/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchclusters.yaml b/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchclusters.yaml index 1ea3615cc..b329b1bc0 100644 --- a/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchclusters.yaml +++ b/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchclusters.yaml @@ -839,10 +839,35 @@ spec: type: object jvm: type: string + keystore: + items: + properties: + keyMappings: + additionalProperties: + type: string + description: Key mappings from secret to keystore keys + type: object + secret: + description: Secret containing key value pairs + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + type: array nodeSelector: additionalProperties: type: string type: object + pluginsList: + items: + type: string + type: array resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/opensearch-operator/pkg/builders/cluster.go b/opensearch-operator/pkg/builders/cluster.go index 60938e9f5..0c5c2f76d 100644 --- a/opensearch-operator/pkg/builders/cluster.go +++ b/opensearch-operator/pkg/builders/cluster.go @@ -870,6 +870,98 @@ func NewBootstrapPod( }) } + // If Keystore Values are set in OpenSearchCluster manifest + if cr.Spec.Bootstrap.Keystore != nil && len(cr.Spec.Bootstrap.Keystore) > 0 { + + // Add volume and volume mount for keystore + volumes = append(volumes, corev1.Volume{ + Name: "keystore", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "keystore", + MountPath: "/usr/share/opensearch/config/opensearch.keystore", + SubPath: "opensearch.keystore", + }) + + initContainerVolumeMounts := []corev1.VolumeMount{ + { + Name: "keystore", + MountPath: "/tmp/keystore", + }, + } + + // Add volumes and volume mounts for keystore secrets + for _, keystoreValue := range cr.Spec.Bootstrap.Keystore { + volumes = append(volumes, corev1.Volume{ + Name: "keystore-" + keystoreValue.Secret.Name, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: keystoreValue.Secret.Name, + }, + }, + }) + + if keystoreValue.KeyMappings == nil || len(keystoreValue.KeyMappings) == 0 { + // If no renames are necessary, mount secret key-value pairs directly + initContainerVolumeMounts = append(initContainerVolumeMounts, corev1.VolumeMount{ + Name: "keystore-" + keystoreValue.Secret.Name, + MountPath: "/tmp/keystoreSecrets/" + keystoreValue.Secret.Name, + }) + } else { + keys := helpers.SortedKeys(keystoreValue.KeyMappings) + for _, oldKey := range keys { + initContainerVolumeMounts = append(initContainerVolumeMounts, corev1.VolumeMount{ + Name: "keystore-" + keystoreValue.Secret.Name, + MountPath: "/tmp/keystoreSecrets/" + keystoreValue.Secret.Name + "/" + keystoreValue.KeyMappings[oldKey], + SubPath: oldKey, + }) + } + } + } + + keystoreInitContainer := corev1.Container{ + Name: "keystore", + Image: image.GetImage(), + ImagePullPolicy: image.GetImagePullPolicy(), + Resources: resources, + Command: []string{ + "sh", + "-c", + ` + #!/usr/bin/env bash + set -euo pipefail + + /usr/share/opensearch/bin/opensearch-keystore create + for i in /tmp/keystoreSecrets/*/*; do + key=$(basename $i) + echo "Adding file $i to keystore key $key" + /usr/share/opensearch/bin/opensearch-keystore add-file "$key" "$i" + done + + # Add the bootstrap password since otherwise the opensearch entrypoint tries to do this on startup + if [ ! -z ${PASSWORD+x} ]; then + echo 'Adding env $PASSWORD to keystore as key bootstrap.password' + echo "$PASSWORD" | /usr/share/opensearch/bin/opensearch-keystore add -x bootstrap.password + fi + + cp -a /usr/share/opensearch/config/opensearch.keystore /tmp/keystore/ + `, + }, + VolumeMounts: initContainerVolumeMounts, + SecurityContext: securityContext, + } + + initContainers = append(initContainers, keystoreInitContainer) + } + + startUpCommand := "./opensearch-docker-entrypoint.sh" + + pluginslist := helpers.RemoveDuplicateStrings(cr.Spec.Bootstrap.PluginsList) + mainCommand := helpers.BuildMainCommand("./bin/opensearch-plugin", pluginslist, true, startUpCommand) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: BootstrapPodName(cr), @@ -881,6 +973,7 @@ func NewBootstrapPod( { Env: env, Name: "opensearch", + Command: mainCommand, Image: image.GetImage(), ImagePullPolicy: image.GetImagePullPolicy(), Resources: resources, diff --git a/opensearch-operator/pkg/builders/cluster_test.go b/opensearch-operator/pkg/builders/cluster_test.go index 7b0e25807..bd6f8824b 100644 --- a/opensearch-operator/pkg/builders/cluster_test.go +++ b/opensearch-operator/pkg/builders/cluster_test.go @@ -42,6 +42,23 @@ func ClusterDescWithKeystoreSecret(secretName string, keyMappings map[string]str } } +func ClusterDescWithBootstrapKeystoreSecret(secretName string, keyMappings map[string]string) opsterv1.OpenSearchCluster { + return opsterv1.OpenSearchCluster{ + Spec: opsterv1.ClusterSpec{ + Bootstrap: opsterv1.BootstrapConfig{ + Keystore: []opsterv1.KeystoreValue{ + { + Secret: corev1.LocalObjectReference{ + Name: secretName, + }, + KeyMappings: keyMappings, + }, + }, + }, + }, + } +} + func ClusterDescWithAdditionalConfigs(addtitionalConfig map[string]string, bootstrapAdditionalConfig map[string]string) opsterv1.OpenSearchCluster { return opsterv1.OpenSearchCluster{ Spec: opsterv1.ClusterSpec{ @@ -449,6 +466,77 @@ var _ = Describe("Builders", func() { Value: mockBootstrapConfig[mockKey2], })) }) + It("should properly setup the main command when installing plugins", func() { + clusterObject := ClusterDescWithVersion("2.2.1") + pluginA := "some-plugin" + pluginB := "another-plugin" + + clusterObject.Spec.Bootstrap.PluginsList = []string{pluginA, pluginB} + result := NewBootstrapPod(&clusterObject, nil, nil) + + installCmd := fmt.Sprintf( + "./bin/opensearch-plugin install --batch '%s' '%s' && ./opensearch-docker-entrypoint.sh", + pluginA, + pluginB, + ) + + expected := []string{ + "/bin/bash", + "-c", + installCmd, + } + + actual := result.Spec.Containers[0].Command + + Expect(expected).To(Equal(actual)) + }) + }) + + When("Constructing a bootstrap pod with Keystore Values", func() { + It("should create a proper initContainer", func() { + mockSecretName := "some-secret" + clusterObject := ClusterDescWithBootstrapKeystoreSecret(mockSecretName, nil) + + result := NewBootstrapPod(&clusterObject, nil, nil) + Expect(result.Spec.InitContainers[1].VolumeMounts).To(ContainElements([]corev1.VolumeMount{ + { + Name: "keystore", + MountPath: "/tmp/keystore", + }, + { + Name: "keystore-" + mockSecretName, + MountPath: "/tmp/keystoreSecrets/" + mockSecretName, + }, + })) + }) + + It("should mount the prefilled keystore into the opensearch container", func() { + mockSecretName := "some-secret" + clusterObject := ClusterDescWithBootstrapKeystoreSecret(mockSecretName, nil) + result := NewBootstrapPod(&clusterObject, nil, nil) + Expect(result.Spec.Containers[0].VolumeMounts).To(ContainElement(corev1.VolumeMount{ + Name: "keystore", + MountPath: "/usr/share/opensearch/config/opensearch.keystore", + SubPath: "opensearch.keystore", + })) + }) + + It("should properly rename secret keys when key mappings are given", func() { + mockSecretName := "some-secret" + oldKey := "old-key" + newKey := "new-key" + + keyMappings := map[string]string{ + oldKey: newKey, + } + clusterObject := ClusterDescWithBootstrapKeystoreSecret(mockSecretName, keyMappings) + result := NewBootstrapPod(&clusterObject, nil, nil) + Expect(result.Spec.InitContainers[1].VolumeMounts).To(ContainElement(corev1.VolumeMount{ + Name: "keystore-" + mockSecretName, + MountPath: "/tmp/keystoreSecrets/" + mockSecretName + "/" + newKey, + SubPath: oldKey, + })) + }) }) When("Constructing a STS for a NodePool with Keystore Values", func() {