Skip to content

[WIP] Use docker image's default startup command and install plugins via init container #1115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+174 −132
Open

[WIP] Use docker image's default startup command and install plugins via init container #1115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions opensearch-operator/controllers/cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,20 +446,6 @@ var _ = Describe("Cluster Reconciler", func() {
return sts.Spec.Template.Spec.Containers[0].Image == "docker.io/opensearchproject/opensearch:1.1.0"
}, timeout, interval).Should(BeTrue())
})
It("should update any plugin URLs", func() {
Eventually(func() bool {
sts := &appsv1.StatefulSet{}
if err := k8sClient.Get(
context.Background(),
client.ObjectKey{
Namespace: OpensearchCluster.Namespace,
Name: clusterName + "-nodes",
}, sts); err != nil {
return false
}
return ArrayElementContains(sts.Spec.Template.Spec.Containers[0].Command, "http://foo-plugin-1.1.0")
}, timeout, interval).Should(BeTrue())
})
})
When("a cluster is upgraded", func() {
Specify("updating the status should succeed", func() {
Expand Down
59 changes: 40 additions & 19 deletions opensearch-operator/pkg/builders/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,23 @@ const (
securityconfigChecksumAnnotation = "securityconfig/checksum"
)

// NewPluginInstallerInitContainer creates an init container for installing OpenSearch plugins
func NewPluginInstallerInitContainer(pluginsList []string, image opsterv1.ImageSpec, resources corev1.ResourceRequirements, securityContext *corev1.SecurityContext) *corev1.Container {
if len(pluginsList) == 0 {
return nil
}

pluginsCommand := helpers.BuildPluginInstallCommand("./bin/opensearch-plugin", pluginsList)
return &corev1.Container{
Name: "plugin-installer",
Image: image.GetImage(),
ImagePullPolicy: image.GetImagePullPolicy(),
Resources: resources,
Command: pluginsCommand,
SecurityContext: securityContext,
}
}

func NewSTSForNodePool(
username string,
cr *opsterv1.OpenSearchCluster,
Expand Down Expand Up @@ -334,25 +351,11 @@ func NewSTSForNodePool(
initHelperImage := helpers.ResolveInitHelperImage(cr)
resources := cr.Spec.InitHelper.Resources

startUpCommand := "./opensearch-docker-entrypoint.sh"
// If a custom command is specified, use it.
var mainCommand []string = nil
if len(cr.Spec.General.Command) > 0 {
startUpCommand = cr.Spec.General.Command
mainCommand = []string{"/bin/bash", "-c", cr.Spec.General.Command}
}

var pluginslist []string
if cr.Spec.General.Monitoring.Enable {
if cr.Spec.General.Monitoring.PluginURL != "" {
pluginslist = append(pluginslist, cr.Spec.General.Monitoring.PluginURL)
} else {
pluginslist = append(pluginslist, fmt.Sprintf(defaultMonitoringPlugin, cr.Spec.General.Version, cr.Spec.General.Version))
}
}

pluginslist = helpers.RemoveDuplicateStrings(append(pluginslist, cr.Spec.General.PluginsList...))

mainCommand := helpers.BuildMainCommand("./bin/opensearch-plugin", pluginslist, true, startUpCommand)

podSecurityContext := cr.Spec.General.PodSecurityContext
securityContext := cr.Spec.General.SecurityContext

Expand All @@ -377,6 +380,19 @@ func NewSTSForNodePool(
})
}

var pluginslist []string
if cr.Spec.General.Monitoring.Enable {
if cr.Spec.General.Monitoring.PluginURL != "" {
pluginslist = append(pluginslist, cr.Spec.General.Monitoring.PluginURL)
} else {
pluginslist = append(pluginslist, fmt.Sprintf(defaultMonitoringPlugin, cr.Spec.General.Version, cr.Spec.General.Version))
}
}
pluginslist = helpers.RemoveDuplicateStrings(append(pluginslist, cr.Spec.General.PluginsList...))
if pluginInitContainer := NewPluginInstallerInitContainer(pluginslist, image, resources, securityContext); pluginInitContainer != nil {
initContainers = append(initContainers, *pluginInitContainer)
}

// If Keystore Values are set in OpenSearchCluster manifest
if len(cr.Spec.General.Keystore) > 0 {

Expand Down Expand Up @@ -984,10 +1000,15 @@ func NewBootstrapPod(
initContainers = append(initContainers, keystoreInitContainer)
}

startUpCommand := "./opensearch-docker-entrypoint.sh"

pluginslist := helpers.RemoveDuplicateStrings(cr.Spec.Bootstrap.PluginsList)
mainCommand := helpers.BuildMainCommand("./bin/opensearch-plugin", pluginslist, true, startUpCommand)
if pluginInitContainer := NewPluginInstallerInitContainer(pluginslist, image, resources, securityContext); pluginInitContainer != nil {
initContainers = append(initContainers, *pluginInitContainer)
}

var mainCommand []string = nil
if len(cr.Spec.General.Command) > 0 {
mainCommand = []string{"/bin/bash", "-c", cr.Spec.General.Command}
}
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: BootstrapPodName(cr),
Expand Down
203 changes: 123 additions & 80 deletions opensearch-operator/pkg/builders/cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,44 @@ func ClusterDescWithVersion(version string) opsterv1.OpenSearchCluster {
}
}

func ClusterDescWithPlugins(plugins []string) opsterv1.OpenSearchCluster {
return opsterv1.OpenSearchCluster{
Spec: opsterv1.ClusterSpec{
General: opsterv1.GeneralConfig{
Version: "2.3.0",
ServiceName: "test-service",
PluginsList: plugins,
},
NodePools: []opsterv1.NodePool{
{
Component: "test-node",
Replicas: 1,
Roles: []string{"master", "data"},
},
},
},
}
}

func ClusterDescWithCustomCommand(command string) opsterv1.OpenSearchCluster {
return opsterv1.OpenSearchCluster{
Spec: opsterv1.ClusterSpec{
General: opsterv1.GeneralConfig{
Version: "2.3.0",
ServiceName: "test-service",
Command: command,
},
NodePools: []opsterv1.NodePool{
{
Component: "test-node",
Replicas: 1,
Roles: []string{"master", "data"},
},
},
},
}
}

func ClusterDescWithKeystoreSecret(secretName string, keyMappings map[string]string) opsterv1.OpenSearchCluster {
return opsterv1.OpenSearchCluster{
Spec: opsterv1.ClusterSpec{
Expand Down Expand Up @@ -240,31 +278,6 @@ var _ = Describe("Builders", func() {
Expect(actualUrl).To(Equal(expectedUrl))
})

It("should properly setup the main command when installing plugins", func() {
clusterObject := ClusterDescWithVersion("2.2.1")
pluginA := "some-plugin"
pluginB := "another-plugin"

clusterObject.Spec.General.PluginsList = []string{pluginA, pluginB}
result := NewSTSForNodePool("foobar", &clusterObject, opsterv1.NodePool{}, "foobar", nil, nil, nil)

installCmd := fmt.Sprintf(
"./bin/opensearch-plugin install --batch '%s' '%s' && ./opensearch-docker-entrypoint.sh",
pluginA,
pluginB,
)

expected := []string{
"/bin/bash",
"-c",
installCmd,
}

actual := result.Spec.Template.Spec.Containers[0].Command

Expect(expected).To(Equal(actual))
})

It("should add experimental flag when the node.roles contains search and the version is below 2.7", func() {
clusterObject := ClusterDescWithVersion("2.2.1")
nodePool := opsterv1.NodePool{
Expand Down Expand Up @@ -425,6 +438,58 @@ var _ = Describe("Builders", func() {
Value: "-Xmx1024M -Xms1024M -Dopensearch.transport.cname_in_publish_address=true",
}))
})

It("should create plugin installer init container when plugins are specified", func() {
cr := ClusterDescWithPlugins([]string{"plugin1", "plugin2"})
nodePool := cr.Spec.NodePools[0]

sts := NewSTSForNodePool("admin", &cr, nodePool, "test-checksum", []corev1.Volume{}, []corev1.VolumeMount{}, map[string]string{})

// Should have init containers including plugin installer
Expect(len(sts.Spec.Template.Spec.InitContainers)).To(BeNumerically(">=", 2)) // init + plugin-installer

// Find the plugin installer init container
var pluginInstaller *corev1.Container
for _, container := range sts.Spec.Template.Spec.InitContainers {
if container.Name == "plugin-installer" {
pluginInstaller = &container
break
}
}

Expect(pluginInstaller).ToNot(BeNil())
Expect(pluginInstaller.Command).To(ContainElement("/bin/bash"))
Expect(pluginInstaller.Command).To(ContainElement("-c"))
Expect(len(pluginInstaller.VolumeMounts)).To(Equal(0)) // No volume mounts needed
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to make sure what paths we have to share with the main container to reflect the plugin installation.
Then use volume for that path.

})

It("should use custom command when specified", func() {
cr := ClusterDescWithCustomCommand("custom-entrypoint.sh")
nodePool := cr.Spec.NodePools[0]

sts := NewSTSForNodePool("admin", &cr, nodePool, "test-checksum", []corev1.Volume{}, []corev1.VolumeMount{}, map[string]string{})

// Main container should use custom command
Expect(sts.Spec.Template.Spec.Containers[0].Command).To(Equal([]string{"/bin/bash", "-c", "custom-entrypoint.sh"}))
})

It("should use image default command when no custom command provided", func() {
cr := ClusterDescWithVersion("2.3.0")
cr.Spec.General.ServiceName = "test-service"
cr.Spec.NodePools = []opsterv1.NodePool{
{
Component: "test-node",
Replicas: 1,
Roles: []string{"master", "data"},
},
}
nodePool := cr.Spec.NodePools[0]

sts := NewSTSForNodePool("admin", &cr, nodePool, "test-checksum", []corev1.Volume{}, []corev1.VolumeMount{}, map[string]string{})

// Main container should not have command (uses image default)
Expect(sts.Spec.Template.Spec.Containers[0].Command).To(BeNil())
})
})

When("Constructing a bootstrap pod", func() {
Expand Down Expand Up @@ -490,29 +555,44 @@ var _ = Describe("Builders", func() {
Value: mockBootstrapConfig[mockKey2],
}))
})
It("should properly setup the main command when installing plugins", func() {
clusterObject := ClusterDescWithVersion("2.2.1")
pluginA := "some-plugin"
pluginB := "another-plugin"

clusterObject.Spec.Bootstrap.PluginsList = []string{pluginA, pluginB}
result := NewBootstrapPod(&clusterObject, nil, nil)
It("should use custom command when specified", func() {
cr := ClusterDescWithCustomCommand("custom-entrypoint.sh")
bootstrap := NewBootstrapPod(&cr, []corev1.Volume{}, []corev1.VolumeMount{})

installCmd := fmt.Sprintf(
"./bin/opensearch-plugin install --batch '%s' '%s' && ./opensearch-docker-entrypoint.sh",
pluginA,
pluginB,
)
// Bootstrap pod should use custom command
Expect(bootstrap.Spec.Containers[0].Command).To(Equal([]string{"/bin/bash", "-c", "custom-entrypoint.sh"}))
})

expected := []string{
"/bin/bash",
"-c",
installCmd,
}
It("should use image default command when no custom command provided", func() {
cr := ClusterDescWithVersion("2.3.0")
cr.Spec.General.ServiceName = "test-service"
bootstrap := NewBootstrapPod(&cr, []corev1.Volume{}, []corev1.VolumeMount{})

actual := result.Spec.Containers[0].Command
// Bootstrap pod should not have command (uses image default)
Expect(bootstrap.Spec.Containers[0].Command).To(BeNil())
})

Expect(expected).To(Equal(actual))
It("should create plugin installer init container when plugins specified", func() {
cr := ClusterDescWithVersion("2.3.0")
cr.Spec.General.ServiceName = "test-service"
cr.Spec.Bootstrap.PluginsList = []string{"bootstrap-plugin"}

bootstrap := NewBootstrapPod(&cr, []corev1.Volume{}, []corev1.VolumeMount{})

// Find the plugin installer init container
var pluginInstaller *corev1.Container
for _, container := range bootstrap.Spec.InitContainers {
if container.Name == "plugin-installer" {
pluginInstaller = &container
break
}
}

Expect(pluginInstaller).ToNot(BeNil())
Expect(pluginInstaller.Command).To(ContainElement("/bin/bash"))
Expect(pluginInstaller.Command).To(ContainElement("-c"))
Expect(len(pluginInstaller.VolumeMounts)).To(Equal(0)) // No volume mounts needed
})
})

Expand Down Expand Up @@ -889,47 +969,10 @@ var _ = Describe("Builders", func() {
Expect(result.Spec.Template.Spec.Containers[0].ReadinessProbe.InitialDelaySeconds).To(Equal(int32(65)))
Expect(result.Spec.Template.Spec.Containers[0].ReadinessProbe.TimeoutSeconds).To(Equal(int32(34)))
Expect(result.Spec.Template.Spec.Containers[0].ReadinessProbe.PeriodSeconds).To(Equal(int32(33)))
Expect(result.Spec.Template.Spec.Containers[0].ReadinessProbe.SuccessThreshold).To(Equal(int32(4)))
Expect(result.Spec.Template.Spec.Containers[0].ReadinessProbe.FailureThreshold).To(Equal(int32(9)))
})
})

When("Using custom command for OpenSearch probes", func() {
It("should have default command when not set", func() {
clusterObject := ClusterDescWithVersion("2.7.0")
nodePool := opsterv1.NodePool{
Component: "masters",
Roles: []string{"search"},
}
result := NewSTSForNodePool("foobar", &clusterObject, nodePool, "foobar", nil, nil, nil)
Expect(result.Spec.Template.Spec.Containers[0].StartupProbe.ProbeHandler.Exec.Command).
To(Equal([]string{"/bin/bash", "-c", "curl -k -u \"$(cat /mnt/admin-credentials/username):$(cat /mnt/admin-credentials/password)\" --silent --fail 'https://localhost:9200'"}))
Expect(result.Spec.Template.Spec.Containers[0].ReadinessProbe.ProbeHandler.Exec.Command).
To(Equal([]string{"/bin/bash", "-c", "curl -k -u \"$(cat /mnt/admin-credentials/username):$(cat /mnt/admin-credentials/password)\" --silent --fail 'https://localhost:9200'"}))
})

It("should have custom command when set", func() {
clusterObject := ClusterDescWithVersion("2.7.0")
nodePool := opsterv1.NodePool{
Component: "masters",
Roles: []string{"search"},
Probes: &opsterv1.ProbesConfig{
Startup: &opsterv1.CommandProbeConfig{
Command: []string{"/bin/bash", "-c", "echo 'startup'"},
},
Readiness: &opsterv1.CommandProbeConfig{
Command: []string{"/bin/bash", "-c", "echo 'ready'"},
},
},
}
result := NewSTSForNodePool("foobar", &clusterObject, nodePool, "foobar", nil, nil, nil)
Expect(result.Spec.Template.Spec.Containers[0].StartupProbe.ProbeHandler.Exec.Command).
To(Equal([]string{"/bin/bash", "-c", "echo 'startup'"}))
Expect(result.Spec.Template.Spec.Containers[0].ReadinessProbe.ProbeHandler.Exec.Command).
To(Equal([]string{"/bin/bash", "-c", "echo 'ready'"}))
})
})

When("Configuring InitHelper Resources", func() {
It("should propagate Resources to all init containers", func() {
clusterObject := ClusterDescWithVersion("2.2.1")
Expand Down
30 changes: 11 additions & 19 deletions opensearch-operator/pkg/helpers/reconcile-helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,27 +124,19 @@ func VersionCheck(instance *opsterv1.OpenSearchCluster) (int32, int32, string) {
return httpPort, securityConfigPort, securityConfigPath
}

func BuildMainCommand(installerBinary string, pluginsList []string, batchMode bool, entrypoint string) []string {
var mainCommand []string
com := installerBinary + " install"

if batchMode {
com = com + " --batch"
// BuildPluginInstallCommand builds a command for installing plugins in an init container
func BuildPluginInstallCommand(installerBinary string, pluginsList []string) []string {
if len(pluginsList) == 0 {
return []string{"/bin/bash", "-c", "echo 'No plugins to install'"}
}

if len(pluginsList) > 0 {
mainCommand = append(mainCommand, "/bin/bash", "-c")
for _, plugin := range pluginsList {
com = com + " '" + strings.ReplaceAll(plugin, "'", "\\'") + "'"
}

com = com + " && " + entrypoint
mainCommand = append(mainCommand, com)
} else {
mainCommand = []string{"/bin/bash", "-c", entrypoint}
var command []string
command = append(command, "/bin/bash", "-c")
com := installerBinary + " install --batch"
for _, plugin := range pluginsList {
com = com + " '" + strings.ReplaceAll(plugin, "'", "\\'") + "'"
}

return mainCommand
command = append(command, com)
return command
}

func BuildMainCommandOSD(installerBinary string, pluginsList []string, entrypoint string) []string {
Expand Down