Support file-based health checks with Liberty (#788)

* Use enableFileBased flag for probes

* Update bundle manifests

* Check nil probes and set defaults

* Update openlibertyapplication_controller.go

* Add MP_HEALTH_FILE_UPDATE_INTERVAL env

* Update utils.go

* Update utils.go

* Fix health check script command

* Parametrize file based health check scripts

* Rely on PATH for Liberty helpers/runtime scripts

* Update utils.go

* Update utils.go

* Add .spec.probes.fileDirectory flag

* Set probe timeout and period defaults

* Fix ReadinessProbeFileBasedScriptName ref

* Prevent default probes being overridden and always update Exec field

* Remove probe handlers before configuring file-based probe

* Allow Exec override in file-based probes config

* Add probe return stmt

* Init probe Exec after period/timeout seconds

* Set ProbeHandler instead of Exec to nil

* Clear probe settings when file-based disabled

* Set instance probes directly

* Add probe nil check

* Use CustomizeProbeDefaults from RCO

* Use DeepCopy instance when setting probes

* Revert new instance change

* Set custom probe file location

* Update utils.go

* Add checkInterval and startupCheckInterval to probes config

* Add probe fields descriptions to CSV

* Debug message

* Parse unstructured obj for image metadata

* Update openlibertyapplication_controller.go

* Read raw extension bytes from DockerImageMetadata

* Add .spec.importPolicy field

* Add image fetcher

* Update go.mod to add openshift image libraries

* Add file-based health check with Liberty version guard

* Parse image ID and tag separately before fallback to latest tag

* Return manage error when version is guarded

* Pull image w/o openshift imagestream

* rm setting image.Namespace

* Set pullSecret to nil when unconfigured

* Add debug statements

* Track deprecated liberty version labels

* Reconcile serviceaccount before pulling images

* Check if SA pull secret exists before pulling image

* Reuse credentials context and add retry flag

* Include imageRef.ID as manifest digest

* Add config to skip liberty version checks

* Prevent liberty version checks on latest tag

* Upgrade liberty version packaged with operator

* Cache the latest image version to skip validation for up-to-date apps

* Remove unused operator configs

* Add caching to track Liberty versions and opt out of version guards when the latest image has drifted out-of-date

* Track latest image last pull

* Track latest liberty image in cache

* Track image ID or digest in cache

* Remove latest image optimization and refresh image 2x/day

* Nest image.go into a package

* Revert liberty version secutil update

* Revert secutil version update

* Add imageVersionChecksRefreshIntervalMinutes config

* Remove importPolicy.insecure flag

* Only use Liberty version checks when the required features are enabled

* Have .probes.checkInterval and .probes.startupCheckInterval take precedence over .spec.env

* Remove 25.0.0.6-beta refs

* Use Container image when referring to images

* Pull runtime-component-operator@fb-probes

* Modify the appContainer directly using CustomizePodSpecFileBasedProbes

* Pull runtime-component-operator@fb-probes

* Make ManageError and ManageSuccess use warnings

* Fix checkInterval typo

* Add com.ibm.websphere.liberty.version to supported Liberty version labels

* Filter to ignore SHA values only and occasionally update images that couldn't be pulled

* Track .status.pulledImageReference to re-pull any applicationImage that has changed

* Exit CustomizePodSpecFileBasedProbes early if instance isn't using it

---------

Co-authored-by: Leo Christy Jesuraj <[email protected]>

Author: kabicin

api/v1/openlibertyapplication_types.go

@@ -80,7 +80,7 @@ type OpenLibertyApplicationSpec struct {
 	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
 
 	// +operator-sdk:csv:customresourcedefinitions:order=15,type=spec,displayName="Probes"
-	Probes *OpenLibertyApplicationProbes `json:"probes,omitempty"`
+	Probes *OpenLibertyApplicationProbesConfig `json:"probes,omitempty"`
 
 	// +operator-sdk:csv:customresourcedefinitions:order=16,type=spec,displayName="Deployment"
 	Deployment *OpenLibertyApplicationDeployment `json:"deployment,omitempty"`
@@ -204,6 +204,26 @@ type OpenLibertyApplicationServiceAccount struct {
 	SkipPullSecretValidation *bool `json:"skipPullSecretValidation,omitempty"`
 }
 
+type OpenLibertyApplicationProbesConfig struct {
+	OpenLibertyApplicationProbes `json:",inline"`
+
+	// Inspects the container filesystem to return health checks based upon files generated by the Liberty runtime. Only supported on Liberty version 25.0.0.6 or higher.
+	// +operator-sdk:csv:customresourcedefinitions:order=100,type=spec,displayName="Enable File Based",xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
+	EnableFileBased *bool `json:"enableFileBased,omitempty"`
+
+	// The Liberty container directory used to track file-based health checks for the 'live', 'ready', and 'started' files. Only used when .spec.probes.enableFiledBased is set to true. Defaults to /output/health.
+	// +operator-sdk:csv:customresourcedefinitions:order=101,type=spec,displayName="File Directory",xDescriptors="urn:alm:descriptor:com.tectonic.ui:text"
+	FileDirectory *string `json:"fileDirectory,omitempty"`
+
+	// The interval at which the Liberty runtime queries to update the file-based health check files. The value is a number followed by an optional time unit of ms for milliseconds or s for seconds. Only used when .spec.probes.enableFiledBased is set to true. Defaults to 5s.
+	// +operator-sdk:csv:customresourcedefinitions:order=102,type=spec,displayName="Check Interval",xDescriptors="urn:alm:descriptor:com.tectonic.ui:text"
+	CheckInterval *string `json:"checkInterval,omitempty"`
+
+	// The interval at which the Liberty runtime will query until an UP status is resolved and the health check file is created. The value is a number followed by an optional time unit of ms for milliseconds or s for seconds. If no time unit is specified for a value, the value is in milliseconds by default. Only used when .spec.probes.enableFiledBased is set to true. Defaults to 100ms.
+	// +operator-sdk:csv:customresourcedefinitions:order=103,type=spec,displayName="Startup Check Interval",xDescriptors="urn:alm:descriptor:com.tectonic.ui:text"
+	StartupCheckInterval *string `json:"startupCheckInterval,omitempty"`
+}
+
 // Define health checks on application container to determine whether it is alive or ready to receive traffic
 type OpenLibertyApplicationProbes struct {
 	// Periodic probe of container liveness. Container will be restarted if the probe fails.
@@ -478,11 +498,12 @@ type SemeruCompilerStatus struct {
 type OpenLibertyApplicationStatus struct {
 	// +listType=atomic
 	// +operator-sdk:csv:customresourcedefinitions:type=status,displayName="Status Conditions",xDescriptors="urn:alm:descriptor:io.kubernetes.conditions"
-	Conditions     []StatusCondition `json:"conditions,omitempty"`
-	Endpoints      []StatusEndpoint  `json:"endpoints,omitempty"`
-	RouteAvailable *bool             `json:"routeAvailable,omitempty"`
-	ImageReference string            `json:"imageReference,omitempty"`
-	Versions       StatusVersions    `json:"versions,omitempty"`
+	Conditions           []StatusCondition `json:"conditions,omitempty"`
+	Endpoints            []StatusEndpoint  `json:"endpoints,omitempty"`
+	RouteAvailable       *bool             `json:"routeAvailable,omitempty"`
+	ImageReference       string            `json:"imageReference,omitempty"`
+	PulledImageReference string            `json:"pulledImageReference,omitempty"`
+	Versions             StatusVersions    `json:"versions,omitempty"`
 
 	// +operator-sdk:csv:customresourcedefinitions:order=61,type=status,displayName="Service Binding"
 	Binding *corev1.LocalObjectReference `json:"binding,omitempty"`
@@ -964,12 +985,12 @@ func (rcss *OpenLibertyApplicationStatefulSet) GetAnnotations() map[string]strin
 	return rcss.Annotations
 }
 
-// GetImageReference returns Docker image reference to be deployed by the CR
+// GetImageReference returns the container image reference to be deployed by the CR
 func (s *OpenLibertyApplicationStatus) GetImageReference() string {
 	return s.ImageReference
 }
 
-// SetImageReference sets Docker image reference on the status portion of the CR
+// SetImageReference sets the container image reference on the status portion of the CR
 func (s *OpenLibertyApplicationStatus) SetImageReference(imageReference string) {
 	s.ImageReference = imageReference
 }

api/v1/zz_generated.deepcopy.go

@@ -702,12 +702,12 @@ func (rcss *OpenLibertyApplicationStatefulSet) GetAnnotations() map[string]strin
 	return rcss.Annotations
 }
 
-// GetImageReference returns Docker image reference to be deployed by the CR
+// GetImageReference returns the container image reference to be deployed by the CR
 func (s *OpenLibertyApplicationStatus) GetImageReference() string {
 	return s.ImageReference
 }
 
-// SetImageReference sets Docker image reference on the status portion of the CR
+// SetImageReference sets the container image reference on the status portion of the CR
 func (s *OpenLibertyApplicationStatus) SetImageReference(imageReference string) {
 	s.ImageReference = imageReference
 }