ci(stability): merge the release/2.9 branch

datacore-bolt-ci · datacore-bolt-ci · commit 562956654aeb · 2025-10-06T15:46:37.000Z
diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml
@@ -30,7 +30,7 @@ jobs:
         run: nix-shell --pure --run "./scripts/helm/test-template.sh" ./scripts/helm/shell.nix
 
   helm-images:
-    runs-on: ubuntu-latest-8-cores
+    runs-on: oracle-vm-8cpu-32gb-x86-64
     steps:
       - uses: actions/checkout@v4
         with:
@@ -48,6 +48,15 @@ jobs:
         run: nix-shell ./scripts/helm/shell.nix --run "echo"
       - name: Generate image list
         run: nix-shell ./scripts/helm/shell.nix --run "./scripts/helm/images.sh generate --dependency-update --exit-code"
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+        if: ${{ env.DOCKERHUB_USERNAME != '' }}
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: BootStrap k8s cluster
         if: github.event_name != 'pull_request'
         run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label"
diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml
@@ -4,7 +4,7 @@ on:
 
 jobs:
   k8s-ci:
-    runs-on: ubuntu-latest-8-cores
+    runs-on: oracle-vm-8cpu-32gb-x86-64
     steps:
       - name: Bind mount /dev/sda1 to /nix
         run: |
@@ -63,22 +63,3 @@ jobs:
           display-options: a
           fail-on-empty: true
           title: Test results
-
-  k8s-ci-vm:
-    runs-on: ubuntu-latest-8-cores
-    steps:
-      - uses: actions/checkout@v4
-      - uses: cachix/install-nix-action@v31.3.0
-        with:
-          enable_kvm: true
-      - name: Pre-populate nix-shell
-        run: |
-          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
-          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
-          nix-shell ./scripts/helm/shell.nix --run "echo"
-      - name: Test on VM
-        run: |
-          nix-shell ./scripts/helm/shell.nix --run "cd chart; helm dependency update"
-          # todo: this is broken currently, as it doesn't take into account changes
-          # to the local images!
-          # nix-build ./tests/helm/test.nix --option sandbox false
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
@@ -18,10 +18,10 @@ jobs:
       matrix:
         include:
           # NOTE: Update the .krew.yaml file (if required) when making changes here.
-          - os: ubuntu-latest-8-cores
+          - os: oracle-vm-8cpu-32gb-x86-64
             target: linux-musl
             arch: x86_64
-          - os: ubuntu-latest-8-cores
+          - os: oracle-vm-8cpu-32gb-x86-64
             target: linux-musl
             arch: aarch64
           - os: macos-14
@@ -30,7 +30,7 @@ jobs:
           - os: macos-13
             target: apple-darwin
             arch: x86_64
-          - os: ubuntu-latest-8-cores
+          - os: oracle-vm-8cpu-32gb-x86-64
             target: windows-gnu
             arch: x86_64
             suffix: .exe
diff --git a/chart/README.md b/chart/README.md
@@ -91,6 +91,7 @@ This removes all the Kubernetes components associated with the chart and deletes
 | agents.&ZeroWidthSpace;ha.&ZeroWidthSpace;cluster.&ZeroWidthSpace;resources.&ZeroWidthSpace;requests.&ZeroWidthSpace;cpu | Cpu requests for ha cluster agent | `"100m"` |
 | agents.&ZeroWidthSpace;ha.&ZeroWidthSpace;cluster.&ZeroWidthSpace;resources.&ZeroWidthSpace;requests.&ZeroWidthSpace;memory | Memory requests for ha cluster agent | `"16Mi"` |
 | agents.&ZeroWidthSpace;ha.&ZeroWidthSpace;node.&ZeroWidthSpace;logLevel | Log level for the ha node service | `"info"` |
+| agents.&ZeroWidthSpace;ha.&ZeroWidthSpace;node.&ZeroWidthSpace;port | Container port for the ha-node service | `50053` |
 | agents.&ZeroWidthSpace;ha.&ZeroWidthSpace;node.&ZeroWidthSpace;priorityClassName | Set PriorityClass, overrides global | `""` |
 | agents.&ZeroWidthSpace;ha.&ZeroWidthSpace;node.&ZeroWidthSpace;resources.&ZeroWidthSpace;limits.&ZeroWidthSpace;cpu | Cpu limits for ha node agent | `"100m"` |
 | agents.&ZeroWidthSpace;ha.&ZeroWidthSpace;node.&ZeroWidthSpace;resources.&ZeroWidthSpace;limits.&ZeroWidthSpace;memory | Memory limits for ha node agent | `"64Mi"` |
@@ -125,6 +126,7 @@ This removes all the Kubernetes components associated with the chart and deletes
 | base.&ZeroWidthSpace;logging.&ZeroWidthSpace;format | Valid values for format are pretty, json and compact | `"pretty"` |
 | base.&ZeroWidthSpace;logging.&ZeroWidthSpace;silenceLevel | Silence specific module components | `nil` |
 | base.&ZeroWidthSpace;metrics.&ZeroWidthSpace;enabled | Enable the metrics exporter | `true` |
+| base.&ZeroWidthSpace;metrics.&ZeroWidthSpace;port | Container port for the metrics exporter service | `9502` |
 | crds.&ZeroWidthSpace;csi.&ZeroWidthSpace;volumeSnapshots.&ZeroWidthSpace;enabled | Install Volume Snapshot CRDs | `true` |
 | crds.&ZeroWidthSpace;enabled | Disables the installation of all CRDs if set to false | `true` |
 | csi.&ZeroWidthSpace;controller.&ZeroWidthSpace;logLevel | Log level for the csi controller | `"info"` |
@@ -147,6 +149,7 @@ This removes all the Kubernetes components associated with the chart and deletes
 | csi.&ZeroWidthSpace;node.&ZeroWidthSpace;kubeletDir | The kubeletDir directory for the csi-node plugin | `"/var/lib/kubelet"` |
 | csi.&ZeroWidthSpace;node.&ZeroWidthSpace;nvme.&ZeroWidthSpace;ctrl_loss_tmo | The ctrl_loss_tmo (controller loss timeout) in seconds | `"1980"` |
 | csi.&ZeroWidthSpace;node.&ZeroWidthSpace;nvme.&ZeroWidthSpace;tcpFallback | Fallback to nvme-tcp if nvme-rdma is enabled for Mayastor but rdma is not available on a particular csi-node | `true` |
+| csi.&ZeroWidthSpace;node.&ZeroWidthSpace;port | Container port for the csi-node service | `10199` |
 | csi.&ZeroWidthSpace;node.&ZeroWidthSpace;priorityClassName | Set PriorityClass, overrides global | `""` |
 | csi.&ZeroWidthSpace;node.&ZeroWidthSpace;resources.&ZeroWidthSpace;limits.&ZeroWidthSpace;cpu | Cpu limits for csi node plugin | `"100m"` |
 | csi.&ZeroWidthSpace;node.&ZeroWidthSpace;resources.&ZeroWidthSpace;limits.&ZeroWidthSpace;memory | Memory limits for csi node plugin | `"128Mi"` |
@@ -182,6 +185,7 @@ This removes all the Kubernetes components associated with the chart and deletes
 | io_engine.&ZeroWidthSpace;nodeSelector | Node selectors to designate storage nodes for diskpool creation Note that if multi-arch images support 'kubernetes.io/arch: amd64' should be removed. | <pre>{<br>"kubernetes.io/arch":"amd64",<br>"openebs.io/engine":"mayastor"<br>}</pre> |
 | io_engine.&ZeroWidthSpace;nvme.&ZeroWidthSpace;ioTimeout | Timeout for IOs The default here is exaggerated for local disks, but we've observed that in shared virtual environments having a higher timeout value is beneficial. Please adjust this according to your hardware and needs. | `"110s"` |
 | io_engine.&ZeroWidthSpace;nvme.&ZeroWidthSpace;tcp.&ZeroWidthSpace;maxQueueDepth | You may need to increase this for a higher outstanding IOs per volume | `"32"` |
+| io_engine.&ZeroWidthSpace;port | Container port for the io-engine service | `10124` |
 | io_engine.&ZeroWidthSpace;priorityClassName | Set PriorityClass, overrides global | `""` |
 | io_engine.&ZeroWidthSpace;pstorRetries | Number of retries for pstor persistence before the volume target self shutdowns | `300` |
 | io_engine.&ZeroWidthSpace;resources.&ZeroWidthSpace;limits.&ZeroWidthSpace;cpu | Cpu limits for the io-engine | `""` |
diff --git a/chart/templates/mayastor/agents/ha/ha-node-daemonset.yaml b/chart/templates/mayastor/agents/ha/ha-node-daemonset.yaml
@@ -73,7 +73,7 @@ spec:
         - "--node-name=$(MY_NODE_NAME)"
         - "--csi-socket={{ default .Values.csi.node.pluginMountPath .Values.csi.node.pluginMounthPath }}/{{ .Values.csi.node.socketPath }}"
         - "--grpc-ip=$(MY_POD_IP)"
-        - "--grpc-port=50053"
+        - "--grpc-port={{ default 50053 .Values.agents.ha.node.port }}"
         - "--cluster-agent=https://{{ .Release.Name }}-agent-core:50052"{{ if .Values.base.jaeger.enabled }}
         - "--jaeger={{ include "jaeger_url" . }}"{{ end }}{{ if .Values.eventing.enabled }}
         - "--events-url=nats://{{ .Release.Name }}-nats:4222"
@@ -97,7 +97,7 @@ spec:
             cpu: {{ .Values.agents.ha.node.resources.requests.cpu | quote }}
             memory: {{ .Values.agents.ha.node.resources.requests.memory | quote }}
         ports:
-          - containerPort: 50053
+          - containerPort: {{ default 50053 .Values.agents.ha.node.port }}
             protocol: TCP
             name: ha-node
       volumes:
diff --git a/chart/templates/mayastor/csi/csi-node-daemonset.yaml b/chart/templates/mayastor/csi/csi-node-daemonset.yaml
@@ -92,7 +92,7 @@ spec:
         - "--enable-rest"{{ end }}
         - "--enable-registration"
         - "--grpc-ip=$(MY_POD_IP)"
-        - "--grpc-port=10199"{{ if .Values.csi.node.nvme.io_timeout }}
+        - "--grpc-port={{ default 10199 .Values.csi.node.port }}"{{ if .Values.csi.node.nvme.io_timeout }}
         - "--nvme-io-timeout={{ .Values.csi.node.nvme.io_timeout }}"
         - "--nvme-core-io-timeout={{ .Values.csi.node.nvme.io_timeout }}"{{ else }}
         - "--nvme-io-timeout={{ .Values.io_engine.nvme.ioTimeout }}10s"
@@ -146,7 +146,7 @@ spec:
             memory: "50Mi"
         # Mayastor node plugin gRPC server
         ports:
-        - containerPort: 10199
+        - containerPort: {{ default 10199 .Values.csi.node.port }}
           protocol: TCP
           name: mayastor-node
       volumes:
diff --git a/chart/templates/mayastor/io/io-engine-daemonset.yaml b/chart/templates/mayastor/io/io-engine-daemonset.yaml
@@ -55,10 +55,12 @@ spec:
             fieldRef:
               fieldPath: status.podIP
         ports:
-          - containerPort: 9502
+          - containerPort: {{ default 9502 .Values.base.metrics.port }}
             protocol: TCP
             name: metrics
         args:
+          - "--metrics-endpoint=[::]:{{ default 9502 .Values.base.metrics.port }}"
+          - "--grpc-port={{ default 10124 .Values.io_engine.port }}"
           - "--fmt-style={{ include "logFormat" . }}"
           - "--ansi-colors={{ .Values.base.logging.color }}"
       {{- end }}
@@ -100,6 +102,7 @@ spec:
         # Note: Ensure that the CPU resources are updated accordingly.
         #       If you use 2 CPUs, the CPU: field should also read 2.
         - "--grpc-ip=$(MY_POD_IP)"
+        - "--grpc-port={{ default 10124 .Values.io_engine.port }}"
         - "-N$(MY_NODE_NAME)"
         - "-Rhttps://{{ .Release.Name }}-agent-core:50051"
         - "-y/var/local/{{ .Release.Name }}/io-engine/config.yaml"
@@ -140,7 +143,7 @@ spec:
             memory: {{ .Values.io_engine.resources.requests.memory | quote }}
             hugepages-2Mi: {{ .Values.io_engine.resources.requests.hugepages2Mi | quote }}
         ports:
-        - containerPort: 10124
+        - containerPort: {{ default 10124 .Values.io_engine.port }}
           protocol: TCP
           name: io-engine
       volumes:
diff --git a/chart/values.yaml b/chart/values.yaml
@@ -87,6 +87,8 @@ base:
   metrics:
     # -- Enable the metrics exporter
     enabled: true
+    # -- Container port for the metrics exporter service
+    port: 9502
   jaeger:
     # Enable jaeger tracing (for development only).
     # Since version 1.31 the Jaeger Operator uses webhooks to validate Jaeger custom resources (CRs).
@@ -233,6 +235,8 @@ agents:
       tolerations: []
       # -- Set PriorityClass, overrides global
       priorityClassName: ""
+      # -- Container port for the ha-node service
+      port: 50053
     cluster:
       # -- Log level for the ha cluster service
       logLevel: info
@@ -398,6 +402,8 @@ csi:
       containers:
         - name: nvme-tcp-probe
           command: ['sh', '-c', 'trap "exit 1" TERM; until [ -d /sys/module/nvme_tcp ]; do [ -z "$WARNED" ] && echo "nvme_tcp module not loaded..."; WARNED=1; sleep 60; done;']
+    # -- Container port for the csi-node service
+    port: 10199
 io_engine:
   # -- Log level for the io-engine service
   logLevel: info
@@ -476,6 +482,8 @@ io_engine:
   runtimeClassName: ""
   # -- Number of retries for pstor persistence before the volume target self shutdowns
   pstorRetries: 300
+  # -- Container port for the io-engine service
+  port: 10124
 etcd:
   # -- Disable when using an external etcd cluster.
   enabled: true
diff --git a/dependencies/control-plane b/dependencies/control-plane
@@ -1 +1 @@
-Subproject commit 7bf12b3791829844884c5260ee030b185a30b3fc
+Subproject commit e99789076ecf497b2cd78b8b099ccbdf186e9a9b
diff --git a/metrics-exporter/src/bin/io_engine/client/grpc_client.rs b/metrics-exporter/src/bin/io_engine/client/grpc_client.rs
@@ -105,14 +105,14 @@ impl GrpcClient {
 }
 
 /// Initialize mayastor grpc client.
-pub(crate) async fn init_client() -> Result<GrpcClient, ExporterError> {
+pub(crate) async fn init_client(grpc_port: u16) -> Result<GrpcClient, ExporterError> {
     let timeout = Timeouts::new(Duration::from_secs(1), Duration::from_secs(5));
     let pod_ip = get_pod_ip()?;
     let _ = get_node_name()?;
 
     let endpoint = Uri::builder()
         .scheme("https")
-        .authority(SocketAddr::new(pod_ip, 10124).to_string())
+        .authority(SocketAddr::new(pod_ip, grpc_port).to_string())
         .path_and_query("")
         .build()
         .map_err(|error| ExporterError::InvalidURI(error.to_string()))?;
diff --git a/metrics-exporter/src/bin/io_engine/main.rs b/metrics-exporter/src/bin/io_engine/main.rs
@@ -49,6 +49,10 @@ pub(crate) struct Cli {
     #[clap(long, short, default_value = "[::]:9502")]
     metrics_endpoint: SocketAddr,
 
+    /// Port for the io-engine gRPC server running on the same pod.
+    #[clap(long, default_value_t = 10124)]
+    grpc_port: u16,
+
     /// Formatting style to be used while logging.
     #[clap(default_value = FmtStyle::Pretty.as_ref(), short, long)]
     fmt_style: FmtStyle,
@@ -85,7 +89,7 @@ async fn main() -> Result<(), ExporterError> {
         .init("metrics-exporter-io_engine");
 
     initialize_cache().await;
-    let client = init_client().await?;
+    let client = init_client(args.grpc_port).await?;
     // Initialize io engine gRPC client.
     GRPC_CLIENT
         .set(client)
diff --git a/scripts/helm/install.sh b/scripts/helm/install.sh
@@ -29,6 +29,8 @@ SCRIPT_DIR="$(dirname "$0")"
 CHART_DIR="$SCRIPT_DIR"/../../chart
 CHART_SOURCE=$CHART_DIR
 DEP_UPDATE=
+INS_LOKI="true"
+HELM_UPGRADE="false"
 RELEASE_NAME="mayastor"
 K8S_NAMESPACE="mayastor"
 FAIL_IF_INSTALLED=
@@ -49,12 +51,14 @@ Options:
   --wait                         Wait for helm to complete install.
   --dry-run                      Don't run any commands, output them only.
   --helm-dry-run                 Install helm with --dry-run.
+  --upgrade                      Upgrades an existing installation.
   --dep-update                   Run helm dependency update.
   --fail-if-installed            Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace.
   --hosted-chart                 Install a hosted chart instead of the local chart.
   --version   <version>          Set the version/version-range for the chart. Works only when used with the '--hosted' option.
   --registry  <registry-url>     Set the registry URL for the hosted chart. Works only when used with the '--hosted' option. (Default: $DEFAULT_REGISTRY)
   --pull-policy <policy>         Set the image pull policy.
+  --no-loki                      Don't deploy Loki.
   --helm      <stringArray>      Pass Helm Args directly to the install/upgrade commands.
 
 Examples:
@@ -72,6 +76,28 @@ die() {
   exit "${_return}"
 }
 
+loki_args() {
+  local replicas=1
+  local minio="true"
+  local mode="standalone"
+  if [ "$replicas" != 1 ]; then
+    mode="distributed"
+  fi
+
+  if [ "$replicas" = "0" ] || [ "$INS_LOKI" = "false" ]; then
+    echo -n "loki.enabled=false,alloy.enabled=false"
+    return 0
+  fi
+
+  echo -n "loki.loki.commonConfig.replication_factor=$replicas,loki.singleBinary.replicas=$replicas,loki.minio.mode=$mode,"
+
+  if [ "$minio" = "true" ]; then
+    echo -n "loki.minio.enabled=true,loki.minio.replicas=$replicas"
+  else
+    echo -n "loki.loki.storage.type=filesystem,loki.singleBinary.persistence.enabled=true,loki.minio.enabled=false"
+  fi
+}
+
 while [ "$#" -gt 0 ]; do
   case $1 in
     -h|--help)
@@ -94,6 +120,9 @@ while [ "$#" -gt 0 ]; do
       HELM="echo $HELM"
       KUBECTL="echo $KUBECTL"
       shift;;
+    --upgrade)
+      HELM_UPGRADE="true"
+      shift;;
     --dep-update)
       DEP_UPDATE="y"
       shift;;
@@ -126,6 +155,9 @@ while [ "$#" -gt 0 ]; do
       test $# -lt 1 && die "Missing pull policy"
       PULL_POLICY="$1"
       shift;;
+    --no-loki)
+      INS_LOKI="false"
+      shift;;
     --helm)
       shift
       test $# -lt 1 && die "Missing helm args"
@@ -163,20 +195,28 @@ if [ -n "$PULL_POLICY" ]; then
 fi
 
 if [ -z "$DRY_RUN" ] && [ "$($HELM ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
+  INS_EXISTS="true"
   already_exists_log="Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE"
   if [ -n "$FAIL_IF_INSTALLED" ]; then
     die "ERROR: $already_exists_log" 1
   fi
   echo "$already_exists_log"
-else
-  echo "Installing Mayastor Chart"
+fi
+if [ "${INS_EXISTS:-"false"}" = "false" ] || [ "$HELM_UPGRADE" = "true" ]; then
+  if [ "$HELM_UPGRADE" = "true" ]; then
+    echo "Upgrading Mayastor Chart"
+    action="upgrade"
+  else
+    echo "Installing Mayastor Chart"
+    action="install"
+  fi
+  LOKI_ARGS="--set="$(loki_args)""
   set -x
-  $HELM install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \
+  $HELM $action "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \
        --set="etcd.livenessProbe.initialDelaySeconds=5,etcd.readinessProbe.initialDelaySeconds=5,etcd.replicaCount=1" \
        --set="obs.callhome.enabled=true,obs.callhome.sendReport=false,localpv-provisioner.analytics.enabled=false" \
        --set="eventing.enabled=true,nats.cluster.enabled=false,nats.cluster.replicas=1" \
-       --set="loki.loki.commonConfig.replication_factor=2,loki.singleBinary.replicas=2,loki.minio.replicas=2" \
-       $HELM_DRY_RUN $WAIT_ARG $PULL_POLICY_ARG $DEP_UPDATE_ARG $VERSION_ARG ${HELM_ARGS:-}
+       $LOKI_ARGS $HELM_DRY_RUN $WAIT_ARG $PULL_POLICY_ARG $DEP_UPDATE_ARG $VERSION_ARG ${HELM_ARGS:-}
   set +x
 fi
 
diff --git a/scripts/k8s/images.sh b/scripts/k8s/images.sh
