issue: 4007026 - [CI] Release pipeline

NirWolfer · AlexanderGrissik · commit 25f17b3d0f56 · 2024-08-06T10:31:10.000+03:00
Unti now, release process is done manually by the dev team. This process
involves creating a tag on a commit that needs to be released, building
that tag and copying the files to NFS location

The idea is to create a jenkins pipeline that will do the entire process
with given inputs, This commit will commence phase 1 - doing the build
and copying of packages created to NFS.
THE CREATION OF A RELEASE TAG ON A COMMIT IS NOT PART OF THIS PR

Add release_jjb and the corosponding matrix file for a new pipeline
definition, the release is run on rhel8.6
Add do_release.sh script to run the build and NFS copy to release the
package
Add Dockerfile for release container with all dependencies installed

Issue: HPCINFRA-1889
Signed-off-by: NirWolfer &lt;nwolfer@nvidia.com&gt;
diff --git a/.ci/do_release.sh b/.ci/do_release.sh
@@ -0,0 +1,86 @@
+#!/bin/bash -Exel
+
+echo "**********************************"
+echo "Starting do_release.sh script..."
+echo "**********************************"
+
+set -o pipefail
+
+print_help() {
+    set +xv  
+    echo -e "\n\n"
+    echo "--------------------------------------------------"
+    echo "Usage: release_folder=<release folder> release_tag=<release tag> [revision=<revision>] [do_release=<true|false>] $0"
+    echo "       Where release folder is a path to NFS folder to copy the package into"
+    echo "       Where release tag is a git tag to release (must be already tagged in the git repo)"
+    echo "       Where revision is a number postfix to add the to package indicating which version of the tag it is - OPTIONAL, default value 1"
+    echo "       Where do_release is a boolean value indicating if the script will copy the created package into the release folder location - OPTIONAL, default value false"
+    exit 1
+}
+
+if [ -z "${release_folder}" ]; then
+    echo "ERROR: 'release_folder' was not set."
+    print_help
+fi
+
+if [ ! -e "${release_folder}" ] || [ ! -d "${release_folder}" ]; then
+    echo "ERROR: [${release_folder}] directory doesn't exist."
+    print_help
+fi
+
+if [ -z "${release_tag}" ]; then
+    echo "ERROR: 'release_tag' was not set."
+    print_help
+fi
+
+if [ -z "${revision}" ]; then
+    echo "WARN: 'revision' was not set, defaulting to 1"
+    revision=1
+fi
+
+if [ -z "${do_release}" ]; then
+    echo "WARN: 'do_release' was not set, defaulting to false (package will not be release)"
+    do_release=false
+fi
+
+env PRJ_RELEASE="${revision}" contrib/build_pkg.sh -s
+
+MAJOR_VERSION=$(grep -e "define(\[prj_ver_major\]" configure.ac | awk '{ printf $2 };' | sed  's/)//g')
+MINOR_VERSION=$(grep -e "define(\[prj_ver_minor\]" configure.ac | awk '{ printf $2 };' | sed  's/)//g')
+REVISION_VERSION=$(grep -e "define(\[prj_ver_revision\]" configure.ac | awk '{ printf $2 };' | sed  's/)//g')
+configure_ac_version="${MAJOR_VERSION}.${MINOR_VERSION}.${REVISION_VERSION}"
+echo "FULL_VERSION from configure.ac: [${configure_ac_version}]"
+
+if [[ "${release_tag}" != "${configure_ac_version}" ]]; then
+    echo "ERROR: FULL_VERSION: ${configure_ac_version} from configure.ac doesn't match tag: ${release_tag} provided! Exit"
+    exit 1
+fi
+
+if [ "${do_release}" = true ] ; then
+    echo "do_release is set to true, will release package into ${release_folder}/libvma_v_${release_tag}-0/src"
+
+    cd pkg/packages || { echo "pkg folder is missing, exiting..."; exit 1; }
+    pkg_name=$(ls -1 libvma-"${release_tag}"-"${revision}".src.rpm)
+    DST_DIR=${release_folder}/libvma_v_${release_tag}-0/src
+
+    if [[ -e "${DST_DIR}/${pkg_name}" ]]; then 
+        echo "ERROR: [${DST_DIR}/${pkg_name}] file already exist. Exit"
+        exit 1
+    fi
+
+    sudo -E -u swx-jenkins mkdir -p "$DST_DIR"
+    sudo -E -u swx-jenkins cp -v "${pkg_name}" "$DST_DIR"
+
+    cd "${release_folder}"
+    sudo -E -u swx-jenkins ln -s "$DST_DIR/${pkg_name}" latest_release
+
+    echo "Release found at $DST_DIR"
+
+else
+     echo "do_release is set to false, skipping package release."
+fi
+
+set +x
+echo "**********************************"
+echo "Finished do_release.sh script..."
+echo "**********************************"
diff --git a/.ci/dockerfiles/Dockerfile.rhel8.6.release b/.ci/dockerfiles/Dockerfile.rhel8.6.release
@@ -0,0 +1,27 @@
+ARG HARBOR_URL=harbor.mellanox.com
+FROM $HARBOR_URL/hpcx/x86_64/rhel8.6/core:latest
+ARG _UID=6213
+ARG _GID=101
+ARG _LOGIN=swx-jenkins
+ARG _HOME=/var/home/$_LOGIN
+
+RUN sed -i 's/mirrorlist/#mirrorlist/;s!#baseurl=http://mirror.centos.org!baseurl=http://vault.centos.org!' /etc/yum.repos.d/* && \
+    echo "[mlnx-opt]" > /etc/yum.repos.d/mlnx-opt.repo && \
+    echo "name=RHEL 8.6 mirror" >> /etc/yum.repos.d/mlnx-opt.repo && \
+    echo "baseurl=http://webrepo.mtr.labs.mlnx/RH/optional/8.6/x86_64/" >> /etc/yum.repos.d/mlnx-opt.repo && \
+    echo "enabled=1" >> /etc/yum.repos.d/mlnx-opt.repo && \
+    echo "gpgcheck=0" >> /etc/yum.repos.d/mlnx-opt.repo && \
+    yum makecache
+
+RUN echo "${_LOGIN} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
+    echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
+    mkdir -p ${_HOME} && \
+    groupadd -f -g "$_GID" "$_LOGIN" && \
+    useradd -u "$_UID" -g "$_GID" -s /bin/bash -m -d ${_HOME} "${_LOGIN}" && \
+    chown -R ${_LOGIN} ${_HOME}
+
+RUN yum install --allowerasing -y \
+    git autoconf automake libtool gcc \
+    sudo gcc-c++ libibverbs-devel rdma-core \
+    librdmacm unzip patch wget make \
+    libnl3-devel rpm-build
diff --git a/.ci/pipeline/release_jjb.yaml b/.ci/pipeline/release_jjb.yaml
@@ -0,0 +1,65 @@
+- job-template:
+    name: "{jjb_proj}"
+    project-type: pipeline
+    folder: libvma
+    properties:
+        - github:
+             url: "https://github.com/Mellanox/libvma"
+        - build-discarder:
+            days-to-keep: 50
+            num-to-keep: 20
+        - inject:
+            keep-system-variables: true
+            properties-content: |
+              jjb_proj={jjb_proj}
+    description: Do NOT edit this job through the Web GUI !
+    concurrent: false
+    parameters:
+        - string:
+            name: "release_tag"
+            default: ""
+            description: "Tag to release"
+        - string:
+            name: sha1
+            default: 'tags/$release_tag'
+            description: 'commit to use, defaults to the commit tag provided under release_tag'
+        - string:
+            name: "revision"
+            default: "1"
+            description: "Release revision"
+        - string:
+            name: "release_folder"
+            default: "/auto/mswg/release/vma"
+            description: "Folder to release packages into"
+        - bool:
+            name: "do_release"
+            default: true
+            description: "Release build packges into the release folder, set to false for debugging"
+        - string:
+            name: "notification_email"
+            default: "55ae26f6.NVIDIA.onmicrosoft.com@amer.teams.ms"
+            description: "Email to send report to upon success/failure"
+        - string:
+            name: "conf_file"
+            default: ".ci/pipeline/release_matrix_job.yaml"
+            description: "job config file. Do not change it"
+    pipeline-scm:
+        scm:
+            - git:
+                url: "{jjb_git}"
+                credentials-id: 'b7d08ca7-378c-45d6-ac4b-3f30bdf49168'
+                branches: ['$sha1']
+                shallow-clone: true
+                depth: 2
+                refspec: "+refs/pull/*:refs/remotes/origin/pr/*"
+                browser: githubweb
+                browser-url: "{jjb_git}"
+        script-path: ".ci/Jenkinsfile"
+- project:
+    name: libvma
+    jjb_email: 'nwolfer@nvidia.com'
+    jjb_proj: 'LibVMA-release'
+    jjb_git: 'git@github.com:Mellanox/libvma.git'
+    jjb_owner: 'Nir Wolfer'
+    jobs:
+        - "{jjb_proj}"
diff --git a/.ci/pipeline/release_matrix_job.yaml b/.ci/pipeline/release_matrix_job.yaml
@@ -0,0 +1,54 @@
+---
+job: LibVMA-release
+registry_host: harbor.mellanox.com
+registry_auth: 1daaea28-800e-425f-a91f-3bd3e9136eea
+registry_path: /swx-infra/media/libvma
+
+kubernetes:
+  privileged: true
+  cloud: il-ipp-blossom-prod
+  nodeSelector: 'beta.kubernetes.io/os=linux'
+  namespace: swx-media
+  limits: '{memory: 8Gi, cpu: 7000m}'
+  requests: '{memory: 8Gi, cpu: 7000m}'
+
+env:
+  MAIL_FROM: jenkins@nvidia.com
+
+volumes:
+  # Default release location
+  - {mountPath: /auto/mswg/release/vma, hostPath: /auto/mswg/release/vma}
+  # User profile for release
+  - {mountPath: /var/home/swx-jenkins, hostPath: /labhome/swx-jenkins}
+
+runs_on_dockers:
+  - {file: '.ci/dockerfiles/Dockerfile.rhel8.6.release', name: 'rhel8.6', uri: '$arch/$name/release', build_args: '--no-cache', arch: 'x86_64', tag: '20240718'}
+
+steps:
+  - name: Release
+    parallel: false
+    run: |
+      .ci/do_release.sh
+    archiveArtifacts: pkg/build_pkg.log,pkg/packages/*.rpm
+
+pipeline_start:
+    shell: action
+    module: groovy
+    run: |
+      echo "Starting release process for LibVMA-${release_tag}"
+      currentBuild.displayName += "-${release_tag}"
+
+pipeline_stop:
+    shell: action
+    module: groovy
+    run: |
+      if (!params.notification_email.isEmpty()) {
+        mail from: "${MAIL_FROM}",
+          mimeType: 'text/html',
+          to: "${notification_email}",
+          subject: "Release build ended for LibVMA - ${release_tag}",
+          body: """
+            <p><b>Tag:</b> ${release_tag}</p>
+            <p><b>Build url:</b> <a href=${currentBuild.absoluteUrl}>link</a></p>
+            <p><b>Status:</b> ${currentBuild.currentResult}</p>"""
+      }
