Support 1password in all opensearch-build jenkins workflows

jenkins/docker/docker-build.jenkinsfile

@@ -22,6 +22,11 @@ def agent_nodes = [
     "windows": "Jenkins-Agent-Windows2019-X64-M54xlarge-Docker-Builder",
 ]
 
+def secret_dockerhub_staging = [
+    [envVar: 'DOCKER_USERNAME', secretRef: 'op://opensearch-infra-secrets/dockerhub-staging-credentials/username'],
+    [envVar: 'DOCKER_PASSWORD', secretRef: 'op://opensearch-infra-secrets/dockerhub-staging-credentials/password']
+]
+
 pipeline {
     options {
         timeout(time: 5, unit: 'HOURS')
@@ -77,9 +82,8 @@ pipeline {
                 script {
                     echo 'The docker-build workflow will only push docker images to staging, please use docker-copy to move the image to other repositories'
                     checkout([$class: 'GitSCM', branches: [[name: "${DOCKER_BUILD_GIT_REPOSITORY_REFERENCE}" ]], userRemoteConfigs: [[url: "${DOCKER_BUILD_GIT_REPOSITORY}" ]]])
-                    def CREDENTIAL_ID = "jenkins-staging-dockerhub-credential"
-                    sh("echo Account: ${CREDENTIAL_ID}")
-                    withCredentials([usernamePassword(credentialsId: CREDENTIAL_ID, usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
+                    echo "Account: dockerhub staging"
+                    withSecrets(secrets: secret_dockerhub_staging){
                         if (isUnix()){
                             sh """
                                 set -e

jenkins/gradle/gradle-check.jenkinsfile

@@ -7,6 +7,11 @@
  * compatible open source license.
  */
 
+def secret_dockerhub_readonly = [
+    [envVar: 'DOCKER_USERNAME', secretRef: 'op://opensearch-infra-secrets/dockerhub-production-readonly-credentials/username'],
+    [envVar: 'DOCKER_PASSWORD', secretRef: 'op://opensearch-infra-secrets/dockerhub-production-readonly-credentials/password']
+]
+
 lib = library(identifier: '[email protected]', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
@@ -102,10 +107,9 @@ pipeline {
 
                     def agent_name_array = params.AGENT_LABEL.tokenize('-')
                     def agent_name = agent_name_array[2] + " " + agent_name_array[4]
-                    def CREDENTIAL_ID = 'jenkins-production-dockerhub-credential-readonly'
 
                     echo("Build Cause: ${BUILD_CAUSE}")
-                    withCredentials([usernamePassword(credentialsId: CREDENTIAL_ID, usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
+                    withSecrets(secrets: secret_dockerhub_readonly){
                         def bwc_checkout_align = "false"
 
                         def dockerLogin = sh(returnStdout: true, script: "set +x && (echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin) || echo docker error").trim()

jenkins/manifests-update.jenkinsFile

@@ -7,6 +7,11 @@
  * compatible open source license.
  */
 
+def secret_github_bot = [
+    [envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
+    [envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
+]
+
 pipeline {
     options {
         timeout(time: 1, unit: 'HOURS')
@@ -29,7 +34,7 @@ pipeline {
             }
             steps {
                 script {
-                    withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
+                    withSecrets(secrets: secret_github_bot){
                         try {
                             sh """
                                 set +x

jenkins/vulnerability-scan/whitesource-scan.jenkinsfile

@@ -7,6 +7,10 @@
  * compatible open source license.
  */
 
+def secret_mend_scan = [
+    [envVar: 'wss_apikey', secretRef: 'op://opensearch-infra-secrets/mend-scan/wss-token']
+]
+
 pipeline {
     agent none
     options {
@@ -29,7 +33,7 @@ pipeline {
             }
             steps {
                 script {
-                    withCredentials([string(credentialsId: 'jenkins-whitesource-scan-apikey', variable: 'wss_apikey')]) {
+                    withSecrets(secrets: secret_mend_scan){
                         echo 'Executing WhiteSource Scan script'
                         scan()
                     }

tests/jenkins/TestDockerBuild.groovy

@@ -48,6 +48,10 @@ class TestDockerBuild extends BuildPipelineTest {
         binding.setVariable('DOCKER_BUILD_SCRIPT_WITH_COMMANDS', dockerBuildScriptwithCommands)
         binding.setVariable('DOCKER_BUILD_OS', dockerBuildOS)
         helper.registerAllowedMethod('isUnix', [], { true })
+        helper.registerAllowedMethod("withSecrets", [Map, Closure], { args, closure ->
+            closure.delegate = delegate
+            return helper.callClosure(closure)
+        })
 
     }
 
@@ -65,12 +69,12 @@ class TestDockerBuild extends BuildPipelineTest {
 
         // Ensure the entire docker command is executed in an external shell script exactly once
         def dockerLoginCommand = getCommands('docker').findAll {
-            shCommand -> shCommand.contains('docker logout && echo DOCKER_PASSWORD | docker login -u DOCKER_USERNAME --password-stdin && eval bash docker/ci/build-image-multi-arch.sh -v <TAG_NAME> -f <DOCKERFILE PATH>')
+            shCommand -> shCommand.contains('docker logout && echo dockerPassword | docker login -u dockerUsername --password-stdin && eval bash docker/ci/build-image-multi-arch.sh -v <TAG_NAME> -f <DOCKERFILE PATH>')
         }
         assertThat(dockerLoginCommand.size(), equalTo(1))
 
         // Validate the docker-build.sh is called with correct predefined credential
-        assertCallStack().contains("docker-build.sh(echo Account: jenkins-staging-dockerhub-credential)")
+        assertCallStack().contains("docker-build.echo(Account: dockerhub staging)")
 
         // Make sure dockerBuildOS is deciding agent_node docker_nodes docker_args correctly
         assertCallStack().contains("docker-build.echo(Executing on agent [docker:[alwaysPull:true, args:-u root -v /var/run/docker.sock:/var/run/docker.sock, containerPerStageRoot:false, label:Jenkins-Agent-Ubuntu2404-X64-M52xlarge-Docker-Builder, image:opensearchstaging/ci-runner:ubuntu2404-x64-docker-buildx0.9.1-qemu8.2-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])")

tests/jenkins/TestWhiteSourceScan.groovy

@@ -30,6 +30,11 @@ class TestWhileSourceScan extends BuildPipelineTest {
                 .build()
             )
         super.setUp()
+        binding.setVariable('wss_apikey', 'wss_apikey')
+        helper.registerAllowedMethod("withSecrets", [Map, Closure], { args, closure ->
+            closure.delegate = delegate
+            return helper.callClosure(closure)
+        })
     }
 
     @Test

tests/jenkins/jenkinsjob-regression-files/docker/docker-build.jenkinsfile.txt

@@ -9,14 +9,13 @@
             docker-build.script(groovy.lang.Closure)
                docker-build.echo(The docker-build workflow will only push docker images to staging, please use docker-copy to move the image to other repositories)
                docker-build.checkout({$class=GitSCM, branches=[{name=main}], userRemoteConfigs=[{url=https://github.com/opensearch-project/opensearch-build}]})
-               docker-build.sh(echo Account: jenkins-staging-dockerhub-credential)
-               docker-build.usernamePassword({credentialsId=jenkins-staging-dockerhub-credential, usernameVariable=DOCKER_USERNAME, passwordVariable=DOCKER_PASSWORD})
-               docker-build.withCredentials([[DOCKER_USERNAME, DOCKER_PASSWORD]], groovy.lang.Closure)
+               docker-build.echo(Account: dockerhub staging)
+               docker-build.withSecrets({secrets=[{envVar=DOCKER_USERNAME, secretRef=op://opensearch-infra-secrets/dockerhub-staging-credentials/username}, {envVar=DOCKER_PASSWORD, secretRef=op://opensearch-infra-secrets/dockerhub-staging-credentials/password}]}, groovy.lang.Closure)
                   docker-build.isUnix()
                   docker-build.sh(
                                 set -e
                                 set +x
-                                docker logout && echo DOCKER_PASSWORD | docker login -u DOCKER_USERNAME --password-stdin && eval bash docker/ci/build-image-multi-arch.sh -v <TAG_NAME> -f <DOCKERFILE PATH>
+                                docker logout && echo dockerPassword | docker login -u dockerUsername --password-stdin && eval bash docker/ci/build-image-multi-arch.sh -v <TAG_NAME> -f <DOCKERFILE PATH>
                             )
          docker-build.script(groovy.lang.Closure)
             docker-build.cleanWs({disableDeferredWipeout=true, deleteDirs=true})

tests/jenkins/jenkinsjob-regression-files/vulnerability-scan/whitesource-scan.jenkinsfile.txt

@@ -6,8 +6,7 @@
          whitesource-scan.stage(Run the scan, groovy.lang.Closure)
             whitesource-scan.echo(Executing on agent [docker:[alwaysPull:true, args:-e LANG=en_US.UTF-8, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-M54xlarge-Docker-Host, image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
             whitesource-scan.script(groovy.lang.Closure)
-               whitesource-scan.string({credentialsId=jenkins-whitesource-scan-apikey, variable=wss_apikey})
-               whitesource-scan.withCredentials([wss_apikey], groovy.lang.Closure)
+               whitesource-scan.withSecrets({secrets=[{envVar=wss_apikey, secretRef=op://opensearch-infra-secrets/mend-scan/wss-token}]}, groovy.lang.Closure)
                   whitesource-scan.echo(Executing WhiteSource Scan script)
                   whitesource-scan.sh(
         curl -SLO https://services.gradle.org/distributions/gradle-7.4.2-bin.zip