Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions jenkins/docker/docker-build.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ def agent_nodes = [
"windows": "Jenkins-Agent-Windows2019-X64-M54xlarge-Docker-Builder",
]

def secret_dockerhub_staging = [
[envVar: 'DOCKER_USERNAME', secretRef: 'op://opensearch-infra-secrets/dockerhub-staging-credentials/username'],
[envVar: 'DOCKER_PASSWORD', secretRef: 'op://opensearch-infra-secrets/dockerhub-staging-credentials/password']
]

pipeline {
options {
timeout(time: 5, unit: 'HOURS')
Expand Down Expand Up @@ -77,9 +82,8 @@ pipeline {
script {
echo 'The docker-build workflow will only push docker images to staging, please use docker-copy to move the image to other repositories'
checkout([$class: 'GitSCM', branches: [[name: "${DOCKER_BUILD_GIT_REPOSITORY_REFERENCE}" ]], userRemoteConfigs: [[url: "${DOCKER_BUILD_GIT_REPOSITORY}" ]]])
def CREDENTIAL_ID = "jenkins-staging-dockerhub-credential"
sh("echo Account: ${CREDENTIAL_ID}")
withCredentials([usernamePassword(credentialsId: CREDENTIAL_ID, usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
echo "Account: dockerhub staging"
withSecrets(secrets: secret_dockerhub_staging){
if (isUnix()){
sh """
set -e
Expand Down
8 changes: 6 additions & 2 deletions jenkins/gradle/gradle-check.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
* compatible open source license.
*/

def secret_dockerhub_readonly = [
[envVar: 'DOCKER_USERNAME', secretRef: 'op://opensearch-infra-secrets/dockerhub-production-readonly-credentials/username'],
[envVar: 'DOCKER_PASSWORD', secretRef: 'op://opensearch-infra-secrets/dockerhub-production-readonly-credentials/password']
]

lib = library(identifier: '[email protected]', retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
Expand Down Expand Up @@ -102,10 +107,9 @@ pipeline {

def agent_name_array = params.AGENT_LABEL.tokenize('-')
def agent_name = agent_name_array[2] + " " + agent_name_array[4]
def CREDENTIAL_ID = 'jenkins-production-dockerhub-credential-readonly'

echo("Build Cause: ${BUILD_CAUSE}")
withCredentials([usernamePassword(credentialsId: CREDENTIAL_ID, usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
withSecrets(secrets: secret_dockerhub_readonly){
def bwc_checkout_align = "false"

def dockerLogin = sh(returnStdout: true, script: "set +x && (echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin) || echo docker error").trim()
Expand Down
7 changes: 6 additions & 1 deletion jenkins/manifests-update.jenkinsFile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
* compatible open source license.
*/

def secret_github_bot = [
[envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
[envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
]

pipeline {
options {
timeout(time: 1, unit: 'HOURS')
Expand All @@ -29,7 +34,7 @@ pipeline {
}
steps {
script {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
withSecrets(secrets: secret_github_bot){
try {
sh """
set +x
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ lib = library(identifier: '[email protected]', retriever: modernSCM([
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_github_bot = [
[envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
[envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
]

pipeline {
options {
timeout(time: 4, unit: 'HOURS')
Expand Down Expand Up @@ -1072,7 +1077,7 @@ def triggerDistributionValidationWorkflow(String version) {
}

def triggerNightlyPlayground() {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
withSecrets(secrets: secret_github_bot){
println("Triggering nightly playground workflow.")
try {
sh(
Expand Down
8 changes: 6 additions & 2 deletions jenkins/opensearch/benchmark-compare.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,15 @@
*/

lib = library(identifier: '[email protected]', retriever: modernSCM([

$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_github_bot = [
[envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
[envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
]

pipeline {
agent { label 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test' }
options {
Expand Down Expand Up @@ -85,7 +89,7 @@ pipeline {
"""
script {
if ("${PULL_REQUEST_NUMBER}" != '' && "${REPOSITORY}" != '') {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
withSecrets(secrets: secret_github_bot){
def pull_request = Integer.parseInt("${PULL_REQUEST_NUMBER}")
sh ("gh pr comment ${pull_request} --repo ${REPOSITORY} --body-file final_result_${BUILD_NUMBER}.md")
}
Expand Down
16 changes: 12 additions & 4 deletions jenkins/opensearch/benchmark-pull-request.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,19 @@
*/

lib = library(identifier: '[email protected]', retriever: modernSCM([

$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_github_bot = [
[envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
[envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
]

def secret_aws_account = [
[envVar: 'PERF_TEST_ACCOUNT_ID', secretRef: 'op://opensearch-infra-secrets/aws-accounts/perf-test-account-id']
]

pipeline {
agent {label 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test'}
options {
Expand Down Expand Up @@ -274,7 +282,7 @@ pipeline {
script {
if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) {
if ("${pull_request_number}" != '' && "${repository}" != '') {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
withSecrets(secrets: secret_github_bot){
def pull_request = Integer.parseInt("${pull_request_number}")
sh ("gh pr comment ${pull_request} --repo ${repository} --body-file final_result_${BUILD_NUMBER}.md")
}
Expand Down Expand Up @@ -306,7 +314,7 @@ pipeline {
script {
if (currentBuild.rawBuild.getCauses().toString().contains("GenericCause")) {
if ("${pull_request_number}" != '' && "${repository}" != '') {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
withSecrets(secrets: secret_github_bot){
def pull_request = Integer.parseInt("${pull_request_number}")
sh ("gh pr comment ${pull_request} --repo ${repository} --body \"The benchmark job ${BUILD_URL} failed.\n Please see logs to debug.\"")
}
Expand All @@ -321,7 +329,7 @@ pipeline {
def stackNames = [
"opensearch-infra-stack-${BUILD_NUMBER}"
]
withCredentials([string(credentialsId: 'perf-test-account-id', variable: 'PERF_TEST_ACCOUNT_ID')]) {
withSecrets(secrets: secret_aws_account){
withAWS(role: 'cfn-set-up', roleAccount: "${PERF_TEST_ACCOUNT_ID}", duration: 900, roleSessionName: 'jenkins-session', region: 'us-east-1') {
try {
for (String stackName : stackNames) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ lib = library(identifier: '[email protected]', retriever: modernSCM([
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_aws_account = [
[envVar: 'PERF_TEST_ACCOUNT_ID', secretRef: 'op://opensearch-infra-secrets/aws-accounts/perf-test-account-id']
]

pipeline {
agent none
options {
Expand Down Expand Up @@ -437,7 +441,7 @@ pipeline {
"opensearch-infra-stack-${BUILD_NUMBER}"
]
}
withCredentials([string(credentialsId: 'perf-test-account-id', variable: 'PERF_TEST_ACCOUNT_ID')]) {
withSecrets(secrets: secret_aws_account){
withAWS(role: 'cfn-set-up', roleAccount: "${PERF_TEST_ACCOUNT_ID}", duration: 900, roleSessionName: 'jenkins-session', region: 'us-east-1') {
try {
for (String stackName : stackNames) {
Expand Down
6 changes: 5 additions & 1 deletion jenkins/opensearch/benchmark-test.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ lib = library(identifier: '[email protected]', retriever: modernSCM([
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_aws_account = [
[envVar: 'PERF_TEST_ACCOUNT_ID', secretRef: 'op://opensearch-infra-secrets/aws-accounts/perf-test-account-id']
]

pipeline {
agent none
options {
Expand Down Expand Up @@ -486,7 +490,7 @@ pipeline {
"opensearch-infra-stack-${BUILD_NUMBER}"
]
}
withCredentials([string(credentialsId: 'perf-test-account-id', variable: 'PERF_TEST_ACCOUNT_ID')]) {
withSecrets(secrets: secret_aws_account){
withAWS(role: 'cfn-set-up', roleAccount: "${PERF_TEST_ACCOUNT_ID}", duration: 900, roleSessionName: 'jenkins-session', region: 'us-east-1') {
try {
for (String stackName : stackNames) {
Expand Down
7 changes: 6 additions & 1 deletion jenkins/opensearch/distribution-build.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ lib = library(identifier: '[email protected]', retriever: modernSCM([
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_github_bot = [
[envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
[envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
]

pipeline {
options {
timeout(time: 4, unit: 'HOURS')
Expand Down Expand Up @@ -1097,7 +1102,7 @@ def triggerDistributionValidationWorkflow(String version) {
}

def triggerNightlyPlayground() {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
withSecrets(secrets: secret_github_bot){
println("Triggering nightly playground workflow.")
try {
sh(
Expand Down
7 changes: 6 additions & 1 deletion jenkins/release-workflows/release-branch.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ lib = library(identifier: '[email protected]', retriever: modernSCM([
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_github_bot = [
[envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
[envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
]

pipeline {
options {
timeout(time: 1, unit: 'HOURS')
Expand Down Expand Up @@ -56,7 +61,7 @@ pipeline {

def manifestList = MANIFEST_FILE.trim().split(' ') as List

withCredentials([usernamePassword(credentialsId: "jenkins-github-bot-token", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
withSecrets(secrets: secret_github_bot){
for (manifest in manifestList) {
if (manifest.contains("builds")) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ lib = library(identifier: '[email protected]', retriever: modernSCM([
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_github_bot = [
[envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
[envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
]

pipeline {
options {
timeout(time: 2, unit: 'HOURS')
Expand Down Expand Up @@ -212,7 +217,7 @@ pipeline {
stage('Create Pull Request') {
steps {
script {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
withSecrets(secrets: secret_github_bot){
try {
sh """
git remote set-url origin "https://opensearch-ci:${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-build"
Expand Down
9 changes: 7 additions & 2 deletions jenkins/release-workflows/release-notes-check.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ lib = library(identifier: '[email protected]', retriever: modernSCM([
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_github_bot = [
[envVar: 'GITHUB_USER', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-username'],
[envVar: 'GITHUB_TOKEN', secretRef: 'op://opensearch-infra-secrets/github-bot/ci-bot-token']
]

pipeline {
options {
timeout(time: 2, unit: 'HOURS')
Expand Down Expand Up @@ -81,7 +86,7 @@ pipeline {
}
steps {
script {
withCredentials([usernamePassword(credentialsId: "jenkins-github-bot-token", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
withSecrets(secrets: secret_github_bot){
if (params.COMMENT == "ADD") {
sh """
#!/bin/bash
Expand Down Expand Up @@ -128,7 +133,7 @@ pipeline {
set +e
./release_notes.sh compile manifests/${RELEASE_VERSION}/opensearch-${RELEASE_VERSION}.yml manifests/${RELEASE_VERSION}/opensearch-dashboards-${RELEASE_VERSION}.yml --output release-notes/opensearch-release-notes-${RELEASE_VERSION}.md
"""
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) {
withSecrets(secrets: secret_github_bot){
try {
sh """
git remote set-url origin "https://opensearch-ci:${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-build"
Expand Down
11 changes: 7 additions & 4 deletions jenkins/release-workflows/release-promotion.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ lib = library(identifier: '[email protected]', retriever: modernSCM([
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def secret_aws = [
[envVar: 'ARTIFACT_PROMOTION_ROLE_NAME', secretRef: 'op://opensearch-infra-secrets/aws-iam-roles/jenkins-artifact-promotion-role'],
[envVar: 'AWS_ACCOUNT_ARTIFACT', secretRef: 'op://opensearch-infra-secrets/aws-accounts/jenkins-aws-production-account'],
[envVar: 'CLOUDFRONT_DISTRIBUTION_ID', secretRef: 'op://opensearch-infra-secrets/aws-resource-arns/cloudfront-distribution-id']
]

pipeline {
options {
timeout(time: 4, unit: 'HOURS')
Expand Down Expand Up @@ -443,10 +449,7 @@ pipeline {
script {
echo 'Invalidating cloudfront'
def majorVersion=params.RELEASE_VERSION.find(/\d/)
withCredentials([
string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'),
string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'),
string(credentialsId: 'cloudfront-distribution-id', variable: 'CLOUDFRONT_DISTRIBUTION_ID')]) {
withSecrets(secrets: secret_aws){
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
cfInvalidate(distribution: "${CLOUDFRONT_DISTRIBUTION_ID}", paths: ["/releases/bundle/opensearch/${majorVersion}.x/*", "/releases/bundle/opensearch-dashboards/${majorVersion}.x/*"], waitForCompletion: true)
}
Expand Down
6 changes: 5 additions & 1 deletion jenkins/vulnerability-scan/whitesource-scan.jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
* compatible open source license.
*/

def secret_mend_scan = [
[envVar: 'wss_apikey', secretRef: 'op://opensearch-infra-secrets/mend-scan/wss-token']
]

pipeline {
agent none
options {
Expand All @@ -29,7 +33,7 @@ pipeline {
}
steps {
script {
withCredentials([string(credentialsId: 'jenkins-whitesource-scan-apikey', variable: 'wss_apikey')]) {
withSecrets(secrets: secret_mend_scan){
echo 'Executing WhiteSource Scan script'
scan()
}
Expand Down
8 changes: 7 additions & 1 deletion tests/jenkins/TestBenchmarkPullRequest.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@ class TestBenchmarkPullRequest extends BuildPipelineTest {
return helper.callClosure(closure)
})
helper.registerAllowedMethod('findFiles', [Map.class], null)
helper.registerAllowedMethod("withCredentials", [Map])
helper.registerAllowedMethod("withSecrets", [Map, Closure], { args, closure ->
closure.delegate = delegate
return helper.callClosure(closure)
})
helper.registerAllowedMethod("downloadBuildManifest", [Map], {
c -> lib.jenkins.BuildManifest.new(readYaml(file: 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml'))
})
Expand Down Expand Up @@ -106,6 +109,9 @@ class TestBenchmarkPullRequest extends BuildPipelineTest {
binding.setVariable('pull_request', 1234)
binding.setVariable('baseline_cluster_config', 'test-cluster-config')
binding.setVariable('repository','opensearch-project/OpenSearch')
binding.setVariable('PERF_TEST_ACCOUNT_ID', 'PERF_TEST_ACCOUNT_ID')
binding.setVariable('GITHUB_USER', "GITHUB_USER")
binding.setVariable('GITHUB_TOKEN', "GITHUB_TOKEN")

helper.registerAllowedMethod("GenericTrigger", [Map], { println 'GenericTrigger called with params: ' + it })
helper.registerAllowedMethod("sh", [Map.class], { map ->
Expand Down
7 changes: 6 additions & 1 deletion tests/jenkins/TestCompareBenchmarks.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,10 @@ class TestCompareBenchmarks extends BuildPipelineTest {
return helper.callClosure(closure)
})
helper.registerAllowedMethod('findFiles', [Map.class], null)
helper.registerAllowedMethod("withCredentials", [Map])
helper.registerAllowedMethod("withSecrets", [Map, Closure], { args, closure ->
closure.delegate = delegate
return helper.callClosure(closure)
})
helper.registerAllowedMethod("downloadBuildManifest", [Map], {
c -> lib.jenkins.BuildManifest.new(readYaml(file: 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml'))
})
Expand All @@ -65,6 +68,8 @@ class TestCompareBenchmarks extends BuildPipelineTest {
binding.setVariable('REPOSITORY','opensearch-project/OpenSearch')
binding.setVariable('BASELINE_TEST_EXECUTION_ID', 'baseline-id')
binding.setVariable('CONTENDER_TEST_EXECUTION_ID', 'contender-id')
binding.setVariable('GITHUB_USER', "GITHUB_USER")
binding.setVariable('GITHUB_TOKEN', "GITHUB_TOKEN")
}

@Test
Expand Down
Loading