diff --git a/.buildkite/pipelines/pull-request/build-benchmark.yml b/.buildkite/pipelines/pull-request/build-benchmark.yml index 8d3215b8393ce..baea187ea5e7a 100644 --- a/.buildkite/pipelines/pull-request/build-benchmark.yml +++ b/.buildkite/pipelines/pull-request/build-benchmark.yml @@ -7,16 +7,19 @@ steps: command: | .ci/scripts/run-gradle.sh :build-tools-internal:bootstrapPerformanceTests .ci/scripts/install-gradle-profiler.sh - .ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-build-benchmark-{{matrix.part}}.scenarios --project-dir . --output-dir profile-out - mkdir build - tar -czf build/$BUILDKITE_BUILD_NUMBER.tar.bz2 profile-out + .ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-{{matrix.part}}.scenarios --measure-config-time --project-dir . --output-dir profile-out + mkdir -p build + buildkite-agent artifact upload "profile-out/*" + cat << EOF | buildkite-agent annotate --style "info" + Benchmark Html Report Benchmark Html Report + EOF matrix: setup: part: - - part1 - - part2 + - esql-tests env: BUILD_PERFORMANCE_TEST: "true" + agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.ci/scripts/install-gradle-profiler.sh b/.ci/scripts/install-gradle-profiler.sh index 84bde5e11b2b2..4fa1fa8e24525 100755 --- a/.ci/scripts/install-gradle-profiler.sh +++ b/.ci/scripts/install-gradle-profiler.sh @@ -2,7 +2,7 @@ set -e # profiler version we wanna install -PROFILER_VERSION="0.16.0" -wget https://repo.gradle.org/gradle/ext-releases-local/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip +PROFILER_VERSION="0.22.0" +wget https://repo1.maven.org/maven2/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip unzip $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip mv $WORKSPACE/gradle-profiler-$PROFILER_VERSION $WORKSPACE/gradle-profiler diff --git a/build-tools-internal/performance/elasticsearch-build-benchmark-part1.scenarios b/build-tools-internal/performance/elasticsearch-build-benchmark-part1.scenarios deleted file mode 100644 index 3f7812985c16b..0000000000000 --- a/build-tools-internal/performance/elasticsearch-build-benchmark-part1.scenarios +++ /dev/null @@ -1,72 +0,0 @@ -// ensure branch scenario is listed first as this is the gradle version that will picked for inspecting the build -default-scenarios = ["buildConfiguration_branch", "buildConfiguration_main", "single_project_branch", "single_project_main"] - -buildConfiguration_main { - title = "configuration phase (main)" - versions = ["@mainWrapper@"] - tasks = ["help"] - gradle-args = ["--no-scan", "--no-build-cache", "--stacktrace"] - run-using = cli // value can be "cli" or "tooling-api" - daemon = warm // value can be "warm", "cold", or "none" - warm-ups = 5 - iterations = 10 - system-properties { - "BUILD_PERFORMANCE_TEST" = "true" - } - git-checkout = { - build = "main" - } -} - -buildConfiguration_branch { - title = "configuration phase (@testGitCommit@)" - versions = ["@branchWrapper@"] - tasks = ["help"] - gradle-args = ["--no-scan", "--no-build-cache", "--stacktrace"] - run-using = cli // value can be "cli" or "tooling-api" - daemon = warm // value can be "warm", "cold", or "none" - warm-ups = 5 - iterations = 10 - system-properties { - "BUILD_PERFORMANCE_TEST" = "true" - } - git-checkout = { - build = "@testGitCommit@" - } -} - -single_project_main { - title = "single project (main)" - versions = ["@mainWrapper@"] - tasks = [":server:precommit"] - gradle-args = ["--no-scan", "--stacktrace"] - apply-abi-change-to = "server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java" - run-using = cli // value can be "cli" or "tooling-api" - daemon = warm // value can be "warm", "cold", or "none" - warm-ups = 5 - iterations = 10 - system-properties { - "BUILD_PERFORMANCE_TEST" = "true" - } - git-checkout = { - build = "main" - } -} - -single_project_branch { - title = "single project (@testGitCommit@)" - versions = ["@branchWrapper@"] - tasks = [":server:precommit"] - gradle-args = ["--no-scan", "--stacktrace"] - apply-abi-change-to = "server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java" - run-using = cli // value can be "cli" or "tooling-api" - daemon = warm // value can be "warm", "cold", or "none" - warm-ups = 5 - iterations = 10 - system-properties { - "BUILD_PERFORMANCE_TEST" = "true" - } - git-checkout = { - build = "@testGitCommit@" - } -} diff --git a/build-tools-internal/performance/elasticsearch-build-benchmark-part2.scenarios b/build-tools-internal/performance/elasticsearch-build-benchmark-part2.scenarios deleted file mode 100644 index 3b5e1372e9cd2..0000000000000 --- a/build-tools-internal/performance/elasticsearch-build-benchmark-part2.scenarios +++ /dev/null @@ -1,38 +0,0 @@ -// ensure branch scenario is listed first as this is the gradle version that will picked for inspecting the build -default-scenarios = ["precommit_branch", "precommit_main"] - -precommit_main { - title = "precommit (main)" - versions = ["@mainWrapper@"] - cleanup-tasks = ["clean"] - tasks = ["precommit"] - gradle-args = ["--no-scan", "--no-build-cache", "--stacktrace"] - run-using = cli // value can be "cli" or "tooling-api" - daemon = warm // value can be "warm", "cold", or "none" - warm-ups = 5 - iterations = 10 - system-properties { - "BUILD_PERFORMANCE_TEST" = "true" - } - git-checkout = { - build = "main" - } -} - -precommit_branch { - title = "precommit (@testGitCommit@)" - versions = ["@branchWrapper@"] - cleanup-tasks = ["clean"] - tasks = ["precommit"] - gradle-args = ["--no-scan", "--no-build-cache", "--stacktrace"] - run-using = cli // value can be "cli" or "tooling-api" - daemon = warm // value can be "warm", "cold", or "none" - warm-ups = 5 - iterations = 10 - system-properties { - "BUILD_PERFORMANCE_TEST" = "true" - } - git-checkout = { - build = "@testGitCommit@" - } -} diff --git a/build-tools-internal/performance/elasticsearch-esql-tests.scenarios b/build-tools-internal/performance/elasticsearch-esql-tests.scenarios new file mode 100644 index 0000000000000..49de7b9d60db4 --- /dev/null +++ b/build-tools-internal/performance/elasticsearch-esql-tests.scenarios @@ -0,0 +1,118 @@ +default-scenarios = ["TEST_ESQL_ABI_CHANGE_TAPI", TEST_ESQL_ABI_CHANGE_TAPI_MAIN, "TEST_ESQL_ABI_CHANGE_TAPI_CC", "TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT", "TEST_ESQL_ABI_CHANGE_TAPI_CC_MAIN", "TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT_MAIN"] + +TEST_ESQL_ABI_CHANGE_TAPI { + apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java" + tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"] + gradle-args = ["-I", ".ci/init.gradle"] + run-using = tooling-api // value can be "cli" or "tooling-api" + daemon = warm // value can be "warm", "cold", or "none" + copy-file = { + source = "build-tools-internal/performance/test-files/BenchmarkingTests.java" + target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java" + schedule = SCENARIO + } + git-checkout = { + cleanup = "@testGitCommit@" + build = "@testGitCommit@" + } + warm-ups = 3 + iterations = 10 +} + +TEST_ESQL_ABI_CHANGE_TAPI_CC { + apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java" + tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"] + gradle-args = ["--configuration-cache", "-I", ".ci/init.gradle"] + run-using = tooling-api // value can be "cli" or "tooling-api" + daemon = warm // value can be "warm", "cold", or "none" + copy-file = { + source = "build-tools-internal/performance/test-files/BenchmarkingTests.java" + target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java" + schedule = SCENARIO + } + + git-checkout = { + cleanup = "@testGitCommit@" + build = "@testGitCommit@" + } + warm-ups = 3 + iterations = 10 +} + +TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT { + apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java" + tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"] + gradle-args = ["--configuration-cache"] + run-using = tooling-api // value can be "cli" or "tooling-api" + daemon = warm // value can be "warm", "cold", or "none" + copy-file = { + source = "build-tools-internal/performance/test-files/BenchmarkingTests.java" + target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java" + schedule = SCENARIO + } + + git-checkout = { + cleanup = "@testGitCommit@" + build = "@testGitCommit@" + } + warm-ups = 3 + iterations = 10 +} + +TEST_ESQL_ABI_CHANGE_TAPI_MAIN { + apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java" + tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"] + gradle-args = ["-I", ".ci/init.gradle"] + run-using = tooling-api // value can be "cli" or "tooling-api" + daemon = warm // value can be "warm", "cold", or "none" + copy-file = { + source = "build-tools-internal/performance/test-files/BenchmarkingTests.java" + target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java" + schedule = SCENARIO + } + git-checkout = { + cleanup = "main" + build = "main" + } + warm-ups = 3 + iterations = 10 +} + +TEST_ESQL_ABI_CHANGE_TAPI_CC_MAIN { + apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java" + tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"] + gradle-args = ["--configuration-cache", "-I", ".ci/init.gradle"] + run-using = tooling-api // value can be "cli" or "tooling-api" + daemon = warm // value can be "warm", "cold", or "none" + copy-file = { + source = "build-tools-internal/performance/test-files/BenchmarkingTests.java" + target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java" + schedule = SCENARIO + } + + git-checkout = { + cleanup = "main" + build = "main" + } + warm-ups = 3 + iterations = 10 +} + +TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT_MAIN { + apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java" + tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"] + gradle-args = ["--configuration-cache"] + run-using = tooling-api // value can be "cli" or "tooling-api" + daemon = warm // value can be "warm", "cold", or "none" + copy-file = { + source = "build-tools-internal/performance/test-files/BenchmarkingTests.java" + target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java" + schedule = SCENARIO + } + git-checkout = { + cleanup = "main" + build = "main" + } + warm-ups = 3 + iterations = 10 +} diff --git a/build-tools-internal/performance/test-files/BenchmarkingTests.java b/build-tools-internal/performance/test-files/BenchmarkingTests.java new file mode 100644 index 0000000000000..8e2a2e5905859 --- /dev/null +++ b/build-tools-internal/performance/test-files/BenchmarkingTests.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.gradle.benchmarking; + +import org.elasticsearch.test.ESTestCase; + +public class BenchmarkingTests extends ESTestCase { + + public void testBenchmarking() { + assertTrue(true); + } +}