Skip to content

[WIP] Benchmark Tests #686

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
prasannspv wants to merge 6 commits into
base: master
Choose a base branch
from
Open

[WIP] Benchmark Tests #686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
00db8d2
Benchmark Tests
pv-atl May 16, 2025
d39e697
instantiate nadel per benchmark
pv-atl May 16, 2025
b532752
Seconds
pv-atl May 16, 2025
5951074
Benchmark fix
pv-atl May 16, 2025
0c6efcf
await
pv-atl Jun 11, 2025
d0f551c
Merge branch 'atlassian-labs:master' into master
prasannspv Jul 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions test/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
Copy link
Collaborator

@dondonz dondonz Jun 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: you can remove WIP from the PR name, to indicate you're ready for a review

kotlin("jvm")
id("com.bnorm.power.kotlin-power-assert")
id("me.champeau.jmh") version "0.7.3"
}

dependencies {
Expand All @@ -22,6 +23,11 @@ dependencies {
testImplementation("org.skyscreamer:jsonassert:1.5.1")
testImplementation("com.google.guava:guava:33.1.0-jre")
testImplementation("com.squareup:kotlinpoet:1.16.0")

// JMH dependencies
jmh("org.openjdk.jmh:jmh-core:1.37")
jmh("org.openjdk.jmh:jmh-generator-annprocess:1.37")
jmhAnnotationProcessor("org.openjdk.jmh:jmh-generator-annprocess:1.37")
}

tasks.withType<Test> {
Expand All @@ -45,3 +51,7 @@ configure<PowerAssertGradleExtension> {
// WARNING: do NOT touch this unless you have read https://github.com/bnorm/kotlin-power-assert/issues/55
functions = listOf("kotlin.assert", "kotlin.test.assertTrue", "kotlin.test.assertFalse")
}

tasks.named<Jar>("jmhJar") {
destinationDirectory.set(file("$rootDir/build/libs"))
}
25 changes: 25 additions & 0 deletions test/src/jmh/kotlin/performance/BasicObjectSchemaBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package performance

import graphql.nadel.tests.next.fixtures.basic.BasicObjectSchemaTest
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class BasicObjectSchemaBenchmark: BenchmarkTestBase() {

@Setup
override fun setup() {
test = BasicObjectSchemaTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
24 changes: 24 additions & 0 deletions test/src/jmh/kotlin/performance/BatchHydrationAtQueryTypeBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package performance

import graphql.nadel.tests.next.fixtures.batchHydration.BatchHydrationAtQueryTypeTest
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class BatchHydrationAtQueryTypeBenchmark: BenchmarkTestBase() {
@Setup
override fun setup() {
test = BatchHydrationAtQueryTypeTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
25 changes: 25 additions & 0 deletions test/src/jmh/kotlin/performance/BatchHydrationDeferBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package performance

import graphql.nadel.tests.next.fixtures.hydration.defer.batch.BatchHydrationDeferTest
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class BatchHydrationDeferBenchmark: BenchmarkTestBase() {

@Setup
override fun setup() {
test = BatchHydrationDeferTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
31 changes: 31 additions & 0 deletions test/src/jmh/kotlin/performance/BenchmarkTestBase.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package performance

import graphql.nadel.Nadel
import graphql.nadel.NadelExecutionInput
import graphql.nadel.tests.next.NadelIntegrationTest
import org.openjdk.jmh.annotations.Fork
import org.openjdk.jmh.annotations.Measurement
import org.openjdk.jmh.annotations.Scope
import org.openjdk.jmh.annotations.State
import org.openjdk.jmh.annotations.Warmup

@State(Scope.Benchmark)
@Warmup(iterations = 5, time = 5)
@Measurement(iterations = 5)
@Fork(2)
Copy link
Collaborator

@dondonz dondonz Jun 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a blocker for this PR: For GraphQL Java benchmarks we had slightly fewer warmup and measurement runs.

Usually 2 for warmup and 2-3 for measurement

Because the GraphQL Java benchmark tests run on every commit we had to be more careful there about the total running time for the tests (we arbitrarily set a limit of about 1 hour).

open class BenchmarkTestBase {

lateinit var test: NadelIntegrationTest
private lateinit var nadel: Nadel
private lateinit var executionInput: NadelExecutionInput

open fun setup() {
executionInput = test.makeExecutionInput().build()
}

fun execute() {
nadel = test.makeNadel()
.build()
nadel.execute(executionInput).get()
}
}
24 changes: 24 additions & 0 deletions test/src/jmh/kotlin/performance/ComprehensiveDeferQueryWithDifferentServiceCallsBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package performance

import graphql.nadel.tests.next.fixtures.defer.ComprehensiveDeferQueryWithDifferentServiceCalls
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class ComprehensiveDeferQueryWithDifferentServiceCallsBenchmark: BenchmarkTestBase() {
@Setup
override fun setup() {
test = ComprehensiveDeferQueryWithDifferentServiceCalls()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
Copy link
Collaborator

@dondonz dondonz Jun 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: this is a typo, should be benchmark

execute()
}
}
25 changes: 25 additions & 0 deletions test/src/jmh/kotlin/performance/EchoBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package performance

import graphql.nadel.tests.next.fixtures.basic.BasicObjectSchemaTest
import graphql.nadel.tests.next.fixtures.basic.EchoTest
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class EchoBenchmark: BenchmarkTestBase() {
@Setup
override fun setup() {
test = EchoTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
24 changes: 24 additions & 0 deletions test/src/jmh/kotlin/performance/IdHydrationBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package performance

import graphql.nadel.tests.next.fixtures.hydration.idHydration.IdHydrationTest
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class IdHydrationBenchmark: BenchmarkTestBase() {
@Setup
override fun setup() {
test = IdHydrationTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
24 changes: 24 additions & 0 deletions test/src/jmh/kotlin/performance/MultipleDeferWithDifferentServiceCallsBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package performance

import graphql.nadel.tests.next.fixtures.defer.MultipleDeferWithDifferentServiceCalls
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class MultipleDeferWithDifferentServiceCallsBenchmark: BenchmarkTestBase() {
@Setup
override fun setup() {
test = MultipleDeferWithDifferentServiceCalls()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
26 changes: 26 additions & 0 deletions test/src/jmh/kotlin/performance/MultipleFieldsInMultipleDeferDirectivesBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package performance

import graphql.nadel.tests.next.fixtures.defer.MultipleFieldsInMultipleDeferDirectivesTest
import graphql.nadel.tests.next.fixtures.defer.transforms.MultipleRenamedFieldsAreDeferredTogether
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class MultipleFieldsInMultipleDeferDirectivesBenchmark: BenchmarkTestBase() {

@Setup
override fun setup() {
test = MultipleFieldsInMultipleDeferDirectivesTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
24 changes: 24 additions & 0 deletions test/src/jmh/kotlin/performance/MultipleRenamedFieldsAreDeferredTogetherBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package performance

import graphql.nadel.tests.next.fixtures.defer.transforms.MultipleRenamedFieldsAreDeferredTogether
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class MultipleRenamedFieldsAreDeferredTogetherBenchmark: BenchmarkTestBase() {
@Setup
override fun setup() {
test = MultipleRenamedFieldsAreDeferredTogether()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
24 changes: 24 additions & 0 deletions test/src/jmh/kotlin/performance/NamespacedPartitionBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package performance

import graphql.nadel.tests.next.fixtures.partition.NamespacedPartitionTest
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class NamespacedPartitionBenchmark: BenchmarkTestBase() {
@Setup
override fun setup() {
test = NamespacedPartitionTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
24 changes: 24 additions & 0 deletions test/src/jmh/kotlin/performance/NestedDefersBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package performance

import graphql.nadel.tests.next.fixtures.defer.NestedDefersTest
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class NestedDefersBenchmark: BenchmarkTestBase() {
@Setup
override fun setup() {
test = NestedDefersTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
25 changes: 25 additions & 0 deletions test/src/jmh/kotlin/performance/NestedPartitionBenchmark.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package performance

import graphql.nadel.tests.next.fixtures.partition.NestedPartitionTest
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.Mode
import org.openjdk.jmh.annotations.OutputTimeUnit
import org.openjdk.jmh.annotations.Setup
import java.util.concurrent.TimeUnit

open class NestedPartitionBenchmark: BenchmarkTestBase() {

@Setup
override fun setup() {
test = NestedPartitionTest()
super.setup()
}

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
fun bechmarkThroughput() {
execute()
}
}
Loading