Skip to content

Commit bbd1f67

Browse files
erikkerbererikkerber
committed
Add experimental Kotlin incremental compilation
Replace commented out snapshot generation code Add new jar dependencies needed for incremental compilation Fix output getting deleted during 2nd pass of incremental compilation Temporarily force persistent working directory for incremental compilation
1 parent cb8b7be commit bbd1f67

File tree

19 files changed

+1525
-53
lines changed

19 files changed

+1525
-53
lines changed

‎MODULE.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ use_repo(
2929
"com_github_jetbrains_kotlin",
3030
"com_github_jetbrains_kotlin_git",
3131
"com_github_pinterest_ktlint",
32+
"kotlin_build_tools_api",
3233
"kotlin_build_tools_impl",
34+
"kotlin_compiler_embeddable",
3335
"kotlinx_serialization_core_jvm",
3436
"kotlinx_serialization_json",
3537
"kotlinx_serialization_json_jvm",

‎kotlin/internal/jvm/compile.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,7 @@ def _run_kt_builder_action(
463463
args.add_all("--classpath", compile_deps.compile_jars)
464464
args.add("--reduced_classpath_mode", toolchains.kt.experimental_reduce_classpath_mode)
465465
args.add("--build_tools_api", toolchains.kt.experimental_build_tools_api)
466+
args.add("--incremental_compilation", toolchains.kt.experimental_incremental_compilation)
466467
args.add_all("--sources", srcs.all_srcs, omit_if_empty = True)
467468
args.add_all("--source_jars", srcs.src_jars + generated_src_jars, omit_if_empty = True)
468469
args.add_all("--deps_artifacts", deps_artifacts, omit_if_empty = True)

‎kotlin/internal/toolchains.bzl‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ def _kotlin_toolchain_impl(ctx):
9292
experimental_report_unused_deps = ctx.attr.experimental_report_unused_deps,
9393
experimental_reduce_classpath_mode = ctx.attr.experimental_reduce_classpath_mode,
9494
experimental_build_tools_api = ctx.attr.experimental_build_tools_api,
95+
experimental_incremental_compilation = ctx.attr.experimental_incremental_compilation,
9596
javac_options = ctx.attr.javac_options[JavacOptions] if ctx.attr.javac_options else None,
9697
kotlinc_options = ctx.attr.kotlinc_options[KotlincOptions] if ctx.attr.kotlinc_options else None,
9798
empty_jar = ctx.file._empty_jar,
@@ -252,6 +253,10 @@ _kt_toolchain = rule(
252253
doc = "Enables experimental support for Build Tools API integration",
253254
default = False,
254255
),
256+
"experimental_incremental_compilation": attr.bool(
257+
doc = "TODO",
258+
default = False,
259+
),
255260
"javac_options": attr.label(
256261
doc = "Compiler options for javac",
257262
providers = [JavacOptions],
@@ -327,6 +332,7 @@ def define_kt_toolchain(
327332
experimental_reduce_classpath_mode = None,
328333
experimental_multiplex_workers = None,
329334
experimental_build_tools_api = None,
335+
experimental_incremental_compilation = None,
330336
javac_options = Label("//kotlin/internal:default_javac_options"),
331337
kotlinc_options = Label("//kotlin/internal:default_kotlinc_options"),
332338
jvm_stdlibs = None,
@@ -356,6 +362,7 @@ def define_kt_toolchain(
356362
experimental_report_unused_deps = experimental_report_unused_deps,
357363
experimental_reduce_classpath_mode = experimental_reduce_classpath_mode,
358364
experimental_build_tools_api = experimental_build_tools_api,
365+
experimental_incremental_compilation = experimental_incremental_compilation,
359366
javac_options = javac_options,
360367
kotlinc_options = kotlinc_options,
361368
visibility = ["//visibility:public"],

‎kotlin/settings/BUILD.bazel‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,9 @@ bool_flag(
4848
build_setting_default = False,
4949
visibility = ["//visibility:public"],
5050
)
51+
52+
bool_flag(
53+
name = "experimental_incremental_compilation",
54+
build_setting_default = False,
55+
visibility = ["//visibility:public"],
56+
)

‎kotlin_rules_maven_install.json‎

Lines changed: 1232 additions & 36 deletions
Large diffs are not rendered by default.

‎src/main/kotlin/io/bazel/kotlin/builder/cmd/BUILD.bazel‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ kt_bootstrap_binary(
2727
"//src/main/kotlin/io/bazel/kotlin/compiler:compiler.jar",
2828
"@com_github_jetbrains_kotlin//:home",
2929
"@kotlin_build_tools_impl//jar",
30+
"@kotlin_compiler_embeddable//jar",
31+
"@kotlin_rules_maven//:org_jetbrains_kotlin_kotlin_daemon_client",
3032
"@kotlinx_serialization_core_jvm//jar",
3133
"@kotlinx_serialization_json//jar",
3234
"@kotlinx_serialization_json_jvm//jar",
@@ -36,10 +38,12 @@ kt_bootstrap_binary(
3638
"-D@com_github_jetbrains_kotlinx...serialization-json=$(rlocationpath @kotlinx_serialization_json//jar)",
3739
"-D@com_github_jetbrains_kotlinx...serialization-json-jvm=$(rlocationpath @kotlinx_serialization_json_jvm//jar)",
3840
"-D@com_github_jetbrains_kotlin...build-tools-impl=$(rlocationpath @kotlin_build_tools_impl//jar)",
41+
"-D@kotlin_compiler_embeddable...kotlin-embedded=$(rlocationpath @kotlin_compiler_embeddable//jar)",
3942
"-D@com_github_jetbrains_kotlin...jvm-abi-gen=$(rlocationpath //kotlin/compiler:jvm-abi-gen)",
4043
"-D@com_github_jetbrains_kotlin...kotlin-compiler=$(rlocationpath //kotlin/compiler:kotlin-compiler)",
4144
"-D@com_github_jetbrains_kotlin...kapt=$(rlocationpath //kotlin/compiler:kotlin-annotation-processing)",
4245
"-D@rules_kotlin...jdeps-gen=$(rlocationpath //src/main/kotlin:jdeps-gen)",
46+
"-D@com_github_jetbrains_kotlin...daemon-client=$(rlocationpath @kotlin_rules_maven//:org_jetbrains_kotlin_kotlin_daemon_client)",
4347
"-D@rules_kotlin...skip-code-gen=$(rlocationpath //src/main/kotlin:skip-code-gen)",
4448
"-D@rules_kotlin...compiler=$(rlocationpath //src/main/kotlin/io/bazel/kotlin/compiler:compiler.jar)",
4549
"-D@com_github_google_ksp...symbol-processing-api=$(rlocationpath //kotlin/compiler:symbol-processing-api)",

‎src/main/kotlin/io/bazel/kotlin/builder/tasks/BUILD.bazel‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ kt_bootstrap_library(
2424
],
2525
visibility = ["//src:__subpackages__"],
2626
deps = [
27+
"//kotlin/compiler:kotlin-compiler",
2728
"//kotlin/compiler:kotlin-preloader",
2829
"//src/main/kotlin/io/bazel/kotlin/builder/toolchain",
2930
"//src/main/kotlin/io/bazel/kotlin/builder/utils",
@@ -32,6 +33,9 @@ kt_bootstrap_library(
3233
"//src/main/protobuf:deps_java_proto",
3334
"//src/main/protobuf:kotlin_model_java_proto",
3435
"//src/main/protobuf:worker_protocol_java_proto",
36+
"@kotlin_build_tools_api//jar",
37+
"@kotlin_build_tools_impl//jar",
38+
"@kotlin_compiler_embeddable//jar",
3539
"@kotlin_rules_maven//:com_google_protobuf_protobuf_java",
3640
"@kotlin_rules_maven//:com_google_protobuf_protobuf_java_util",
3741
"@kotlin_rules_maven//:javax_inject_javax_inject",

‎src/main/kotlin/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ class KotlinBuilder
8888
INSTRUMENT_COVERAGE("--instrument_coverage"),
8989
KSP_GENERATED_JAVA_SRCJAR("--ksp_generated_java_srcjar"),
9090
BUILD_TOOLS_API("--build_tools_api"),
91+
INCREMENTAL_COMPILATION("--incremental_compilation"),
9192
}
9293
}
9394

@@ -173,6 +174,9 @@ class KotlinBuilder
173174
argMap.optionalSingle(KotlinBuilderFlags.BUILD_TOOLS_API)?.let {
174175
buildToolsApi = it == "true"
175176
}
177+
argMap.optionalSingle(KotlinBuilderFlags.INCREMENTAL_COMPILATION)?.let {
178+
incrementalCompilation = it == "true"
179+
}
176180
this
177181
}
178182

@@ -288,8 +292,17 @@ class KotlinBuilder
288292
argMap.optional(KotlinBuilderFlags.COMPILER_PLUGIN_CLASS_PATH) ?: emptyList(),
289293
)
290294

295+
// Kotlin compiler always requires absolute path for source input in incremental mode
296+
val useAbsolutePath = argMap.optionalSingle(KotlinBuilderFlags.INCREMENTAL_COMPILATION) == "true"
291297
argMap
292298
.optional(KotlinBuilderFlags.SOURCES)
299+
?.map {
300+
if (useAbsolutePath) {
301+
FileSystems.getDefault().getPath(it).toAbsolutePath().toString()
302+
} else {
303+
it
304+
}
305+
}
293306
?.iterator()
294307
?.partitionJvmSources(
295308
{ addKotlinSources(it) },

‎src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/ClasspathSnapshotGenerator.kt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package io.bazel.kotlin.builder.tasks.jvm
2+
3+
import java.nio.file.Path
4+
import kotlin.system.measureTimeMillis
5+
import org.jetbrains.kotlin.buildtools.api.CompilationService
6+
import org.jetbrains.kotlin.buildtools.api.ExperimentalBuildToolsApi
7+
import org.jetbrains.kotlin.buildtools.api.KotlinLogger
8+
import java.io.File
9+
import java.util.logging.Logger
10+
import kotlin.io.path.exists
11+
12+
@OptIn(ExperimentalBuildToolsApi::class)
13+
class ClasspathSnapshotGenerator(
14+
private val inputJar: Path,
15+
private val outputSnapshot: Path,
16+
private val granularity: SnapshotGranularity
17+
) {
18+
19+
fun run() {
20+
// if (outputSnapshot.exists()) { return }
21+
val timeSpent = measureTimeMillis {
22+
val compilationService = CompilationService.loadImplementation(this.javaClass.classLoader!!)
23+
val snapshot =
24+
compilationService.calculateClasspathSnapshot(
25+
inputJar.toFile(), granularity.toClassSnapshotGranularity)
26+
snapshot.saveSnapshot(outputSnapshot.toFile())
27+
}
28+
// TODO: Log impl
29+
// LOG.info("$timeSpent ms for input jar: $inputJar")
30+
}
31+
}

‎src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/CompilationTask.kt‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import java.io.BufferedInputStream
3535
import java.io.ByteArrayOutputStream
3636
import java.io.File
3737
import java.io.ObjectOutputStream
38+
import java.nio.file.FileSystems
3839
import java.nio.file.Files
3940
import java.nio.file.Files.isDirectory
4041
import java.nio.file.Files.walk
@@ -91,6 +92,11 @@ fun JvmCompilationTask.baseArgs(overrides: Map<String, String> = emptyMap()): Co
9192
overrides[LANGUAGE_VERSION_ARG] ?: info.toolchainInfo.common.languageVersion,
9293
).flag("-jvm-target", info.toolchainInfo.jvm.jvmTarget)
9394
.flag("-module-name", info.moduleName)
95+
.apply {
96+
if (info.buildToolsApi) {
97+
flag("-label", info.label)
98+
}
99+
}
94100
}
95101

96102
internal fun JvmCompilationTask.plugins(
@@ -308,8 +314,13 @@ private fun JvmCompilationTask.runKspPlugin(
308314
baseArgs(overrides)
309315
.plus(kspArgs(plugins))
310316
.flag("-d", directories.generatedClasses)
311-
.values(inputs.kotlinSourcesList)
312317
.values(inputs.javaSourcesList)
318+
.values(inputs.kotlinSourcesList)
319+
.apply {
320+
if (info.incrementalCompilation) {
321+
flag("-incremental_id", "ksp")
322+
}
323+
}
313324
.list()
314325
.let { args ->
315326
context.executeCompilerTask(
@@ -422,6 +433,28 @@ internal fun JvmCompilationTask.createGeneratedKspKotlinSrcJar() {
422433
}
423434
}
424435

436+
internal fun JvmCompilationTask.createClasspathSnapshots() {
437+
inputs.classpathList.forEach {
438+
ClasspathSnapshotGenerator(Paths.get(it), Paths.get("$it.snapshot"), SnapshotGranularity.CLASS_MEMBER_LEVEL).run()
439+
}
440+
}
441+
442+
internal fun JvmCompilationTask.createClasspathSnapshotsPaths(): List<String> {
443+
return inputs.classpathList.map { it: String ->
444+
"$it.snapshot"
445+
}
446+
}
447+
448+
449+
val ROOT: String by lazy {
450+
FileSystems
451+
.getDefault()
452+
.getPath("")
453+
.toAbsolutePath()
454+
.toString() + File.separator
455+
}
456+
457+
425458
/**
426459
* Compiles Kotlin sources to classes. Does not compile Java sources.
427460
*/
@@ -441,9 +474,23 @@ fun JvmCompilationTask.compileKotlin(
441474
options = inputs.compilerPluginOptionsList,
442475
classpath = inputs.compilerPluginClasspathList,
443476
)
444-
).values(inputs.javaSourcesList)
477+
)
478+
.values(inputs.javaSourcesList)
445479
.values(inputs.kotlinSourcesList)
446480
.flag("-d", directories.classes)
481+
.apply {
482+
if (info.incrementalCompilation) {
483+
flag("-incremental_id", "kotlin")
484+
flag("-snapshot")
485+
paths(
486+
createClasspathSnapshotsPaths(),
487+
) {
488+
it
489+
.map(Path::toString)
490+
.joinToString(File.pathSeparator)
491+
}
492+
}
493+
}
447494
.list()
448495
.let {
449496
context.whenTracing {

0 commit comments

Comments
 (0)