[WIP] Kotlin 2 semanticdb plugin

.tool-versions

@@ -1 +1 @@
-golang 1.17.5
+golang 1.23.1

build.sbt

@@ -9,7 +9,7 @@ import scala.util.control.NoStackTrace
 
 lazy val V =
   new {
-    val protobuf = "3.15.6"
+    val protobuf = "3.25.6"
     val protoc =
       "3.17.3" // the oldest protoc version with Apple M1 support, see https://github.com/scalapb/ScalaPB/issues/1024#issuecomment-860126568
     val coursier = "2.1.9"
@@ -107,6 +107,31 @@ lazy val agent = project
         "Premain-Class" -> "com.sourcegraph.semanticdb_javac.SemanticdbAgent"
       )
   )
+
+import kotlin.Keys._
+lazy val kotlincPlugin = project
+  .in(file("semanticdb-kotlinc"))
+  .enablePlugins(KotlinPlugin)
+  .settings(
+    kotlinVersion := "2.0.21",
+    kotlincJvmTarget := "1.8",
+    kotlinLib("stdlib"),
+    kotlinLib("compiler-embeddable"),
+    // the source generator that sbt-protoc adds only includes java and scala files
+    Compile / sourceGenerators +=
+      (Compile / PB.generate)
+        .map(
+          _.filter { file =>
+            file.getName.endsWith(".kt")
+          }
+        )
+        .taskValue,
+    (Compile / PB.protoSources) :=
+      Seq((semanticdb / Compile / sourceDirectory).value / "protobuf"),
+    (Compile / PB.targets) :=
+      Seq(PB.gens.kotlin(V.protobuf) -> (Compile / sourceManaged).value)
+  )
+
 lazy val gradlePlugin = project
   .in(file("semanticdb-gradle-plugin"))
   .settings(

project/build.properties

@@ -1 +1 @@
-sbt.version=1.10.10
+sbt.version=1.10.11

project/plugins.sbt

@@ -11,6 +11,7 @@ addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.6.1")
 addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3")
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
 addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1")
+addSbtPlugin("org.jetbrains.scala" % "sbt-kotlin-plugin" % "3.1.4")
 // sbt-jdi-tools appears to fix an error related to this message:
 // [error] (plugin / Compile / compileIncremental) java.lang.NoClassDefFoundError: com/sun/tools/javac/code/Symbol
 addSbtPlugin("org.scala-debugger" % "sbt-jdi-tools" % "1.1.1")

semanticdb-java/src/main/protobuf/semanticdb.proto

@@ -7,6 +7,8 @@ syntax = "proto3";
 
 package com.sourcegraph.semanticdb_javac;
 
+option java_multiple_files = true;
+
 enum Schema {
   LEGACY = 0;
   SEMANTICDB3 = 3;

semanticdb-kotlinc/src/main/kotlin/Run.kt

@@ -0,0 +1,6 @@
+package demo
+
+fun main(args: Array<String>) {
+    // Test some Kotlin 1.9 features
+    println(args[0])
+}

semanticdb-kotlinc/src/main/kotlin/com/sourcegraph/semanticdb_kotlinc/Analyzer.kt

@@ -0,0 +1,106 @@
+package com.sourcegraph.semanticdb_kotlinc
+
+import com.sourcegraph.semanticdb_javac.Semanticdb
+import java.io.PrintWriter
+import java.io.Writer
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import kotlin.contracts.ExperimentalContracts
+import org.jetbrains.kotlin.analyzer.AnalysisResult
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
+import org.jetbrains.kotlin.cli.common.messages.PrintingMessageCollector
+import org.jetbrains.kotlin.com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.psi.*
+import org.jetbrains.kotlin.resolve.BindingTrace
+import org.jetbrains.kotlin.resolve.jvm.extensions.AnalysisHandlerExtension
+
+@ExperimentalContracts
+class Analyzer(
+    val sourceroot: Path,
+    val targetroot: Path,
+    val callback: (Semanticdb.TextDocument) -> Unit
+) : AnalysisHandlerExtension {
+    private val globals = GlobalSymbolsCache()
+
+    private val messageCollector =
+        CompilerConfiguration()
+            .get(
+                CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY,
+                PrintingMessageCollector(System.err, MessageRenderer.PLAIN_FULL_PATHS, false))
+
+    override fun analysisCompleted(
+        project: Project,
+        module: ModuleDescriptor,
+        bindingTrace: BindingTrace,
+        files: Collection<KtFile>
+    ): AnalysisResult? =
+        try {
+            val resolver = DescriptorResolver(bindingTrace).also { globals.resolver = it }
+            for (file in files) {
+                try {
+                    val lineMap = LineMap(project, file)
+                    val document =
+                        SemanticdbVisitor(sourceroot, resolver, file, lineMap, globals).build()
+                    semanticdbOutPathForFile(file)?.apply {
+                        val builder = Semanticdb.TextDocuments.newBuilder()
+                        builder.addDocuments(document)
+                        Files.write(this, builder.build().toByteArray())
+                    }
+                    callback(document)
+                } catch (e: Exception) {
+                    handleException(e)
+                }
+            }
+
+            super.analysisCompleted(project, module, bindingTrace, files)
+        } catch (e: Exception) {
+            handleException(e)
+            super.analysisCompleted(project, module, bindingTrace, files)
+        }
+
+    private fun semanticdbOutPathForFile(file: KtFile): Path? {
+        val normalizedPath = Paths.get(file.virtualFilePath).normalize()
+        if (normalizedPath.startsWith(sourceroot)) {
+            val relative = sourceroot.relativize(normalizedPath)
+            val filename = relative.fileName.toString() + ".semanticdb"
+            val semanticdbOutPath =
+                targetroot
+                    .resolve("META-INF")
+                    .resolve("semanticdb")
+                    .resolve(relative)
+                    .resolveSibling(filename)
+
+            Files.createDirectories(semanticdbOutPath.parent)
+            return semanticdbOutPath
+        }
+        System.err.println(
+            "given file is not under the sourceroot.\n\tSourceroot: $sourceroot\n\tFile path: ${file.virtualFilePath}\n\tNormalized file path: $normalizedPath")
+        return null
+    }
+
+    private fun handleException(e: Exception) {
+        val writer =
+            PrintWriter(
+                object : Writer() {
+                    val buf = StringBuffer()
+                    override fun close() =
+                        messageCollector.report(CompilerMessageSeverity.EXCEPTION, buf.toString())
+
+                    override fun flush() = Unit
+                    override fun write(data: CharArray, offset: Int, len: Int) {
+                        buf.append(data, offset, len)
+                    }
+                },
+                false)
+        writer.println("Exception in semanticdb-kotlin compiler plugin:")
+        e.printStackTrace(writer)
+        writer.println(
+            "Please report a bug to https://github.com/sourcegraph/lsif-kotlin with the stack trace above.")
+        writer.close()
+    }
+}

semanticdb-kotlinc/src/main/kotlin/com/sourcegraph/semanticdb_kotlinc/AnalyzerCommandLineProcessor.kt

@@ -0,0 +1,44 @@
+package com.sourcegraph.semanticdb_kotlinc
+
+import java.nio.file.Path
+import java.nio.file.Paths
+import org.jetbrains.kotlin.compiler.plugin.AbstractCliOption
+import org.jetbrains.kotlin.compiler.plugin.CliOption
+import org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor
+import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.CompilerConfigurationKey
+
+const val VAL_SOURCES = "sourceroot"
+val KEY_SOURCES = CompilerConfigurationKey<Path>(VAL_SOURCES)
+
+const val VAL_TARGET = "targetroot"
+val KEY_TARGET = CompilerConfigurationKey<Path>(VAL_TARGET)
+
+@OptIn(ExperimentalCompilerApi::class)
+class AnalyzerCommandLineProcessor : CommandLineProcessor {
+    override val pluginId: String = "semanticdb-kotlinc"
+    override val pluginOptions: Collection<AbstractCliOption> =
+        listOf(
+            CliOption(
+                VAL_SOURCES,
+                "<path>",
+                "the absolute path to the root of the Kotlin sources",
+                required = true),
+            CliOption(
+                VAL_TARGET,
+                "<path>",
+                "the absolute path to the directory where to generate SemanticDB files.",
+                required = true))
+
+    override fun processOption(
+        option: AbstractCliOption,
+        value: String,
+        configuration: CompilerConfiguration
+    ) {
+        when (option.optionName) {
+            VAL_SOURCES -> configuration.put(KEY_SOURCES, Paths.get(value))
+            VAL_TARGET -> configuration.put(KEY_TARGET, Paths.get(value))
+        }
+    }
+}

semanticdb-kotlinc/src/main/kotlin/com/sourcegraph/semanticdb_kotlinc/AnalyzerRegistrar.kt

@@ -0,0 +1,29 @@
+package com.sourcegraph.semanticdb_kotlinc
+
+import com.sourcegraph.semanticdb_javac.Semanticdb
+import java.lang.IllegalArgumentException
+import kotlin.contracts.ExperimentalContracts
+import org.jetbrains.kotlin.com.intellij.mock.MockProject
+import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
+import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.resolve.jvm.extensions.AnalysisHandlerExtension
+
+@OptIn(ExperimentalCompilerApi::class)
+@ExperimentalContracts
+class AnalyzerRegistrar(private val callback: (Semanticdb.TextDocument) -> Unit = {}) :
+    ComponentRegistrar {
+    override fun registerProjectComponents(
+        project: MockProject,
+        configuration: CompilerConfiguration
+    ) {
+        AnalysisHandlerExtension.registerExtension(
+            project,
+            Analyzer(
+                sourceroot = configuration[KEY_SOURCES]
+                        ?: throw IllegalArgumentException("configuration key $KEY_SOURCES missing"),
+                targetroot = configuration[KEY_TARGET]
+                        ?: throw IllegalArgumentException("configuration key $KEY_TARGET missing"),
+                callback = callback))
+    }
+}

semanticdb-kotlinc/src/main/kotlin/com/sourcegraph/semanticdb_kotlinc/DeclarationExtensions.kt

@@ -0,0 +1,11 @@
+package com.sourcegraph.semanticdb_kotlinc
+
+import org.jetbrains.kotlin.descriptors.ClassDescriptor
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
+import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
+import org.jetbrains.kotlin.descriptors.impl.LocalVariableDescriptor
+
+fun DeclarationDescriptor.isObjectDeclaration(): Boolean =
+    this is ClassDescriptor && this.visibility == DescriptorVisibilities.LOCAL
+
+fun DeclarationDescriptor.isLocalVariable(): Boolean = this is LocalVariableDescriptor

semanticdb-kotlinc/src/main/kotlin/com/sourcegraph/semanticdb_kotlinc/DescriptorResolver.kt

@@ -0,0 +1,29 @@
+package com.sourcegraph.semanticdb_kotlinc
+
+import org.jetbrains.kotlin.descriptors.ConstructorDescriptor
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
+import org.jetbrains.kotlin.descriptors.ValueParameterDescriptor
+import org.jetbrains.kotlin.psi.*
+import org.jetbrains.kotlin.resolve.BindingContext
+import org.jetbrains.kotlin.resolve.BindingTrace
+import org.jetbrains.kotlin.types.KotlinType
+
+class DescriptorResolver(/* leave public for debugging */ val bindingTrace: BindingTrace) {
+    fun fromDeclaration(declaration: KtDeclaration): Sequence<DeclarationDescriptor> = sequence {
+        val descriptor = bindingTrace[BindingContext.DECLARATION_TO_DESCRIPTOR, declaration]
+        if (descriptor is ValueParameterDescriptor) {
+            bindingTrace[BindingContext.VALUE_PARAMETER_AS_PROPERTY, descriptor]?.let { yield(it) }
+        }
+        descriptor?.let { yield(it) }
+    }
+
+    fun syntheticConstructor(klass: KtClass): ConstructorDescriptor? =
+        bindingTrace[BindingContext.CONSTRUCTOR, klass]
+
+    fun fromReference(reference: KtReferenceExpression): DeclarationDescriptor? =
+        bindingTrace[BindingContext.REFERENCE_TARGET, reference]
+
+    fun fromTypeReference(reference: KtTypeReference): KotlinType =
+        bindingTrace[BindingContext.TYPE, reference]
+            ?: bindingTrace[BindingContext.ABBREVIATED_TYPE, reference]!!
+}

semanticdb-kotlinc/src/main/kotlin/com/sourcegraph/semanticdb_kotlinc/LineMap.kt

@@ -0,0 +1,38 @@
+package com.sourcegraph.semanticdb_kotlinc
+
+import org.jetbrains.kotlin.com.intellij.navigation.NavigationItem
+import org.jetbrains.kotlin.com.intellij.openapi.editor.Document
+import org.jetbrains.kotlin.com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.com.intellij.psi.PsiDocumentManager
+import org.jetbrains.kotlin.com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.diagnostics.PsiDiagnosticUtils
+import org.jetbrains.kotlin.diagnostics.PsiDiagnosticUtils.LineAndColumn
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi.KtPropertyAccessor
+
+/** Maps between an element and its identifier positions */
+class LineMap(project: Project, file: KtFile) {
+    private val document: Document = PsiDocumentManager.getInstance(project).getDocument(file)!!
+
+    private fun offsetToLineAndCol(offset: Int): LineAndColumn =
+        PsiDiagnosticUtils.offsetToLineAndColumn(document, offset)
+
+    /** Returns the non-0-based start character */
+    fun startCharacter(element: PsiElement): Int = offsetToLineAndCol(element.textOffset).column
+
+    /** Returns the non-0-based end character */
+    fun endCharacter(element: PsiElement): Int =
+        startCharacter(element) + nameForOffset(element).length
+
+    /** Returns the non-0-based line number */
+    fun lineNumber(element: PsiElement): Int = document.getLineNumber(element.textOffset) + 1
+
+    companion object {
+        fun nameForOffset(element: PsiElement): String =
+            when (element) {
+                is KtPropertyAccessor -> element.namePlaceholder.text
+                is NavigationItem -> element.name ?: element.text
+                else -> element.text
+            }
+    }
+}