Add lexer token counting to kotlincd_actions telemetry

Summary:
Add num_kotlin_tokens and num_java_tokens fields to the kotlincd_actions
Scuba table. Token counts are computed at compilation time using
KotlinLexer for .kt/.kts files and StreamTokenizer for .java files.

This enables measuring Kotlin build speed normalized by token count
(tokens/second) to identify abnormally slow targets and compare across
apps. Token counting is best-effort — failures are caught and logged
as extras without breaking the build.

Performance impact: benchmarking with a realistic workload (200 .kt files
at ~6.7 KB each + 50 .java files at ~5.4 KB each, ~1.6 MB total) shows
token counting completes in ~22 ms on average (~0.09 ms per file). This
is negligible relative to compilation time (seconds to minutes), and
source file contents will already be in the OS page cache since the
compiler reads the same files immediately after.

Differential Revision: D94540188

fbshipit-source-id: d49877ae92a87b5c061618ff74299d39cbb1d111

Author: RuslanLa

prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/buildtools/BuildToolsKotlinc.kt

@@ -16,6 +16,7 @@ import com.facebook.buck.core.filesystems.RelPath
 import com.facebook.buck.core.util.log.Logger
 import com.facebook.buck.jvm.core.BuildTargetValue
 import com.facebook.buck.jvm.kotlin.cd.analytics.KotlinCDLoggingContext
+import com.facebook.buck.jvm.kotlin.cd.analytics.SourceTokenCounter
 import com.facebook.buck.jvm.kotlin.kotlinc.Kotlinc
 import com.facebook.buck.jvm.kotlin.kotlinc.incremental.KotlincMode
 import com.facebook.buck.util.ClassLoaderCache
@@ -134,6 +135,18 @@ class BuildToolsKotlinc : Kotlinc {
           )
         }
 
+    try {
+      val tokenCounts = SourceTokenCounter.countTokens(expandedSources, ruleCellRoot.path)
+      kotlinCDLoggingContext.numKotlinTokens = tokenCounts.kotlinTokens
+      kotlinCDLoggingContext.numJavaTokens = tokenCounts.javaTokens
+    } catch (e: Exception) {
+      // Token counting is best-effort telemetry; don't fail the build
+      kotlinCDLoggingContext.addExtras(
+          BuildToolsKotlinc::class.java.simpleName,
+          "Token counting failed: ${e.message}",
+      )
+    }
+
     val resolvedExpandedSources =
         expandedSources.map { path -> ruleCellRoot.resolve(path).toString() }
 

prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/BUCK

@@ -17,5 +17,6 @@ buck_kotlin_library(
     deps = [
         "prelude//toolchains/android/src/com/facebook/buck/core/filesystems:filesystems",
         "prelude//toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin:language-version",
+        "prelude//toolchains/android/third-party:kotlin-compiler",
     ],
 )

prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/KotlinCDLoggingContext.kt

@@ -21,6 +21,9 @@ class KotlinCDLoggingContext(
 
   val classpathChangesParam = (mode as? ModeParam.Incremental)?.classpathChangesParam
 
+  var numKotlinTokens: Long = 0L
+  var numJavaTokens: Long = 0L
+
   private val _extras: MutableMap<String, MutableList<String>> = mutableMapOf()
   val extras: Map<String, List<String>>
     get() = _extras

prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/SourceTokenCounter.kt

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is dual-licensed under either the MIT license found in the
+ * LICENSE-MIT file in the root directory of this source tree or the Apache
+ * License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+ * of this source tree. You may select, at your option, one of the
+ * above-listed licenses.
+ */
+
+package com.facebook.buck.jvm.kotlin.cd.analytics
+
+import java.nio.file.Files
+import java.nio.file.Path
+import org.jetbrains.kotlin.lexer.KotlinLexer
+import org.jetbrains.kotlin.lexer.KtTokens
+
+object SourceTokenCounter {
+  fun countTokens(sourcePaths: List<Path>, rootPath: Path): TokenCounts {
+    var kotlinTokens = 0L
+    var javaTokens = 0L
+    for (path in sourcePaths) {
+      val resolved = rootPath.resolve(path)
+      val ext = path.toString().substringAfterLast('.', "")
+      when (ext) {
+        "kt",
+        "kts" -> kotlinTokens += countKotlinTokens(resolved)
+        "java" -> javaTokens += countJavaTokens(resolved)
+      }
+    }
+    return TokenCounts(kotlinTokens, javaTokens)
+  }
+
+  private val SKIPPED_TOKEN_TYPES =
+      setOf(
+          KtTokens.WHITE_SPACE,
+          KtTokens.EOL_COMMENT,
+          KtTokens.BLOCK_COMMENT,
+          KtTokens.DOC_COMMENT,
+          KtTokens.SHEBANG_COMMENT,
+      )
+
+  private fun countKotlinTokens(file: Path): Long {
+    val text = Files.readString(file)
+    val lexer = KotlinLexer()
+    lexer.start(text)
+    var count = 0L
+    while (lexer.tokenType != null) {
+      if (lexer.tokenType !in SKIPPED_TOKEN_TYPES) {
+        count++
+      }
+      lexer.advance()
+    }
+    return count
+  }
+
+  private fun countJavaTokens(file: Path): Long {
+    // Use StreamTokenizer for Java - lighter weight, no extra deps
+    val text = Files.readString(file)
+    val reader = java.io.StreamTokenizer(java.io.StringReader(text))
+    reader.slashSlashComments(true)
+    reader.slashStarComments(true)
+    reader.ordinaryChar('/'.code)
+    var count = 0L
+    while (reader.nextToken() != java.io.StreamTokenizer.TT_EOF) {
+      count++
+    }
+    return count
+  }
+}
+
+data class TokenCounts(val kotlinTokens: Long, val javaTokens: Long) {
+  val totalTokens: Long
+    get() = kotlinTokens + javaTokens
+}

prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalytics.kt

@@ -89,6 +89,8 @@ constructor(
         extras = buildJson(context.extras),
         addedAndModifiedFiles = addedAndModifiedFiles,
         removedFiles = removedFiles,
+        numKotlinTokens = context.numKotlinTokens.takeIf { it > 0 },
+        numJavaTokens = context.numJavaTokens.takeIf { it > 0 },
     )
   }
 

prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/model/KotlinCDLogEntry.kt

@@ -27,4 +27,6 @@ data class KotlinCDLogEntry(
     val extras: String?,
     val addedAndModifiedFiles: Set<String>?,
     val removedFiles: Set<String>?,
+    val numKotlinTokens: Long? = null,
+    val numJavaTokens: Long? = null,
 )

prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalyticsTest.kt

@@ -165,15 +165,42 @@ internal class KotlinCDLoggerAnalyticsTest {
     verify(kotlinCDLogger, times(1)).log(expectedEntry)
   }
 
+  @Test
+  fun `when token counts are set, they are logged`() {
+    val kotlinCDAnalytics = createFakeKotlinCDAnalytics()
+    val expectedEntry =
+        createExpectedKotlinCDLogEntry(numKotlinTokens = 1000L, numJavaTokens = 500L)
+
+    kotlinCDAnalytics.log(
+        createKotlinCDLoggingContext(numKotlinTokens = 1000L, numJavaTokens = 500L)
+    )
+
+    verify(kotlinCDLogger, times(1)).log(expectedEntry)
+  }
+
+  @Test
+  fun `when token counts are zero, they are not logged`() {
+    val kotlinCDAnalytics = createFakeKotlinCDAnalytics()
+    val expectedEntry = createExpectedKotlinCDLogEntry()
+
+    kotlinCDAnalytics.log(createKotlinCDLoggingContext(numKotlinTokens = 0L, numJavaTokens = 0L))
+
+    verify(kotlinCDLogger, times(1)).log(expectedEntry)
+  }
+
   private fun createKotlinCDLoggingContext(
       step: StepParam = StepParam.KOTLINC,
       languageVersion: String = DEFAULT_LANGUAGE_VERSION,
       kotlincMode: ModeParam? =
           ModeParam.Incremental(ClasspathChangesParam.NO_CHANGES, emptySet(), emptySet()),
       extras: Map<String, List<String>> = mapOf(),
+      numKotlinTokens: Long = 0L,
+      numJavaTokens: Long = 0L,
   ): KotlinCDLoggingContext {
     val context = KotlinCDLoggingContext(step, LanguageVersion(languageVersion), kotlincMode)
     extras.forEach { (key, extras) -> extras.forEach { item -> context.addExtras(key, item) } }
+    context.numKotlinTokens = numKotlinTokens
+    context.numJavaTokens = numJavaTokens
     return context
   }
 
@@ -198,6 +225,8 @@ internal class KotlinCDLoggerAnalyticsTest {
       extras: String? = null,
       modifiedFiles: Set<String> = emptySet(),
       removedFiles: Set<String> = emptySet(),
+      numKotlinTokens: Long? = null,
+      numJavaTokens: Long? = null,
   ) =
       KotlinCDLogEntry(
           time = Instant.now(clock).epochSecond,
@@ -216,6 +245,8 @@ internal class KotlinCDLoggerAnalyticsTest {
           extras = extras,
           addedAndModifiedFiles = modifiedFiles,
           removedFiles = removedFiles,
+          numKotlinTokens = numKotlinTokens,
+          numJavaTokens = numJavaTokens,
       )
 
   companion object TestParams {

prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/cd/analytics/logger/SourceTokenCounterTest.kt

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is dual-licensed under either the MIT license found in the
+ * LICENSE-MIT file in the root directory of this source tree or the Apache
+ * License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+ * of this source tree. You may select, at your option, one of the
+ * above-listed licenses.
+ */
+
+package com.facebook.buck.jvm.kotlin.cd.analytics
+
+import java.nio.file.Path
+import org.junit.Assert.assertEquals
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+
+internal class SourceTokenCounterTest {
+
+  @get:Rule val tempDir = TemporaryFolder()
+
+  @Test
+  fun `countTokens returns zero for empty source list`() {
+    val result = SourceTokenCounter.countTokens(emptyList(), tempDir.root.toPath())
+    assertEquals(0L, result.kotlinTokens)
+    assertEquals(0L, result.javaTokens)
+    assertEquals(0L, result.totalTokens)
+  }
+
+  @Test
+  fun `countTokens counts Kotlin tokens`() {
+    val ktFile = tempDir.newFile("Test.kt")
+    ktFile.writeText("fun main() { println(\"hello\") }")
+    val sourcePaths = listOf(Path.of("Test.kt"))
+
+    val result = SourceTokenCounter.countTokens(sourcePaths, tempDir.root.toPath())
+
+    assert(result.kotlinTokens > 0) { "Expected positive Kotlin token count" }
+    assertEquals(0L, result.javaTokens)
+    assertEquals(result.kotlinTokens, result.totalTokens)
+  }
+
+  @Test
+  fun `countTokens counts Java tokens`() {
+    val javaFile = tempDir.newFile("Test.java")
+    javaFile.writeText("public class Test { public static void main(String[] args) {} }")
+    val sourcePaths = listOf(Path.of("Test.java"))
+
+    val result = SourceTokenCounter.countTokens(sourcePaths, tempDir.root.toPath())
+
+    assertEquals(0L, result.kotlinTokens)
+    assert(result.javaTokens > 0) { "Expected positive Java token count" }
+    assertEquals(result.javaTokens, result.totalTokens)
+  }
+
+  @Test
+  fun `countTokens counts both Kotlin and Java files`() {
+    val ktFile = tempDir.newFile("Test.kt")
+    ktFile.writeText("val x = 1")
+    val javaFile = tempDir.newFile("Test.java")
+    javaFile.writeText("class Test { int x = 1; }")
+    val sourcePaths = listOf(Path.of("Test.kt"), Path.of("Test.java"))
+
+    val result = SourceTokenCounter.countTokens(sourcePaths, tempDir.root.toPath())
+
+    assert(result.kotlinTokens > 0) { "Expected positive Kotlin token count" }
+    assert(result.javaTokens > 0) { "Expected positive Java token count" }
+    assertEquals(result.kotlinTokens + result.javaTokens, result.totalTokens)
+  }
+
+  @Test
+  fun `countTokens handles kts extension`() {
+    val ktsFile = tempDir.newFile("build.gradle.kts")
+    ktsFile.writeText("plugins { id(\"java\") }")
+    val sourcePaths = listOf(Path.of("build.gradle.kts"))
+
+    val result = SourceTokenCounter.countTokens(sourcePaths, tempDir.root.toPath())
+
+    assert(result.kotlinTokens > 0) { "Expected positive Kotlin token count for .kts file" }
+  }
+
+  @Test
+  fun `countTokens ignores non-kotlin-non-java files`() {
+    val txtFile = tempDir.newFile("readme.txt")
+    txtFile.writeText("this is not source code")
+    val sourcePaths = listOf(Path.of("readme.txt"))
+
+    val result = SourceTokenCounter.countTokens(sourcePaths, tempDir.root.toPath())
+
+    assertEquals(0L, result.kotlinTokens)
+    assertEquals(0L, result.javaTokens)
+  }
+
+  @Test
+  fun `countTokens excludes whitespace and comments from Kotlin token count`() {
+    val ktFile = tempDir.newFile("Commented.kt")
+
+    // Code without comments
+    ktFile.writeText("val x = 1")
+    val baseResult =
+        SourceTokenCounter.countTokens(listOf(Path.of("Commented.kt")), tempDir.root.toPath())
+
+    // Same code with added comments and extra whitespace — should yield same count
+    ktFile.writeText(
+        "// line comment\nval   x   =   1 /* block comment */ /** doc comment */\n// trailing"
+    )
+    val commentedResult =
+        SourceTokenCounter.countTokens(listOf(Path.of("Commented.kt")), tempDir.root.toPath())
+
+    assertEquals(baseResult.kotlinTokens, commentedResult.kotlinTokens)
+  }
+
+  @Test
+  fun `countTokens handles empty source file`() {
+    val ktFile = tempDir.newFile("Empty.kt")
+    ktFile.writeText("")
+    val sourcePaths = listOf(Path.of("Empty.kt"))
+
+    val result = SourceTokenCounter.countTokens(sourcePaths, tempDir.root.toPath())
+
+    assertEquals(0L, result.kotlinTokens)
+  }
+}