[KAFKA-20032] KIP-1265 Automatic detection of internal API usage

build.gradle

@@ -315,6 +315,8 @@ subprojects {
   apply plugin: 'java-library'
   apply plugin: 'checkstyle'
   apply plugin: "com.github.spotbugs"
+  apply plugin: 'org.apache.kafka.public-api-checker'
+
 
   // We use the shadow plugin for the jmh-benchmarks module and the `-all` jar can get pretty large, so
   // don't publish it
@@ -698,6 +700,20 @@ subprojects {
 
   check.dependsOn('javadoc')
 
+kafkaPublicApiChecker {
+
+    // NEW: Add all project JARs for @PublicApi scanning
+    projectJarFiles.from(
+        jar.archiveFile,                    // Main JAR
+        // Add other relevant JARs if needed
+    )
+
+    // NEW: Enable/disable dual validation
+    enforceJavadocConsistency = true
+
+    enabled = true
+    failOnViolation = true
+}
   task systemTestLibs(dependsOn: jar)
 
   if (!sourceSets.test.allSource.isEmpty()) {

buildSrc/build.gradle

@@ -0,0 +1,192 @@
+plugins {
+    id 'java-gradle-plugin'
+    id 'maven-publish'
+    id 'java-library'
+}
+
+group = 'org.apache.kafka'
+version = project.findProperty('kafkaPluginsVersion') ?: '1.0.0-SNAPSHOT'
+
+repositories {
+    gradlePluginPortal()
+    mavenCentral()
+}
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_11
+    targetCompatibility = JavaVersion.VERSION_11
+}
+
+dependencies {
+    // For Maven plugin functionality
+    implementation 'org.apache.maven:maven-plugin-api:3.9.4'
+    implementation 'org.apache.maven:maven-core:3.9.4'
+    implementation 'org.apache.maven:maven-artifact:3.9.4'
+    implementation 'org.apache.maven.plugin-tools:maven-plugin-annotations:3.9.0'
+
+    // For testing
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.9.2'
+    testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.9.2'
+    testImplementation 'org.junit.platform:junit-platform-launcher:1.9.2'
+    testImplementation 'org.mockito:mockito-core:5.3.1'
+    testImplementation gradleTestKit()
+}
+
+gradlePlugin {
+    plugins {
+        kafkaPublicApiChecker {
+            id = 'org.apache.kafka.public-api-checker'
+            implementationClass = 'org.apache.kafka.gradle.KafkaPublicApiCheckerPlugin'
+            displayName = 'Kafka Public API Checker'
+            description = 'Internal plugin for checking public API consistency in Kafka codebase'
+        }
+
+        kafkaInternalApiChecker {
+            id = 'org.apache.kafka.internal-api-checker'
+            implementationClass = 'org.apache.kafka.gradle.KafkaInternalApiCheckerPlugin'
+            displayName = 'Kafka Internal API Checker'
+            description = 'Plugin for external projects to check they don\'t use internal Kafka APIs'
+        }
+    }
+}
+
+// Configure publishing for both Gradle and Maven plugins
+publishing {
+    publications {
+        // Maven plugin (separate from Gradle plugin publication)
+        mavenPlugin(MavenPublication) {
+            group  = 'org.apache.kafka'
+            artifactId = 'kafka-internal-api-checker-maven-plugin'
+            version = project.findProperty('kafkaPluginsVersion') ?: '1.0.0-SNAPSHOT'
+
+            from components.java
+
+            pom {
+                name = 'Apache Kafka Internal API Checker Maven Plugin'
+                description = 'Maven plugin to check that external projects don\'t use internal Kafka APIs'
+                url = 'https://kafka.apache.org'
+                packaging = 'maven-plugin'
+
+                licenses {
+                    license {
+                        name = 'The Apache License, Version 2.0'
+                        url = 'https://www.apache.org/licenses/LICENSE-2.0.txt'
+                    }
+                }
+
+                developers {
+                    developer {
+                        id = 'apache-kafka'
+                        name = 'Apache Kafka Team'
+                        email = '[email protected]'
+                    }
+                }
+
+                scm {
+                    connection = 'scm:git:https://github.com/apache/kafka.git'
+                    developerConnection = 'scm:git:https://github.com/apache/kafka.git'
+                    url = 'https://github.com/apache/kafka'
+                }
+            }
+        }
+    }
+
+    // Configure the automatically generated plugin marker publications
+    afterEvaluate {
+        publications.matching { it.name.endsWith('PluginMarkerMaven') }.configureEach {
+            pom {
+                name = 'Apache Kafka Gradle Plugins'
+                description = 'Gradle plugins for Apache Kafka public API validation'
+                url = 'https://kafka.apache.org'
+
+                licenses {
+                    license {
+                        name = 'The Apache License, Version 2.0'
+                        url = 'https://www.apache.org/licenses/LICENSE-2.0.txt'
+                    }
+                }
+
+                developers {
+                    developer {
+                        id = 'apache-kafka'
+                        name = 'Apache Kafka Team'
+                        email = '[email protected]'
+                    }
+                }
+
+                scm {
+                    connection = 'scm:git:https://github.com/apache/kafka.git'
+                    developerConnection = 'scm:git:https://github.com/apache/kafka.git'
+                    url = 'https://github.com/apache/kafka'
+                }
+            }
+        }
+
+        // Also configure the main plugin publication
+        publications.matching { it.name == 'pluginMaven' }.configureEach {
+            pom {
+                name = 'Apache Kafka Gradle Plugins'
+                description = 'Gradle plugins for Apache Kafka public API validation'
+                url = 'https://kafka.apache.org'
+
+                licenses {
+                    license {
+                        name = 'The Apache License, Version 2.0'
+                        url = 'https://www.apache.org/licenses/LICENSE-2.0.txt'
+                    }
+                }
+
+                developers {
+                    developer {
+                        id = 'apache-kafka'
+                        name = 'Apache Kafka Team'
+                        email = '[email protected]'
+                    }
+                }
+
+                scm {
+                    connection = 'scm:git:https://github.com/apache/kafka.git'
+                    developerConnection = 'scm:git:https://github.com/apache/kafka.git'
+                    url = 'https://github.com/apache/kafka'
+                }
+            }
+        }
+    }
+}
+
+test {
+    useJUnitPlatform()
+}
+
+// Task to run plugin validation
+tasks.register('validateKafkaPlugins') {
+    group = 'verification'
+    description = 'Validates that all plugins are properly configured'
+
+    doLast {
+        gradlePlugin.plugins.each { plugin ->
+            println "✅ Plugin '${plugin.id}' -> ${plugin.implementationClass}"
+        }
+
+        // Validate that source files exist
+        def pluginSourceFiles = [
+            'src/main/java/org/apache/kafka/gradle/KafkaPublicApiCheckerPlugin.java',
+            'src/main/java/org/apache/kafka/gradle/KafkaInternalApiCheckerPlugin.java',
+            'src/main/java/org/apache/kafka/publicapi/PublicApiChecker.java',
+            'src/main/java/org/apache/kafka/publicapi/PublicApiViolation.java',
+            'src/main/java/org/apache/kafka/publicapi/ViolationReporter.java',
+            'src/main/java/org/apache/kafka/maven/KafkaInternalApiCheckerMojo.java'
+        ]
+
+        pluginSourceFiles.each { sourceFile ->
+            def file = new File(project.projectDir, sourceFile)
+            if (file.exists()) {
+                println "✅ Source file exists: ${sourceFile}"
+            } else {
+                throw new GradleException("❌ Source file not found: ${sourceFile}")
+            }
+        }
+    }
+}
+
+check.dependsOn validateKafkaPlugins

buildSrc/src/main/java/org/apache/kafka/gradle/KafkaInternalApiCheckerExtension.java

@@ -0,0 +1,68 @@
+package org.apache.kafka.gradle;
+
+import org.gradle.api.Project;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.provider.Property;
+
+import java.io.File;
+import java.util.Arrays;
+
+/**
+ * Configuration extension for the KafkaInternalApiChecker plugin.
+ * This plugin is used by external projects to ensure they don't use internal Kafka APIs.
+ */
+public class KafkaInternalApiCheckerExtension {
+    private final Property<Boolean> enabled;
+    private final Property<Boolean> failOnViolation;
+    private final ConfigurableFileCollection sourceDirs;
+    private final RegularFileProperty reportFile;
+
+    public KafkaInternalApiCheckerExtension(Project project) {
+        this.enabled = project.getObjects().property(Boolean.class);
+        this.enabled.convention(true);
+
+        this.failOnViolation = project.getObjects().property(Boolean.class);
+        this.failOnViolation.convention(true);
+
+        this.sourceDirs = project.getObjects().fileCollection();
+        // Default to standard Java source directories
+        this.sourceDirs.from(project.file("src/main/java"));
+
+        this.reportFile = project.getObjects().fileProperty();
+        this.reportFile.convention(project.getLayout().getBuildDirectory().file("reports/kafka-internal-api-usage.txt"));
+    }
+
+    public Property<Boolean> getEnabled() {
+        return enabled;
+    }
+
+    public void setEnabled(boolean enabled) {
+        this.enabled.set(enabled);
+    }
+
+    public Property<Boolean> getFailOnViolation() {
+        return failOnViolation;
+    }
+
+    public void setFailOnViolation(boolean failOnViolation) {
+        this.failOnViolation.set(failOnViolation);
+    }
+
+    public ConfigurableFileCollection getSourceDirs() {
+        return sourceDirs;
+    }
+
+    public void setSourceDirs(Object... sourceDirs) {
+        this.sourceDirs.setFrom(sourceDirs);
+    }
+
+    public RegularFileProperty getReportFile() {
+        return reportFile;
+    }
+
+    public void setReportFile(File reportFile) {
+        this.reportFile.set(reportFile);
+    }
+}

buildSrc/src/main/java/org/apache/kafka/gradle/KafkaInternalApiCheckerPlugin.java

@@ -0,0 +1,52 @@
+package org.apache.kafka.gradle;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.tasks.TaskProvider;
+
+/**
+ * Gradle plugin for checking that external projects don't use internal Kafka APIs.
+ * This plugin is intended to be published and used by external Kafka plugin/application developers.
+ */
+public class KafkaInternalApiCheckerPlugin implements Plugin<Project> {
+
+    @Override
+    public void apply(Project project) {
+        // Create the extension for configuration
+        KafkaInternalApiCheckerExtension extension = project.getExtensions()
+            .create("kafkaInternalApiChecker", KafkaInternalApiCheckerExtension.class, project);
+
+        // Register the task
+        TaskProvider<KafkaInternalApiCheckerTask> taskProvider = project.getTasks()
+            .register("kafkaInternalApiChecker", KafkaInternalApiCheckerTask.class, task -> {
+                task.getCheckerEnabled().set(extension.getEnabled());
+                task.getFailOnViolation().set(extension.getFailOnViolation());
+                task.getSourceDirs().set(extension.getSourceDirs());
+                task.getReportFile().set(extension.getReportFile());
+            });
+
+        // Configure task to run as part of verification
+        project.afterEvaluate(p -> {
+
+            TaskProvider<DefaultTask> compileJava = project.getTasks().named("compileJava", DefaultTask.class);
+
+            // Integrate into the standard build lifecycle - add to 'check' task
+            project.getTasks().named("check").configure(checkTask -> {
+                checkTask.dependsOn(taskProvider);
+            });
+        });
+
+        // Add helpful task to skip checking
+        project.getTasks().register("skipInternalApiCheck", task -> {
+            task.setGroup("verification");
+            task.setDescription("Disables internal API checking for this build");
+            task.doLast(t -> {
+                extension.getEnabled().set(false);
+                project.getLogger().info("Internal API checking disabled for this build");
+            });
+        });
+
+        project.getLogger().debug("Applied KafkaInternalApiChecker plugin to project: {}", project.getName());
+    }
+}