Merge pull request #13 from vivetiwa/upgrade-kafka-version-java-11

Upgrade kafka version java 11

Author: nipunnair

Dockerfile

@@ -10,10 +10,10 @@
  # governing permissions and limitations under the License.
 ##
 
-FROM anapsix/alpine-java:jdk8
+FROM adoptopenjdk/openjdk11:jre-11.0.11_9-alpine
 
 ENV SCALA_VERSION="2.12" \
-    KAFKA_VERSION="2.3.0"
+    KAFKA_VERSION="2.8.0"
 ENV KAFKA_HOME=/opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION}
 
 ARG KAFKA_DIST=kafka_${SCALA_VERSION}-${KAFKA_VERSION}
@@ -23,7 +23,7 @@ ARG KAFKA_DIST_ASC=${KAFKA_DIST}.tgz.asc
 RUN set -x && \
     apk add --no-cache unzip curl ca-certificates gnupg jq && \
     eval $(gpg-agent --daemon) && \
-    MIRROR=`curl -sSL https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred'` && \
+    MIRROR=`curl -sSL https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.http[0]'` && \
     curl -sSLO "${MIRROR}kafka/${KAFKA_VERSION}/${KAFKA_DIST_TGZ}" && \
     curl -sSLO https://dist.apache.org/repos/dist/release/kafka/${KAFKA_VERSION}/${KAFKA_DIST_ASC} && \
     curl -sSL  https://kafka.apache.org/KEYS | gpg -q --import - && \
@@ -32,11 +32,13 @@ RUN set -x && \
     mv ${KAFKA_DIST_TGZ} /tmp && \
     tar xfz /tmp/${KAFKA_DIST_TGZ} -C /opt && \
     rm /tmp/${KAFKA_DIST_TGZ} && \
-    apk del unzip ca-certificates gnupg
+    apk del unzip ca-certificates gnupg && \
+    apk add bash
 
 ENV PATH=$PATH:/${KAFKA_HOME}/bin \
     CONNECT_CFG=${KAFKA_HOME}/config/connect-distributed.properties \
-    CONNECT_BIN=${KAFKA_HOME}/bin/connect-distributed.sh
+    CONNECT_BIN=${KAFKA_HOME}/bin/connect-distributed.sh \
+    CONNECT_LOG_CFG=$KAFKA_HOME/config/log4j.properties
 
 ENV CONNECT_PORT=8083
 ENV EXTRA_ARGS="-javaagent:$KAFKA_HOME/connectors/jmx_prometheus_javaagent-0.12.0.jar=9999:${KAFKA_HOME}/config/prometheus.yml"

build.gradle

@@ -49,7 +49,8 @@ subprojects {
   version = System.getenv("PROJECT_VERSION") ?: "${versionMain}${versionQualifier}"
 
   checkstyle {
-    configFile = '../config/checkstyle/checkstyle.xml' as File
+    configFile = rootProject.file('config/checkstyle/checkstyle.xml')
+    configProperties 'checkstyle.header.file': file('../config/checkstyle/adobe.header.txt')
     maxWarnings = 0
     toolVersion '8.10.1'
   }
@@ -101,7 +102,7 @@ subprojects {
   }
 
   jacoco {
-    toolVersion = "0.8.1"
+    toolVersion = "0.8.6"
   }
 
   jacocoTestReport {

config/checkstyle/checkstyle.xml

@@ -11,7 +11,7 @@
 <!-- governing permissions and limitations under the License.-->
 <!DOCTYPE module PUBLIC
   "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
-  "http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
+  "https://checkstyle.org/dtds/configuration_1_3.dtd">
 
 <module name="Checker">
   <property name="charset" value="UTF-8"/>
@@ -29,7 +29,7 @@
   <module name="NewlineAtEndOfFile"/>
 
   <module name="RegexpHeader">
-    <property name="headerFile" value="config/checkstyle/adobe.header.txt"/>
+    <property name="headerFile" value="${checkstyle.header.file}"/>
   </module>
 
   <module name="TreeWalker">

dependencies.gradle

@@ -26,8 +26,9 @@ versions.jmxPrometheusJavaAgent = '0.12.0'
 versions.junitJupiter = '5.2.0'
 versions.jjwt = '0.9.0'
 versions.orgJson = '20180130'
-versions.kafkaConnect = '2.3.0'
+versions.kafkaConnect = '2.8.0'
 versions.slf4j = '1.7.25'
+versions.jaxb = '2.3.0'
 
 libraries.collections4 = [
   "org.apache.commons:commons-collections4:$versions.collections4"
@@ -84,3 +85,7 @@ libraries.orgJson = [
 libraries.slf4j = [
   "org.slf4j:slf4j-api:$versions.slf4j"
 ]
+
+libraries.jaxb = [
+  "javax.xml.bind:jaxb-api:$versions.jaxb"
+]

docker-entrypoint.sh

@@ -47,6 +47,37 @@ export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true \
   -Dcom.sun.management.jmxremote.authenticate=false \
   -Dcom.sun.management.jmxremote.ssl=false"
 
+#
+# Configure the log files ...
+#
+if [[ -n "$CONNECT_LOG4J_LOGGERS" ]]; then
+  sed -i -r -e "s|^(log4j.rootLogger)=.*|\1=${CONNECT_LOG4J_LOGGERS}|g" $CONNECT_LOG_CFG
+  unset CONNECT_LOG4J_LOGGERS
+fi
+env | grep '^CONNECT_LOG4J' | while read -r VAR;
+do
+  env_var=`echo "$VAR" | sed -r "s/([^=]*)=.*/\1/g"`
+  prop_name=`echo "$VAR" | sed -r "s/^CONNECT_([^=]*)=.*/\1/g" | tr '[:upper:]' '[:lower:]' | tr _ .`
+  prop_value=`echo "$VAR" | sed -r "s/^CONNECT_[^=]*=(.*)/\1/g"`
+  if egrep -q "(^|^#)$prop_name=" $CONNECT_LOG_CFG; then
+    #note that no config names or values may contain an '@' char
+    sed -r -i "s@(^|^#)($prop_name)=(.*)@\2=${prop_value}@g" $CONNECT_LOG_CFG
+  else
+    echo "$prop_name=${prop_value}" >> $CONNECT_LOG_CFG
+  fi
+  if [[ "$SENSITIVE_PROPERTIES" = *"$env_var"* ]]; then
+    echo "--- Setting logging property from $env_var: $prop_name=[hidden]"
+  else
+    echo "--- Setting logging property from $env_var: $prop_name=${prop_value}"
+  fi
+  unset $env_var
+done
+
+if [[ -n "$LOG_LEVEL" ]]; then
+  sed -i -r -e "s|=INFO, stdout|=$LOG_LEVEL, stdout|g" $CONNECT_LOG_CFG
+  sed -i -r -e "s|^(log4j.appender.stdout.threshold)=.*|\1=${LOG_LEVEL}|g" $CONNECT_LOG_CFG
+fi
+
 # Extend CLASSPATH for custom connectors
 export CLASSPATH=${CLASSPATH}:${KAFKA_HOME}/connectors/*
 

streaming-connect-common/build.gradle

@@ -14,7 +14,7 @@ apply from: '../dependencies.gradle'
 
 dependencies {
   libraries.with {
-    compile collections4, commonslang, guava, httpClient, jackson, jmockit, junitJupiter, jjwt, orgJson, slf4j
+    compile collections4, commonslang, guava, httpClient, jackson, jmockit, junitJupiter, jjwt, orgJson, slf4j, jaxb
     testCompile jmockit, junitJupiter
   }
 }

streaming-connect-common/src/test/java/com/adobe/platform/streaming/auth/impl/JWTTokenProviderTest.java

@@ -22,7 +22,7 @@
  */
 class JWTTokenProviderTest {
 
-  private static final String ENDPOINT = "https://ims-na1-stg1.adobelogin.com";
+  private static final String ENDPOINT = "https://ims-na1.adobelogin.com";
   private static final String TEST_CLIENT = "testClientId";
   private static final String TEST_ORG = "testImsOrg";
   private static final String TEST_ACCOUNT_ID = "testTechnicalAccountId";

streaming-connect-sink/src/test/java/com/adobe/platform/streaming/sink/utils/SinkUtilsTest.java

@@ -58,3 +58,4 @@ void testGetPropertyWithMultiplier() {
     ));
   }
 }
+