diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 7d4149c8a..d809e2f24 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -72,7 +72,7 @@ global_job_config: - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_DEV_REGISTRY - export LATEST_TAG=$BRANCH_TAG-latest - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" - - export DOCKER_REPOS="confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm" + - export DOCKER_REPOS="confluentinc/cp-base-refresh confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm" - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 diff --git a/base/Docker-refresh b/base/Docker-refresh new file mode 100644 index 000000000..ae953d754 --- /dev/null +++ b/base/Docker-refresh @@ -0,0 +1,32 @@ +ARG MICRODIR=/microdir +ARG PACKAGES_TO_INSTALL="temurin-17-jre shadow-utils" + +FROM registry.access.redhat.com/ubi8 AS BUILD +ARG MICRODIR +ARG PACKAGES_TO_INSTALL +RUN mkdir ${MICRODIR} + +RUN printf "[temurin-jre] \n\ +name=temurin-jre \n\ +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo + +RUN yum --nodocs install -y --setopt=install_weak_deps=False ${PACKAGES_TO_INSTALL} + +RUN yum clean all + +FROM registry.access.redhat.com/ubi8-micro AS REFRESH +ARG MICRODIR +COPY --from=BUILD /usr/lib/jvm/temurin-17-jre/ ${MICRODIR}/jre/ + +ENV PATH="${MICRODIR}/jre/bin:${PATH}" + +RUN echo "appuser:x:1001:1001::/home/appuser:/bin/sh" >> /etc/passwd && \ + mkdir -p /home/appuser && \ + chown 1001:1001 /home/appuser + +USER appuser +WORKDIR /home/appuser diff --git a/pom.xml b/pom.xml index ae82f30b1..f568f634b 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ utility-belt docker-utils + refresh base base-lite jmxterm @@ -34,6 +35,7 @@ 7.9.0-0 8.10-1086 + 8.10-13 3.0.9 diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 new file mode 100644 index 000000000..3a849067b --- /dev/null +++ b/refresh/Dockerfile.ubi8 @@ -0,0 +1,74 @@ +ARG MICRODIR=/microdir +ARG UBI_MICRO_VERSION=8.10-13 +ARG TEMURIN_JDK_VERSION="17.0.13.0.0.11-2" +ARG DOCKER_UPSTREAM_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" +ARG DOCKER_UPSTREAM_TAG +ARG GOLANG_VERSION +ARG UBI_MINIMAL_VERSION="latest" + +FROM docker.io/golang:${GOLANG_VERSION} AS build_package_dedupe +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser package_dedupe/package_dedupe.go ./ +RUN go build -ldflags="-w -s" ./package_dedupe.go + +FROM registry.access.redhat.com/ubi8 AS BUILD + +RUN printf "[temurin-jre] \n\ +name=temurin-jre \n\ +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo +RUN echo "installing temurin-17-jre:${TEMURIN_JDK_VERSION}" +RUN yum --nodocs install -y --setopt=install_weak_deps=False temurin-17-jre + +RUN yum clean all + +FROM 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/confluentinc/cp-base-lite:7.9.x-latest-ubi8 AS BASE-LITE + +FROM registry.access.redhat.com/ubi8-minimal:${UBI_MINIMAL_VERSION} AS REFRESH +ARG PROJECT_VERSION +ARG ARTIFACT_ID + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for new Confluent lightweight Docker images." +LABEL description="Common base image for Confluent lightweight Docker images." +LABEL io.confluent.docker=true +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" + +#ARG MICRODIR +COPY --from=BUILD /usr/lib/jvm/temurin-17-jre/ usr/lib +COPY --from=BASE-LITE /usr/bin /usr/bin +COPY --from=build_package_dedupe /build/package_dedupe /usr/lib/bin/package_dedupe + +ENV PATH="/usr/lib/bin:${PATH}" + +RUN echo "appuser:x:1001:1001::/home/appuser:/bin/sh" >> /etc/passwd && \ + mkdir -p /home/appuser && \ + chown 1001:1001 /home/appuser + +COPY target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ + +RUN cd /usr/share/java \ + && package_dedupe $(pwd) + +USER appuser +WORKDIR /home/appuser diff --git a/refresh/include/etc/confluent/docker/bash-config b/refresh/include/etc/confluent/docker/bash-config new file mode 100644 index 000000000..43c08d9b5 --- /dev/null +++ b/refresh/include/etc/confluent/docker/bash-config @@ -0,0 +1,23 @@ +# +# Copyright 2018 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset \ + -o errexit + +# Trace may expose passwords/credentials by printing them to stdout, so turn on with care. +if [ "${TRACE:-}" == "true" ]; then + set -o verbose \ + -o xtrace +fi diff --git a/refresh/include/etc/confluent/docker/mesos-setup.sh b/refresh/include/etc/confluent/docker/mesos-setup.sh new file mode 100644 index 000000000..b3874daf6 --- /dev/null +++ b/refresh/include/etc/confluent/docker/mesos-setup.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set +o nounset + +if [ -z $SKIP_MESOS_AUTO_SETUP ]; then + if [ -n $MESOS_SANDBOX ] && [ -e $MESOS_SANDBOX/.ssl/scheduler.crt ] && [ -e $MESOS_SANDBOX/.ssl/scheduler.key ]; then + echo "Entering Mesos auto setup for Java SSL truststore. You should not see this if you are not on mesos ..." + + openssl pkcs12 -export -in $MESOS_SANDBOX/.ssl/scheduler.crt -inkey $MESOS_SANDBOX/.ssl/scheduler.key \ + -out /tmp/keypair.p12 -name keypair \ + -CAfile $MESOS_SANDBOX/.ssl/ca-bundle.crt -caname root -passout pass:export + + keytool -importkeystore \ + -deststorepass changeit -destkeypass changeit -destkeystore /tmp/kafka-keystore.jks \ + -srckeystore /tmp/keypair.p12 -srcstoretype PKCS12 -srcstorepass export \ + -alias keypair + + keytool -import \ + -trustcacerts \ + -alias root \ + -file $MESOS_SANDBOX/.ssl/ca-bundle.crt \ + -storepass changeit \ + -keystore /tmp/kafka-truststore.jks -noprompt + fi +fi + +set -o nounset diff --git a/refresh/include/etc/cp-base-new/log4j.properties b/refresh/include/etc/cp-base-new/log4j.properties new file mode 100644 index 000000000..40fd0b732 --- /dev/null +++ b/refresh/include/etc/cp-base-new/log4j.properties @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +log4j.rootLogger=OFF + +# Only log errors from Kafka and ZKClient +log4j.logger.org.apache.kafka=ERROR +log4j.logger.org.I0Itec.zkclient.ZkClient=ERROR + +# Log informational messages from the CLI and Zookeeper +log4j.logger.io.confluent.admin.utils=INFO, stderr +log4j.logger.org.apache.zookeeper=INFO, stderr +# STDERR Appender +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout.ConversionPattern=%m%n diff --git a/refresh/package_dedupe/package_dedupe.go b/refresh/package_dedupe/package_dedupe.go new file mode 100644 index 000000000..7af27b59d --- /dev/null +++ b/refresh/package_dedupe/package_dedupe.go @@ -0,0 +1,68 @@ +package main + +import ( + "crypto/sha1" + + "fmt" + + "io" + "log" + "os" + "path/filepath" +) + +func dedupe_packages(rootPath string) { + sha2path := make(map[string]string) + err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil + } + sha, err := shaSum(path) + if err != nil { + return err + } + if orig, exists := sha2path[sha]; exists { + relPath, err := filepath.Rel(filepath.Dir(path), orig) + if err != nil { + return err + } + os.Remove(path) + err = os.Symlink(relPath, path) + if err != nil { + return err + } + log.Printf("DEDUP: ln -sf %s %s\n", orig, path) + } else { + sha2path[sha] = path + } + return nil + }) + if err != nil { + log.Fatal(err) + } +} + +func shaSum(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha1.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func main() { + if len(os.Args) != 2 { + fmt.Println("Usage: dedupe_packages ") + os.Exit(1) + } + basePath := os.Args[1] + dedupe_packages(basePath) +} \ No newline at end of file diff --git a/refresh/pom.xml b/refresh/pom.xml new file mode 100644 index 000000000..9b6fb8770 --- /dev/null +++ b/refresh/pom.xml @@ -0,0 +1,120 @@ + + + + + 4.0.0 + + + io.confluent + common-docker + 7.9.0-0 + + + pom + + cp-base-jre17 + + Refreshed Base for Confluent Docker images + ${project.artifactId} + + + false + false + true + + + + + io.confluent + utility-belt + ${io.confluent.common-docker.version} + + + junit + junit + ${junit.version} + test + + + + + + + maven-resources-plugin + ${maven-resources-plugin.version} + + + copy-resources + compile + + copy-resources + + + true + target + + + ${basedir}/../docker-utils/target + + docker-utils-${CONFLUENT_VERSION}-jar-with-dependencies.jar + + + + + + + + + com.spotify + dockerfile-maven-plugin + + + ${ubi.image.version} + ${ubi.micro.image.version} + -${ubi.temurin.jdk.version} + ${docker.skip-security-update-check} + ${golang.version} + + + + + io.fabric8 + docker-maven-plugin + 0.43.4 + + + + + + ${ubi.image.version} + ${ubi.micro.image.version} + -${ubi.temurin.jdk.version} + + ${docker.skip-security-update-check} + + ${golang.version} + + + + + + + + + diff --git a/refresh/requirements.txt b/refresh/requirements.txt new file mode 100644 index 000000000..879b1e1d1 --- /dev/null +++ b/refresh/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/confluentinc/confluent-docker-utils@v0.0.96 diff --git a/refresh/setup.py b/refresh/setup.py new file mode 100644 index 000000000..9c69fc03c --- /dev/null +++ b/refresh/setup.py @@ -0,0 +1,23 @@ +from setuptools import setup + + +setup( + name='common-tests', + version='4.1.0', + + author="Confluent, Inc.", + + description='Docker image tests', + + url="https://github.com/confluentinc/common", + + dependency_links=open('requirements.txt').read().split("\n"), + + packages=['test'], + + include_package_data=True, + + python_requires='>=2.7', + setup_requires=['setuptools-git'], + +) diff --git a/refresh/test/test_refresh_image.py b/refresh/test/test_refresh_image.py new file mode 100644 index 000000000..94ae271f7 --- /dev/null +++ b/refresh/test/test_refresh_image.py @@ -0,0 +1,28 @@ +import os +import unittest + +import confluent.docker_utils as utils + + +class BaseRefreshImageTest(unittest.TestCase): + + def setUp(self): + self.image = "{0}confluentinc/cp-base-refresh:{1}".format(os.environ["DOCKER_REGISTRY"], os.environ["DOCKER_TAG"]) + + def test_image_build(self): + self.assertTrue(utils.image_exists(self.image)) + + def test_jre_17_installed(self): + jre_cmd = "java --version" + result = utils.run_docker_command(image=self.image, command=jre_cmd) + self.assertTrue(b'17' in result) + + def test_ub_exists(self): + self.assertTrue(utils.path_exists_in_image(self.image, "/usr/bin/ub")) + + def test_ub_runnable(self): + ub_cmd = "bash -c '/usr/bin/ub -h'" + self.assertTrue(b"utility commands" in utils.run_docker_command(image=self.image, command=ub_cmd)) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/refresh/tox.ini b/refresh/tox.ini new file mode 100644 index 000000000..af5b6fe10 --- /dev/null +++ b/refresh/tox.ini @@ -0,0 +1,41 @@ +[tox] +envlist = test +toxworkdir = /var/tmp + +[testenv] +deps = + -rrequirements.txt + flake8 + pytest == 4.6.4 + pytest-xdist == 1.29.0 + pytest-cov == 2.7.1 + sphinx!=1.2b2,<2.0.0 +install_command = pip install -U {packages} +recreate = True +skipsdist = True +usedevelop = True +setenv = + PIP_PROCESS_DEPENDENCY_LINKS=1 + PIP_DEFAULT_TIMEOUT=60 + ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future +basepython = python +envdir = {toxworkdir}/confluent + +[testenv:test] +commands = + py.test --color=no {env:PYTESTARGS:} test + +[testenv:style] +commands = + flake8 --config tox.ini + +[testenv:cover] +commands = + py.test {env:PYTESTARGS:} --cov . --cov-report=xml --cov-report=html --cov-report=term test + +[flake8] +ignore = E111,E121,W292,E123,E226 +max-line-length = 160 + +[pytest] +addopts = -n 1