Skip to content

[WIP] feat: build docker image optimisation for reusing the jar again #569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: staging
Choose a base branch
from
Draft

[WIP] feat: build docker image optimisation for reusing the jar again #569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 19 additions & 24 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,41 +1,36 @@
# Build Stage
FROM golang:1.23-alpine AS base
# Runtime base (shared across all drivers): OS + Java + JAR + destination specs
FROM alpine:3.18 AS runtime-base

WORKDIR /home/app
COPY . .
RUN apk add --no-cache openjdk17

# Copy the pre-built JAR and destination specs (these rarely change)
COPY destination/iceberg/olake-iceberg-java-writer/target/olake-iceberg-java-writer-0.0.1-SNAPSHOT.jar /home/olake-iceberg-java-writer.jar
COPY destination/iceberg/resources/spec.json /destination/iceberg/resources/spec.json
COPY destination/parquet/resources/spec.json /destination/parquet/resources/spec.json

WORKDIR /home

# Build the Go driver binary (driver-specific)
FROM golang:1.23-alpine AS builder

ARG DRIVER_NAME=olake
# Build the Go binary
WORKDIR /home/app
COPY . .
WORKDIR /home/app/drivers/${DRIVER_NAME}
RUN go build -o /olake main.go

# Final Runtime Stage
FROM alpine:3.18

# Install Java 17 instead of Java 11
RUN apk add --no-cache openjdk17

# Copy the binary from the build stage
COPY --from=base /olake /home/olake
# Final image: reuse runtime base, then add only driver-specific layers
FROM runtime-base

ARG DRIVER_VERSION=dev
ARG DRIVER_NAME=olake

# Copy the pre-built JAR file from Maven
# First try to copy from the source location (works after Maven build)
COPY destination/iceberg/olake-iceberg-java-writer/target/olake-iceberg-java-writer-0.0.1-SNAPSHOT.jar /home/olake-iceberg-java-writer.jar

# Copy the spec files for driver and destinations
COPY --from=base /home/app/drivers/${DRIVER_NAME}/resources/spec.json /drivers/${DRIVER_NAME}/resources/spec.json
COPY --from=base /home/app/destination/iceberg/resources/spec.json /destination/iceberg/resources/spec.json
COPY --from=base /home/app/destination/parquet/resources/spec.json /destination/parquet/resources/spec.json
COPY --from=builder /olake /home/olake
COPY drivers/${DRIVER_NAME}/resources/spec.json /drivers/${DRIVER_NAME}/resources/spec.json

# Metadata
LABEL io.eggwhite.version=${DRIVER_VERSION}
LABEL io.eggwhite.name=olake/source-${DRIVER_NAME}

# Set working directory
WORKDIR /home

# Entrypoint
ENTRYPOINT ["./olake"]
37 changes: 33 additions & 4 deletions release-tool.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,44 @@ function chalk() {
# Function to build the Java project with Maven
function build_java_project() {
echo "Building Java project with Maven..."
# Use a fixed timestamp based on the last Java source change for reproducible JARs
local MODULE_DIR="destination/iceberg/olake-iceberg-java-writer"
local MODULE_SRC_DIR="${MODULE_DIR}/src"
export SOURCE_DATE_EPOCH="$(git log -1 --format=%ct -- "${MODULE_SRC_DIR}")"
# Change to the directory containing the POM file
cd destination/iceberg/olake-iceberg-java-writer || fail "Failed to change to Maven project directory"
cd "${MODULE_DIR}" || fail "Failed to change to Maven project directory"
echo "Building Maven project in $(pwd)"
mvn clean package -Dmaven.test.skip=true || fail "Maven build failed"
mvn clean package -Dmaven.test.skip=true -Dproject.build.outputTimestamp="${SOURCE_DATE_EPOCH}" || fail "Maven build failed"
# Record commit that produced this JAR for safe reuse decisions
local CURRENT_MODULE_COMMIT
CURRENT_MODULE_COMMIT="$(git log -1 --format=%H -- "${MODULE_SRC_DIR}")"
echo -n "$CURRENT_MODULE_COMMIT" > target/.built-commit
# Return to the original directory
cd - || fail "Failed to return to original directory"
echo "$(chalk green "✅ Java project successfully built")"
}

# Build the JAR only if a prebuilt artifact isn't already present (enables sharing across jobs)
function ensure_java_jar() {
local MODULE_DIR="destination/iceberg/olake-iceberg-java-writer"
local MODULE_SRC_DIR="${MODULE_DIR}/src"
local jar_path="${MODULE_DIR}/target/olake-iceberg-java-writer-0.0.1-SNAPSHOT.jar"
local built_commit_file="${MODULE_DIR}/target/.built-commit"
local current_module_commit
current_module_commit="$(git log -1 --format=%H -- "${MODULE_SRC_DIR}")"

if [[ -f "$jar_path" && -s "$jar_path" && -f "$built_commit_file" ]]; then
local built_commit
built_commit="$(cat "$built_commit_file" 2>/dev/null || echo)"
if [[ "$built_commit" == "$current_module_commit" ]]; then
echo "Using prebuilt JAR: $jar_path (commit $built_commit)"
return 0
fi
echo "Java sources changed since last build ($built_commit -> $current_module_commit); rebuilding JAR"
fi
build_java_project
}

# Function to fail with a message
function fail() {
local error="${1:-Unknown error}"
Expand Down Expand Up @@ -138,7 +167,7 @@ connector=$DRIVER
type="source"


# Build Java project
build_java_project
# Build Java project (or reuse prebuilt artifact if present)
ensure_java_jar

release "$VERSION" "$platform" "$CURRENT_BRANCH"
Loading