Skip to content

Commit 8bd4e2b

Browse files
authored
[VL][CI] Adding docker image for maven cache (#11655)
This patch added a docker image with maven cache for spark package/test, covering 3.3/3.4/3.5/4.0/4.1. A second patch will be made to enable this cache in the CI jobs
1 parent 985b19a commit 8bd4e2b

File tree

2 files changed

+100
-0
lines changed

2 files changed

+100
-0
lines changed

.github/workflows/docker_image.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ on:
2424
- '.github/workflows/util/install-resources.sh'
2525
- 'dev/docker/*'
2626
- 'dev/docker/cudf/*'
27+
- 'dev/docker/ubuntu/*'
2728
schedule:
2829
- cron: '0 20 * * 0'
2930

@@ -112,6 +113,31 @@ jobs:
112113
push: true
113114
tags: ${{ env.DOCKERHUB_REPO }}:centos-9-jdk8-cudf
114115

116+
build-ubuntu22-m2-cache:
117+
if: ${{ startsWith(github.repository, 'apache/') }}
118+
runs-on: ubuntu-22.04
119+
120+
steps:
121+
- name: Checkout repository
122+
uses: actions/checkout@v4
123+
124+
- name: Set up Docker Buildx
125+
uses: docker/setup-buildx-action@v3
126+
127+
- name: Login to Docker Hub
128+
uses: docker/login-action@v2
129+
with:
130+
username: ${{ secrets.DOCKERHUB_USER }}
131+
password: ${{ secrets.DOCKERHUB_TOKEN }}
132+
133+
- name: Build and push Docker image
134+
uses: docker/build-push-action@v6
135+
with:
136+
context: .
137+
file: dev/docker/ubuntu/Dockerfile.ubuntu22-m2-cache
138+
push: true
139+
tags: ${{ env.DOCKERHUB_REPO }}:ubuntu22-m2-cache
140+
115141
build-centos-9-jdk17-cuda12_9-cudf:
116142
if: ${{ startsWith(github.repository, 'apache/') }}
117143
runs-on: ubuntu-22.04
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
FROM ubuntu:22.04
17+
18+
ENV DEBIAN_FRONTEND=noninteractive
19+
ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-arm64
20+
ENV PATH=$JAVA_HOME/bin:$PATH
21+
22+
RUN apt-get update && apt-get install -y \
23+
sudo git build-essential cmake ninja-build \
24+
python3 curl ccache wget unzip \
25+
openjdk-17-jdk maven \
26+
&& rm -rf /var/lib/apt/lists/*
27+
28+
RUN git clone --depth=1 https://github.com/apache/incubator-gluten.git /opt/gluten
29+
30+
WORKDIR /opt/gluten
31+
32+
# Build Velox dependencies with Arrow enabled
33+
RUN bash ./dev/builddeps-veloxbe.sh \
34+
--run_setup_script=ON \
35+
build_arrow
36+
37+
# Download Maven dependencies for common Spark versions
38+
# This pre-populates the Maven cache with all necessary dependencies
39+
RUN set -ex; \
40+
export MVN_CMD="build/mvn -ntp"; \
41+
42+
# Spark 4.1
43+
$MVN_CMD -Pjava-17,scala-2.13,spark-4.1,backends-velox,hadoop-3.3,spark-ut,delta \
44+
dependency:go-offline -DskipTests || true; \
45+
46+
# Spark 4.0
47+
$MVN_CMD -Pjava-17,scala-2.13,spark-4,backends-velox,hadoop-3.3,spark-ut \
48+
-Piceberg,iceberg-test,delta,paimon \
49+
dependency:go-offline -DskipTests || true; \
50+
51+
# Spark 3.5
52+
$MVN_CMD -Pjava-17,spark-3.5,backends-velox,hadoop-3.3,spark-ut \
53+
-Piceberg,iceberg-test,delta,paimon \
54+
dependency:go-offline -DskipTests || true; \
55+
56+
# Spark 3.4
57+
$MVN_CMD -Pjava-17,spark-3.4,backends-velox,hadoop-3.3,spark-ut \
58+
-Piceberg,iceberg-test,delta,paimon \
59+
dependency:go-offline -DskipTests || true; \
60+
61+
# Spark 3.3
62+
$MVN_CMD -Pjava-17,spark-3.3,backends-velox,hadoop-3.3,spark-ut \
63+
-Piceberg,delta,paimon \
64+
dependency:go-offline -DskipTests || true
65+
66+
WORKDIR /work
67+
68+
# Clean up build artifacts but keep m2
69+
RUN rm -rf /opt/gluten/cpp/build \
70+
&& rm -rf /opt/gluten/ep/build-velox/build \
71+
&& rm -rf /root/.cache/ccache
72+
73+
# Default command
74+
CMD ["/bin/bash"]

0 commit comments

Comments
 (0)