Skip to content

Commit ab059c4

Browse files
okuminokumin
committed
HIVE-29260: Add smoke tests for Docker images
1 parent f6d0b0d commit ab059c4

File tree

19 files changed

+984
-3
lines changed

19 files changed

+984
-3
lines changed

‎.github/workflows/docker-GA-images.yml‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,61 @@ jobs:
112112
- name: Set up Docker Buildx
113113
uses: docker/setup-buildx-action@v2
114114

115+
- name: Build Hive Image locally
116+
uses: docker/build-push-action@v4
117+
with:
118+
context: ./packaging/src/docker/
119+
file: ./packaging/src/docker/Dockerfile
120+
push: false
121+
load: true
122+
tags: hive:test
123+
build-args:
124+
|
125+
HIVE_VERSION=${{ env.HIVE_VERSION }}
126+
HADOOP_VERSION=${{ env.HADOOP_VERSION }}
127+
TEZ_VERSION=${{ env.TEZ_VERSION }}
128+
BUILD_ENV=${{ env.BUILD_ENV }}
129+
130+
- name: Build Standalone Metastore Image locally
131+
uses: docker/build-push-action@v4
132+
with:
133+
context: ./standalone-metastore/packaging/src/docker/
134+
file: ./standalone-metastore/packaging/src/docker/Dockerfile
135+
push: false
136+
load: true
137+
tags: hive:standalone-metastore-test
138+
build-args:
139+
|
140+
HIVE_VERSION=${{ env.HIVE_VERSION }}
141+
HADOOP_VERSION=${{ env.HADOOP_VERSION }}
142+
BUILD_ENV=${{ env.BUILD_ENV }}
143+
144+
- name: Create k8s cluster
145+
uses: helm/kind-action@v1
146+
147+
- name: Set up Helm
148+
uses: azure/setup-helm@v4
149+
150+
- name: Load images
151+
run: kind load docker-image hive:test hive:standalone-metastore-test --name chart-testing
152+
153+
- name: Deploy Hive
154+
run: |
155+
./itests/test-docker/scripts/setup.sh
156+
157+
- name: Run JUnit
158+
run: |
159+
mvn --batch-mode test -Pitests -pl itests/test-docker
160+
161+
- name: Test HiveServer2
162+
run: |
163+
./itests/test-docker/scripts/test_hive_server2.sh
164+
165+
- name: Clean up k8s
166+
run: |
167+
./itests/test-docker/scripts/teardown.sh
168+
kind delete cluster --name chart-testing
169+
115170
- name: Build and push Hive Image to docker hub
116171
uses: docker/build-push-action@v4
117172
with:
@@ -141,8 +196,16 @@ jobs:
141196
HADOOP_VERSION=${{ env.HADOOP_VERSION }}
142197
BUILD_ENV=${{ env.BUILD_ENV }}
143198

144-
- name: Dump disk usage on failure
145-
if: failure()
199+
- name: Dump resource space at the end
200+
run: |
201+
df -h
202+
free -h
203+
204+
- name: Dump resource usage on failure
205+
if: failure() || cancelled()
146206
run: |
147207
df -h
148208
du -xh / --max-depth=3 2>/dev/null | sort -h | tail -50
209+
free -h
210+
ps aux --sort -rss | head -n 30
211+
kubectl get pods

‎itests/hive-iceberg/pom.xml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
<hive.path.to.root>../..</hive.path.to.root>
2828
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
2929
<log4j2.debug>false</log4j2.debug>
30-
<iceberg.version>1.9.1</iceberg.version>
3130
</properties>
3231
<dependencies>
3332
<dependency>

‎itests/pom.xml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
<properties>
2727
<hive.path.to.root>..</hive.path.to.root>
2828
<wiremock-jre8.version>2.32.0</wiremock-jre8.version>
29+
<iceberg.version>1.9.1</iceberg.version>
30+
<assertj.version>3.27.3</assertj.version>
2931
</properties>
3032
<modules>
3133
<module>custom-serde</module>
@@ -44,6 +46,7 @@
4446
<module>qtest-druid</module>
4547
<module>qtest-kudu</module>
4648
<module>qtest-iceberg</module>
49+
<module>test-docker</module>
4750
<module>hive-iceberg</module>
4851
</modules>
4952
<dependencyManagement>
@@ -69,6 +72,11 @@
6972
<artifactId>hive-metastore</artifactId>
7073
<version>${project.version}</version>
7174
</dependency>
75+
<dependency>
76+
<groupId>org.apache.hive</groupId>
77+
<artifactId>hive-standalone-metastore-client</artifactId>
78+
<version>${standalone-metastore.version}</version>
79+
</dependency>
7280
<dependency>
7381
<groupId>org.apache.hive</groupId>
7482
<artifactId>hive-standalone-metastore-common</artifactId>
@@ -497,6 +505,12 @@
497505
<artifactId>hive-iceberg-handler</artifactId>
498506
<version>${project.version}</version>
499507
</dependency>
508+
<dependency>
509+
<groupId>org.assertj</groupId>
510+
<artifactId>assertj-core</artifactId>
511+
<version>${assertj.version}</version>
512+
<scope>test</scope>
513+
</dependency>
500514
</dependencies>
501515
</dependencyManagement>
502516
<build>

‎itests/test-docker/helm/ozone/values.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
datanode:
18+
replicas: 1
19+
env:
20+
# GitHub Actions provide tiny spaces
21+
- name: OZONE-SITE.XML_hdds.datanode.volume.min.free.space
22+
value: "256MB"
23+
- name: OZONE-SITE.XML_hdds.scm.safemode.enabled
24+
value: "false"
25+
- name: OZONE-SITE.XML_ozone.scm.container.size
26+
value: 128MB
27+
- name: OZONE-SITE.XML_ozone.scm.block.size
28+
value: 32MB
29+
- name: OZONE-SITE.XML_ozone.server.default.replication
30+
value: "1"

‎itests/test-docker/k8s/beeline/deployment.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
apiVersion: apps/v1
18+
kind: Deployment
19+
metadata:
20+
labels:
21+
app: beeline
22+
name: beeline
23+
spec:
24+
replicas: 1
25+
selector:
26+
matchLabels:
27+
app: beeline
28+
template:
29+
metadata:
30+
labels:
31+
app: beeline
32+
spec:
33+
containers:
34+
- name: loop
35+
image: hive:test
36+
command:
37+
- timeout
38+
args:
39+
- infinity
40+
- sleep
41+
- infinity
42+
env:
43+
- name: TERM
44+
value: dumb

‎itests/test-docker/k8s/hadoop/configmap.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
apiVersion: v1
18+
kind: ConfigMap
19+
metadata:
20+
name: hadoop-config
21+
data:
22+
core-site.xml: |
23+
<configuration>
24+
<property>
25+
<name>fs.s3a.endpoint</name>
26+
<value>http://ozone-s3g-rest:9878</value>
27+
</property>
28+
<property>
29+
<name>fs.s3a.path.style.access</name>
30+
<value>true</value>
31+
</property>
32+
<property>
33+
<name>fs.s3a.change.detection.version.required</name>
34+
<value>false</value>
35+
</property>
36+
<property>
37+
<name>fs.s3a.change.detection.mode</name>
38+
<value>none</value>
39+
</property>
40+
<property>
41+
<name>fs.s3a.access.key</name>
42+
<value>hadoop</value>
43+
</property>
44+
<property>
45+
<name>fs.s3a.secret.key</name>
46+
<value>dummy</value>
47+
</property>
48+
</configuration>

‎itests/test-docker/k8s/hive-metastore/configmap.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
apiVersion: v1
18+
kind: ConfigMap
19+
metadata:
20+
name: hive-metastore-config
21+
data:
22+
metastore-site.xml: |
23+
<configuration>
24+
<property>
25+
<name>metastore.warehouse.dir</name>
26+
<value>s3a://test/test-warehouse</value>
27+
</property>
28+
<property>
29+
<name>hive.metastore.warehouse.external.dir</name>
30+
<value>s3a://test/test-warehouse</value>
31+
</property>
32+
<property>
33+
<name>metastore.event.db.notification.api.auth</name>
34+
<value>false</value>
35+
</property>
36+
<property>
37+
<name>metastore.catalog.servlet.port</name>
38+
<value>9001</value>
39+
</property>
40+
<property>
41+
<name>metastore.catalog.servlet.auth</name>
42+
<value>none</value>
43+
</property>
44+
</configuration>

‎itests/test-docker/k8s/hive-metastore/deployment.yaml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
apiVersion: apps/v1
18+
kind: Deployment
19+
metadata:
20+
labels:
21+
app: hive-metastore
22+
name: hive-metastore
23+
spec:
24+
replicas: 1
25+
selector:
26+
matchLabels:
27+
app: hive-metastore
28+
template:
29+
metadata:
30+
labels:
31+
app: hive-metastore
32+
spec:
33+
initContainers:
34+
- name: download-aws-sdk
35+
image: ubuntu:26.04
36+
command:
37+
- /bin/bash
38+
args:
39+
- -c
40+
- |
41+
apt-get update
42+
apt-get install -y --no-install-recommends ca-certificates wget
43+
wget https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.26.19/bundle-2.26.19.jar -P /tmp/ext-jars
44+
volumeMounts:
45+
- name: ext-jars
46+
mountPath: /tmp/ext-jars
47+
containers:
48+
- name: hive-metastore
49+
image: hive:standalone-metastore-test
50+
env:
51+
- name: HADOOP_OPTIONAL_TOOLS
52+
value: hadoop-aws
53+
- name: HIVE_CUSTOM_CONF_DIR
54+
value: /etc/hive/conf
55+
readinessProbe:
56+
tcpSocket:
57+
port: 9083
58+
volumeMounts:
59+
- name: ext-jars
60+
mountPath: /tmp/ext-jars
61+
- name: hadoop-config
62+
mountPath: /etc/hive/conf/core-site.xml
63+
subPath: core-site.xml
64+
- name: hive-metastore-config
65+
mountPath: /etc/hive/conf/metastore-site.xml
66+
subPath: metastore-site.xml
67+
volumes:
68+
- name: ext-jars
69+
emptyDir: {}
70+
- name: hadoop-config
71+
configMap:
72+
name: hadoop-config
73+
- name: hive-metastore-config
74+
configMap:
75+
name: hive-metastore-config

0 commit comments

Comments
 (0)