Skip to content

Commit ed29a6a

Browse files
ajfabbriajfabbri
authored
HADOOP-19877: run s3a integration and scale tests in CI (#8467)
- create composite actions for creating builder image - Trigger when haddop-aws code or related workflows change - Specify localstack container in workflow `services:` instead of using the 3rd-party setup-localstack action; it had more issues than it was worth. - Separate container OS vs. runner OS params - Pass localstack license (dontated) CI key secret through to workflow - Faster build; skip javadoc - Add s3a test excludes - Don't use path-style access for s3 - Sanity check local maven repo path - Fix cache path issue: container vs host path - Try to skip waiting for localstack readiness - Actions cache is immutable, use accordingly. - Consolodate common MAVEN_OPTS - Address broken mvn verify return status.. - Update test-reporter action and filter artifact download - Rename test exclude file, localstack-specific.. - Update aws integration test reporting workflow - Consistent workflow name report_foo.yml - Fix test report to avoid parsing failsafe summary - Mind cancellations, tweak report workflow - Use commit status to improve report usability - Naming tweaks: concise and consistent - Rename input to `toolchain_branch` for clarity - Remove unnecessary `ports:` section for localstack service. - Tweak s3a report name / context * ci: add missing apache headers, fix typo in test excludes * ci: pin localstack version * ci/aws: remove checks: write permission
1 parent 19c6bcf commit ed29a6a

8 files changed

Lines changed: 629 additions & 0 deletions

File tree

‎.github/actions/build_image/action.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
name: Create hadoop build image
19+
20+
inputs:
21+
branch:
22+
description: Git branch to use.
23+
required: true
24+
os:
25+
description: Runner OS name
26+
required: true
27+
build_image_url:
28+
description: URL of build (infra) image.
29+
required: true
30+
31+
outputs:
32+
uid:
33+
description: User ID this action ran as.
34+
value: ${{ steps.variables.outputs.uid }}
35+
36+
runs:
37+
using: composite
38+
steps:
39+
- name: Login to GitHub Container Registry
40+
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
41+
with:
42+
registry: ghcr.io
43+
username: ${{ github.actor }}
44+
password: ${{ github.token }}
45+
- name: Set up Docker Buildx
46+
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
47+
- name: Build and push ${{ inputs.os }} base build image for ${{ inputs.branch }}
48+
id: docker_build_base
49+
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
50+
with:
51+
context: ./dev-support/docker/
52+
file: ./dev-support/docker/Dockerfile_${{ inputs.os }}
53+
push: true
54+
tags: ${{ inputs.build_image_url }}-base
55+
cache-from: type=registry,ref=ghcr.io/apache/hadoop/gha-build-${{ inputs.os }}-image-cache:${{ inputs.branch }}
56+
- name: User-specific Dockerfile
57+
shell: bash
58+
run: |
59+
USER_ID=$(id -u "${USER}")
60+
GROUP_ID=$(id -g "${USER}")
61+
cat > /tmp/Dockerfile.gha <<UserSpecificDocker
62+
FROM ${{ inputs.build_image_url }}-base
63+
RUN rm -f /var/log/faillog /var/log/lastlog
64+
RUN groupadd --non-unique -g ${GROUP_ID} ${USER}
65+
RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER}
66+
RUN echo "${USER} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}"
67+
UserSpecificDocker
68+
- name: Build and push ${{ inputs.os }} build image for ${{ inputs.branch }}
69+
id: docker_build_gha
70+
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
71+
with:
72+
context: ./dev-support/docker/
73+
file: /tmp/Dockerfile.gha
74+
push: true
75+
tags: ${{ inputs.build_image_url }}
76+
- name: Set up Outputs
77+
id: variables
78+
shell: bash
79+
run: |
80+
echo "uid=$(id -u "${USER}")" >> $GITHUB_OUTPUT

‎.github/actions/build_image_url/action.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
name: Get Build Image URL
18+
description:
19+
20+
inputs:
21+
branch:
22+
description: Git branch to use.
23+
required: true
24+
os:
25+
description: Operating system to run the build on
26+
required: true
27+
28+
outputs:
29+
build_image_url:
30+
description: URL of build (infra) image.
31+
value: ${{ steps.compute.outputs.image_url }}
32+
33+
runs:
34+
using: composite
35+
steps:
36+
- id: compute
37+
shell: bash
38+
run: |
39+
# Convert to lowercase to meet Docker repo name requirement
40+
REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
41+
OS=$(echo "${{ inputs.os }}" | tr '[:upper:]' '[:lower:]')
42+
echo "image_url=ghcr.io/${REPO_OWNER}/gha-build-${OS}:${{ inputs.branch }}-${{ github.run_id }}" \
43+
>> $GITHUB_OUTPUT
44+

‎.github/gha-tests/hadoop-aws-localstack-excludes.txt‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
# Initial stab at excluding tests that won't run in our
21+
# CI container environment with Localstack mocks of AWS services.
22+
23+
# TODO see if we can enable any of these...
24+
25+
# tests that depend on public S3 buckets
26+
**/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
27+
**/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java
28+
**/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
29+
**/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java
30+
**/ITestS3AAnalyticsAcceleratorStreamReading.java
31+
**/ITestS3AEndpointRegion.java
32+
33+
34+
# Tests requiring IAM roles / STS
35+
# We should be able to re-enable some of these. See:
36+
# https://docs.localstack.cloud/aws/services/sts/
37+
**/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
38+
**/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
39+
**/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java
40+
**/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
41+
**/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
42+
43+
44+
# failures that need to be investigated
45+
46+
# Two methods fail: 1. testUpdateDeepDirectoryStructureNoChange():
47+
# AssertionFailedError: Files Skipped value 0 too below minimum 1 ==>
48+
# expected: <true> but was: <false>
49+
# 2. testUpdateDeepDirectoryStructureToRemote():
50+
# AssertionFailedError: Files Copied value 2 above maximum 1 ==> expected: <true> but was: <false>
51+
**/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
52+
53+
# A number of failures with vectored read tests
54+
**/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java
55+
56+
# Access key errors:
57+
# (test case)->AbstractS3ATestBase.setup:111->AbstractFSContractTestBase.setup:197->AbstractFSContractTestBase.mkdirs:355
58+
# » AccessDenied s3a://hadoop-ci/job-00/test: getFileStatus on
59+
# s3a://hadoop-ci/job-00/test:
60+
# software.amazon.awssdk.services.s3.model.S3Exception: The AWS Access Key Id you
61+
# provided does not exist in our records. (Service: S3, Status Code: 403
62+
**/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
63+
**/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
64+
65+
# Localstack issue (guessing lack of persistence of upload parts across sessions)
66+
**/org/apache/hadoop/fs/s3a/commit/ITestUploadRecovery.java
67+
**/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
68+
69+
# Error: ITestConnectionTimeouts.testObjectUploadTimeouts:258 Expected a
70+
# java.lang.Exception to be thrown, but got the result: : "01234567890123456789..."
71+
**/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java
72+
73+
# ITestS3AAWSCredentialsProvider.testBadCredentials:183->lambda$testBadCredentials$0:184
74+
# ->createFailingFS:160 Expected exception - got S3AFileSystem{
75+
**/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
76+
77+
# testSeeksWithLruEviction java.util.concurrent.TimeoutException: timed out
78+
# after 180 seconds
79+
**/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java

‎.github/workflows/cloud_aws.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: "Cloud-AWS"
21+
22+
on:
23+
pull_request:
24+
paths:
25+
- 'hadoop-tools/hadoop-aws/**'
26+
- '.github/workflows/*cloud_aws.yml'
27+
- '.github/actions/build_image**'
28+
- '.github/gha-tests/hadoop-aws*excludes.txt'
29+
30+
jobs:
31+
run-aws-integration:
32+
# Security: write privileges are needed to update PR status and upload test results.
33+
# Package write is for building toolchain container images on demand, but ghcr.io access is
34+
# scoped to the repository the actions run on.
35+
permissions:
36+
packages: write
37+
contents: read
38+
name: Run
39+
uses: ./.github/workflows/tmpl_cloud_aws.yml
40+
secrets: inherit # inherit LOCALSTACK_CI_KEY
41+
with:
42+
java: 25
43+
os: ubuntu_24
44+
runner_os: ubuntu-24.04

‎.github/workflows/report_cloud_aws.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
name: Cloud-AWS Test Summary
19+
20+
on:
21+
workflow_run:
22+
workflows: ["Cloud-AWS"]
23+
types:
24+
- completed
25+
26+
permissions:
27+
contents: read
28+
actions: read
29+
checks: write
30+
statuses: write
31+
32+
jobs:
33+
report:
34+
runs-on: ubuntu-latest
35+
if: github.event.workflow_run.event == 'pull_request'
36+
steps:
37+
- name: S3A Test Report
38+
id: report
39+
uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
40+
with:
41+
artifact: /s3a-test-reports-java-.*/ # see tmpl_cloud_aws.yml
42+
name: S3A Integration Tests
43+
path: "**/TEST-*.xml"
44+
reporter: java-junit
45+
only-summary: 'false'
46+
# use-actions-summary is currently mutually exclusive with check run
47+
# creation in dorny/test-reporter.:
48+
# For now, try check run method.
49+
use-actions-summary: 'false'
50+
list-suites: 'failed'
51+
list-tests: 'failed'
52+
# list-files: 'none'
53+
fail-on-error: 'false'
54+
55+
- name: Post S3A commit status
56+
uses: actions/github-script@v7
57+
if: always()
58+
with:
59+
script: |
60+
const conclusion = '${{ steps.report.outputs.conclusion }}';
61+
const passed = '${{ steps.report.outputs.passed }}';
62+
const failed = '${{ steps.report.outputs.failed }}';
63+
const skipped = '${{ steps.report.outputs.skipped }}';
64+
const url = '${{ steps.report.outputs.url_html }}';
65+
66+
// Map dorny/test-reporter conclusion to a GitHub commit-status state.
67+
let state, description;
68+
if (conclusion === 'failure') {
69+
state = 'failure';
70+
description = `${passed} pass, ${failed} fail, ${skipped} skip`;
71+
} else if (conclusion === 'success') {
72+
state = 'success';
73+
description = `${passed} pass, ${failed} fail, ${skipped} skip`;
74+
} else {
75+
state = 'error';
76+
description = 'Test report unavailable (no results found or reporter error)';
77+
}
78+
79+
const params = {
80+
...context.repo,
81+
sha: context.payload.workflow_run.head_sha,
82+
state,
83+
description,
84+
context: 'S3A Test Summary',
85+
};
86+
if (url) params.target_url = url;
87+
88+
await github.rest.repos.createCommitStatus(params);

0 commit comments

Comments
 (0)