Skip to content
This repository was archived by the owner on Nov 20, 2024. It is now read-only.

Commit 0037a72

Browse files
anurag4DSBanurag4DSB
authored
Merge pull request #11 from scality/feature/COSI-14-add-iam-s3-logging-and-cleanup-in-CI
COSI-14: E2E test setup for S3 and IAM
2 parents 46dfc80 + bdcec3e commit 0037a72

File tree

6 files changed

+251
-1
lines changed

6 files changed

+251
-1
lines changed

‎.github/s3_and_iam_deployment/.env‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
VAULT_IMAGE="ghcr.io/scality/vault:7.70.26"
2+
CLOUDSERVER_IMAGE="ghcr.io/scality/cloudserver:7.70.55"

‎.github/s3_and_iam_deployment/cloudserver-config.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"port": 8000,
3+
"listenOn": [],
4+
"restEndpoints": {
5+
"localhost": "us-east-1",
6+
"127.0.0.1": "us-east-1",
7+
"cloudserver-front": "us-east-1",
8+
"s3.docker.test": "us-east-1",
9+
"127.0.0.2": "us-east-1",
10+
"s3.amazonaws.com": "us-east-1"
11+
},
12+
"websiteEndpoints": [
13+
"s3-website-us-east-1.amazonaws.com",
14+
"s3-website.us-east-2.amazonaws.com",
15+
"s3-website-us-west-1.amazonaws.com",
16+
"s3-website-us-west-2.amazonaws.com",
17+
"s3-website.ap-south-1.amazonaws.com",
18+
"s3-website.ap-northeast-2.amazonaws.com",
19+
"s3-website-ap-southeast-1.amazonaws.com",
20+
"s3-website-ap-southeast-2.amazonaws.com",
21+
"s3-website-ap-northeast-1.amazonaws.com",
22+
"s3-website.eu-central-1.amazonaws.com",
23+
"s3-website-eu-west-1.amazonaws.com",
24+
"s3-website-sa-east-1.amazonaws.com",
25+
"s3-website.localhost",
26+
"s3-website.scality.test"
27+
],
28+
"vaultd": {
29+
"host": "localhost",
30+
"port": 8500
31+
},
32+
"clusters": 1,
33+
"log": {
34+
"logLevel": "trace",
35+
"dumpLevel": "error"
36+
},
37+
"healthChecks": {
38+
"allowFrom": ["127.0.0.1/8", "::1"]
39+
},
40+
"recordLog": {
41+
"enabled": false,
42+
"recordLogName": "s3-recordlog"
43+
},
44+
"requests": {
45+
"viaProxy": false,
46+
"trustedProxyCIDRs": [],
47+
"extractClientIPFromHeader": ""
48+
}
49+
}

‎.github/s3_and_iam_deployment/docker-compose.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
services:
2+
s3:
3+
profiles: ['iam_s3']
4+
image: ${CLOUDSERVER_IMAGE}
5+
network_mode: host
6+
environment:
7+
S3VAULT: scality
8+
S3_CONFIG_FILE: /conf/config.json
9+
command: /bin/sh -c "yarn run mem_backend > /logs/s3/s3.log 2>&1"
10+
volumes:
11+
- ./cloudserver-config.json:/conf/config.json:ro
12+
- ./logs/s3:/logs/s3
13+
14+
iam:
15+
profiles: ['iam_s3']
16+
image: ${VAULT_IMAGE}
17+
network_mode: host
18+
command: /bin/sh -c "chmod 400 tests/utils/keyfile && yarn start > /logs/iam/iam.log 2>&1"
19+
environment:
20+
VAULT_CONFIG_FILE: /conf/config.json
21+
VAULT_DB_BACKEND: LEVELDB
22+
volumes:
23+
- ./vault-config.json:/conf/config.json:ro
24+
- ./data/vaultdb:/data
25+
- ./logs/iam:/logs/iam

‎.github/s3_and_iam_deployment/vault-config.json‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{
2+
"clusters": 1,
3+
"healthChecks": {
4+
"allowFrom": ["127.0.0.1/8", "::1"]
5+
},
6+
"interfaces": {
7+
"S3": {
8+
"address": "0.0.0.0",
9+
"port": 8500,
10+
"allowFrom": ["0.0.0.0/8", "::1"]
11+
},
12+
"administration": {
13+
"address": "0.0.0.0",
14+
"port": 8600
15+
},
16+
"sts": {
17+
"address": "127.0.0.1",
18+
"port": 8800
19+
},
20+
"sso": {
21+
"address": "127.0.0.1",
22+
"port": 8700,
23+
"cert": "./tests/utils/sso/defaultCert.crt",
24+
"key": "./tests/utils/sso/defaultCert.key"
25+
}
26+
},
27+
"sso": {
28+
"clients": [
29+
{
30+
"id": "grafana",
31+
"secret": "123123",
32+
"redirectUri": [
33+
"http://localhost:3000/login/generic_oauth",
34+
"https://localhost:3000/login/generic_oauth"
35+
],
36+
"scopes": ["dev", "prod"]
37+
}
38+
],
39+
"publicClients": [
40+
{
41+
"id": "identisee",
42+
"redirectUri": "https://127.0.0.1:8700/user/info",
43+
"scopes": ["admin", "superadmin"]
44+
}
45+
],
46+
"authCodeTTL": 480,
47+
"accessTokenTTL": 7600
48+
},
49+
"map": ["127.0.0.1:4300", "127.0.0.2:4301", "127.0.0.3:4302", "127.0.0.4:4303", "127.0.0.5:4304"],
50+
"keyFilePath": "./tests/utils/keyfile",
51+
"adminCredentialsFilePath": "./tests/utils/admincredentials.json.encrypted",
52+
"log": {
53+
"level": "trace",
54+
"dump": "error"
55+
},
56+
"accountSeeds": [
57+
{
58+
"role": {
59+
"roleName": "scality-role1",
60+
"trustPolicy": {
61+
"Version": "2012-10-17",
62+
"Statement": [
63+
{
64+
"Effect": "Allow",
65+
"Principal": { "AWS": "arn:aws:iam::000000000000:user/root" },
66+
"Action": "sts:AssumeRole",
67+
"Condition": {}
68+
}
69+
]
70+
}
71+
},
72+
"permissionPolicy": {
73+
"policyName": "scality-policy1",
74+
"policyDocument": {
75+
"Version": "2012-10-17",
76+
"Statement": [
77+
{
78+
"Sid": "FullAccess",
79+
"Effect": "Allow",
80+
"Action": ["s3:*"],
81+
"Resource": ["*"]
82+
}
83+
]
84+
}
85+
}
86+
}
87+
],
88+
"utapi": {
89+
"host": "127.0.0.1",
90+
"port": 8100
91+
}
92+
}

‎.github/scripts/wait_for_local_port.bash‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
wait_for_local_port() {
3+
local port=$1
4+
local timeout=$2
5+
local count=0
6+
local ret=1
7+
echo "waiting for storage-service:$port"
8+
while [[ "$ret" -eq "1" && "$count" -lt "$timeout" ]] ; do
9+
nc -z -w 1 localhost $port
10+
ret=$?
11+
if [ ! "$ret" -eq "0" ]; then
12+
echo -n .
13+
sleep 1
14+
count=$(($count+1))
15+
fi
16+
done
17+
18+
echo ""
19+
20+
if [[ "$count" -eq "$timeout" ]]; then
21+
echo "Server did not start in less than $timeout seconds. Exiting..."
22+
exit 1
23+
fi
24+
25+
echo "Server got ready in ~${count} seconds. Starting test now..."
26+
}
27+
28+
wait_for_local_port $1 $2

‎.github/workflows/ci-e2e-tests.yml‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,67 @@ jobs:
3737
with:
3838
detached: true
3939

40-
4140
- name: Setup COSI Controller, CRDs and Driver
4241
run: |
4342
pwd
4443
chmod +x .github/scripts/setup_cosi_resources.sh
4544
.github/scripts/setup_cosi_resources.sh
4645
46+
- name: Login to Registry
47+
uses: docker/login-action@v3
48+
with:
49+
registry: ghcr.io
50+
username: "${{ github.repository_owner }}"
51+
password: "${{ github.token }}"
52+
53+
- name: Restore Cached Docker Images
54+
id: cache_docker_images
55+
uses: actions/cache@v4
56+
with:
57+
path: /tmp/.docker_cache
58+
key: docker-${{ runner.os }}-${{ hashFiles('.github/s3_and_iam_deployment/.env') }}
59+
restore-keys: |
60+
docker-${{ runner.os }}-
61+
62+
- name: Load Cached Images
63+
run: |
64+
for image in /tmp/.docker_cache/*.tar; do
65+
docker load -i "$image" || true
66+
done
67+
continue-on-error: true
68+
69+
- name: Setup IAM and S3 Services
70+
run: |-
71+
set -e -o pipefail;
72+
mkdir -p logs/s3 logs/iam logs/cosi_driver data/vaultdb && chown -R runner:docker logs data && chmod -R ugo+rwx logs data
73+
docker compose --profile iam_s3 up -d --quiet-pull
74+
bash ../scripts/wait_for_local_port.bash 8600 30
75+
bash ../scripts/wait_for_local_port.bash 8000 30
76+
working-directory: .github/s3_and_iam_deployment
77+
78+
- name: Save Images to Cache if not present
79+
if: steps.cache_docker_images.outputs.cache-hit != 'true'
80+
run: |
81+
source .github/s3_and_iam_deployment/.env
82+
echo "Vault Image: $VAULT_IMAGE"
83+
echo "CloudServer Image: $CLOUDSERVER_IMAGE"
84+
mkdir -p /tmp/.docker_cache
85+
docker save "$VAULT_IMAGE" -o /tmp/.docker_cache/vault_image.tar
86+
docker save "$CLOUDSERVER_IMAGE" -o /tmp/.docker_cache/cloudserver_image.tar
87+
shell: bash
88+
89+
- name: Cleaup IAM and S3 Services
90+
run: docker compose --profile iam_s3 down
91+
working-directory: .github/s3_and_iam_deployment
92+
93+
- name: Move S3 and IAM logs and data to artifacts directory
94+
run: |-
95+
set -e -o pipefail;
96+
mkdir -p .github/e2e_tests/artifacts/logs .github/e2e_tests/artifacts/data
97+
cp -r .github/s3_and_iam_deployment/logs/* .github/e2e_tests/artifacts/logs/
98+
cp -r .github/s3_and_iam_deployment/data/* .github/e2e_tests/artifacts/data/
99+
if: always()
100+
47101
- name: Capture Kubernetes Logs in artifacts directory
48102
run: |
49103
chmod +x .github/scripts/capture_k8s_logs.sh

0 commit comments

Comments
 (0)