Skip to content

Commit 30daef1

Browse files
anurag4DSBanurag4DSB
authored
Merge pull request #120 from scality/bugfix/COSI-79-fix-role-permissions
COSI-79: Allow end-users to create buckets and access from any namespace
2 parents 1ba6548 + 908c8ce commit 30daef1

File tree

14 files changed

+199
-56
lines changed

14 files changed

+199
-56
lines changed

‎.github/scripts/capture_k8s_logs.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -e
44
# Create a directory to store the logs
55
mkdir -p logs/kind_cluster_logs
66
LOG_FILE_PATH=".github/e2e_tests/artifacts/logs/kind_cluster_logs"
7-
mkdir -p "$(dirname "$LOG_FILE_PATH")" # Ensure the log directory exists
7+
mkdir -p "$LOG_FILE_PATH" # Ensure the log directory exists
88
# Define namespaces to capture logs from
99
namespaces=("default" "container-object-storage-system")
1010

‎.github/scripts/e2e_tests_brownfield_use_case.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ SECRET_NAME="brownfield-bucket-secret"
99
IAM_ENDPOINT="http://$HOST_IP:8600"
1010
S3_ENDPOINT="http://$HOST_IP:8000"
1111
BUCKET_NAME="brownfield-bucket"
12-
NAMESPACE="container-object-storage-system"
12+
NAMESPACE="default"
1313
REGION="us-west-1"
1414

1515
# Error handling function

‎.github/workflows/helm-validation.yml‎

Lines changed: 102 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,29 @@ on:
1818
default: 5
1919

2020
jobs:
21-
smoke-test-installation-with-helm:
21+
e2e-tests-with-helm:
2222
runs-on: ubuntu-latest
2323

2424
steps:
2525
- name: Check out repository
2626
uses: actions/checkout@v4
2727

28+
- name: Login to Registry
29+
uses: docker/login-action@v3
30+
with:
31+
registry: ghcr.io
32+
username: "${{ github.repository_owner }}"
33+
password: "${{ github.token }}"
34+
35+
- name: Restore Cached Docker Images
36+
id: cache_docker_images
37+
uses: actions/cache@v4
38+
with:
39+
path: /tmp/.docker_cache
40+
key: docker-${{ runner.os }}-${{ hashFiles('.github/s3_and_iam_deployment/.env') }}
41+
restore-keys: |
42+
docker-${{ runner.os }}-
43+
2844
- name: Set up Helm
2945
uses: azure/setup-helm@v4.2.0
3046
with:
@@ -37,6 +53,35 @@ jobs:
3753
wait: 90s
3854
cluster_name: helm-test-cluster
3955

56+
- name: Verify KIND cluster is running
57+
run: |
58+
kubectl cluster-info
59+
kubectl get nodes
60+
61+
62+
- name: Setup COSI, S3 and IAM environments
63+
run: |
64+
set -e -o pipefail
65+
(
66+
echo "=== Setup COSI Controller, CRDs and Driver ==="
67+
kubectl create -k github.com/kubernetes-sigs/container-object-storage-interface
68+
make container
69+
kind load docker-image ghcr.io/scality/cosi-driver:latest --name helm-test-cluster
70+
) &
71+
(
72+
echo "=== Loading cached S3 and IAM Docker images ==="
73+
if [ -d /tmp/.docker_cache ] && [ "$(ls -A /tmp/.docker_cache 2>/dev/null)" ]; then
74+
for image in /tmp/.docker_cache/*.tar; do
75+
docker load -i "$image" || true # continue on failure
76+
done
77+
else
78+
echo "No cached images found. Skipping load."
79+
fi
80+
) &
81+
82+
# Wait for both background processes
83+
wait
84+
4085
- name: "Debug: SSH to runner"
4186
uses: scality/actions/action-ssh-to-runner@v1
4287
with:
@@ -49,15 +94,29 @@ jobs:
4994
timeout-minutes: 10
5095
continue-on-error: true
5196

52-
- name: Build COSI Driver Docker Image
53-
run: |
54-
make container
97+
- name: Setup IAM and S3 Services
98+
run: |-
99+
set -e -o pipefail;
100+
mkdir -p logs/s3 logs/iam logs/cosi_driver data/vaultdb
101+
chown -R runner:docker logs data
102+
chmod -R ugo+rwx logs data
103+
docker compose --profile iam_s3 up -d --quiet-pull
104+
bash ../scripts/wait_for_local_port.bash 8600 30
105+
bash ../scripts/wait_for_local_port.bash 8000 30
106+
working-directory: .github/s3_and_iam_deployment
55107

56-
- name: Load Docker Image into Kind Cluster
108+
- name: Save Images to Cache if not present
109+
if: steps.cache_docker_images.outputs.cache-hit != 'true'
57110
run: |
58-
kind load docker-image ghcr.io/scality/cosi-driver:latest --name helm-test-cluster
111+
source .github/s3_and_iam_deployment/.env
112+
echo "Vault Image: $VAULT_IMAGE"
113+
echo "CloudServer Image: $CLOUDSERVER_IMAGE"
114+
mkdir -p /tmp/.docker_cache
115+
docker save "$VAULT_IMAGE" -o /tmp/.docker_cache/vault_image.tar
116+
docker save "$CLOUDSERVER_IMAGE" -o /tmp/.docker_cache/cloudserver_image.tar
117+
shell: bash
59118

60-
- name: Install Scality COSI Helm Chart
119+
- name: Install Scality COSI Driver using Helm Chart
61120
run: |
62121
helm install scality-cosi-driver ./helm/scality-cosi-driver \
63122
--namespace container-object-storage-system \
@@ -73,6 +132,14 @@ jobs:
73132
run: |
74133
.github/scripts/verify_helm_install.sh
75134
135+
- name: E2E tests for greenfield use case using kustomize
136+
run: |
137+
.github/scripts/e2e_tests_greenfield_use_case.sh
138+
139+
- name: E2E tests for brownfield use case using kustomize
140+
run: |
141+
.github/scripts/e2e_tests_brownfield_use_case.sh
142+
76143
# the script accepts number of requests for APIs: CREATE_BUCKET, DELETE_BUCKET, GET_INFO
77144
# GRANT_ACCESS and REVOKE_ACCESS in order
78145
# Example below we are testing for those API counts:
@@ -83,7 +150,7 @@ jobs:
83150
# - 0 REVOKE_ACCESS
84151
- name: Verify metrics for healthcheck route
85152
run: |
86-
.github/scripts/e2e_tests_metrics.sh 0 0 1 0 0
153+
.github/scripts/e2e_tests_metrics.sh 2 1 1 2 2
87154
88155
- name: "Delay completion"
89156
if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
@@ -92,8 +159,35 @@ jobs:
92159
completion_delay_m: ${{ inputs.debug_delay_duration_minutes }}
93160
continue-on-error: true
94161

162+
- name: Cleaup IAM and S3 Services
163+
run: docker compose --profile iam_s3 down
164+
working-directory: .github/s3_and_iam_deployment
165+
166+
- name: Move S3 and IAM logs and data to artifacts directory
167+
if: always()
168+
run: |-
169+
set -e -o pipefail;
170+
mkdir -p .github/e2e_tests/artifacts/logs .github/e2e_tests/artifacts/data
171+
cp -r .github/s3_and_iam_deployment/logs/* .github/e2e_tests/artifacts/logs/
172+
cp -r .github/s3_and_iam_deployment/data/* .github/e2e_tests/artifacts/data/
173+
174+
- name: Capture Kubernetes Logs in artifacts directory
175+
if: always()
176+
run: |
177+
.github/scripts/capture_k8s_logs.sh
178+
95179
- name: Cleanup Helm Release and Namespace
96180
run: |
97181
helm uninstall scality-cosi-driver -n container-object-storage-system
98182
kubectl delete namespace container-object-storage-system
99183
if: always()
184+
185+
- name: Upload logs and data to Scality artifacts
186+
if: always()
187+
uses: scality/action-artifacts@v4
188+
with:
189+
method: upload
190+
url: https://artifacts.scality.net
191+
user: ${{ secrets.ARTIFACTS_USER }}
192+
password: ${{ secrets.ARTIFACTS_PASSWORD }}
193+
source: .github/e2e_tests/artifacts

.github/workflows/e2e-feature-tests.yml renamed to .github/workflows/kustomize-validation.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: End to End Feature Tests
1+
name: Kustomize Validation
22

33
on:
44
push:
@@ -18,7 +18,7 @@ on:
1818
default: 5
1919

2020
jobs:
21-
e2e-tests-with-kind:
21+
e2e-tests-with-kustomize:
2222
runs-on: ubuntu-latest
2323

2424
steps:

‎cosi-examples/brownfield/bucket.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ apiVersion: objectstorage.k8s.io/v1alpha1
22
kind: Bucket
33
metadata:
44
name: brownfield-bucket # should be same as bucket name
5-
namespace: container-object-storage-system
65
spec:
76
bucketClaim: {}
87
bucketClassName: brownfield-bucket-class

‎cosi-examples/brownfield/bucketaccess.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ apiVersion: objectstorage.k8s.io/v1alpha1
22
kind: BucketAccess
33
metadata:
44
name: brownfield-bucket-access
5-
namespace: container-object-storage-system
65
spec:
76
bucketAccessClassName: brownfield-bucket-access-class
87
bucketClaimName: brownfield-bucket-claim

‎cosi-examples/brownfield/bucketaccessclass.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ kind: BucketAccessClass
22
apiVersion: objectstorage.k8s.io/v1alpha1
33
metadata:
44
name: brownfield-bucket-access-class
5-
namespace: container-object-storage-system
65
driverName: cosi.scality.com
76
authenticationType: KEY
87
parameters:

‎cosi-examples/brownfield/bucketclaim.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ apiVersion: objectstorage.k8s.io/v1alpha1
22
kind: BucketClaim
33
metadata:
44
name: brownfield-bucket-claim
5-
namespace: container-object-storage-system
65
spec:
76
bucketClassName: brownfield-bucket-class
87
existingBucketName: brownfield-bucket # name of Bucket object

‎cosi-examples/brownfield/bucketclass.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ apiVersion: objectstorage.k8s.io/v1alpha1
22
kind: BucketClass
33
metadata:
44
name: brownfield-bucket-class
5-
namespace: container-object-storage-system
65
driverName: cosi.scality.com
76
deletionPolicy: Delete
87
parameters:

‎docs/Usage.md‎

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ In the **Scality COSI Driver**, both **Greenfield** and **Brownfield** provision
7777
7878
> Note:
7979
> For **fully working** examples, see the YAMLs in the [cosi-examples/brownfield](../cosi-examples/brownfield/) and [cosi-examples/greenfield](../cosi-examples/greenfield/) directories.
80-
> For brownfield scenario it is madatory to create COSI CRs in the same namespace as COSI driver and controller.
8180
8281
### 1.1 Greenfield: Creating a New Bucket
8382
@@ -92,7 +91,6 @@ Greenfield provisioning will create a brand-new S3 bucket in your object store,
9291
kind: BucketClass
9392
metadata:
9493
name: greenfield-bucketclass
95-
namespace: container-object-storage-system
9694
driverName: cosi.scality.com
9795
deletionPolicy: Delete
9896
parameters:
@@ -115,7 +113,6 @@ Greenfield provisioning will create a brand-new S3 bucket in your object store,
115113
kind: BucketClaim
116114
metadata:
117115
name: my-greenfield-bucketclaim
118-
namespace: container-object-storage-system
119116
spec:
120117
bucketClassName: greenfield-bucketclass
121118
protocols:
@@ -131,8 +128,6 @@ Greenfield provisioning will create a brand-new S3 bucket in your object store,
131128
132129
Brownfield provisioning allows you to manage an **already-existing** S3 bucket in Kubernetes.
133130
134-
> Note: For brownfield scenario, COSI CRs for Bucket and Access provisioning should be created in the same namespace as COSI driver and controller.
135-
136131
1. **Verify Existing Bucket**
137132
138133
Ensure the bucket already exists in S3 either through Storage Administrator or by running the following AWS CLI command:
@@ -151,7 +146,6 @@ Brownfield provisioning allows you to manage an **already-existing** S3 bucket i
151146
kind: BucketClass
152147
metadata:
153148
name: brownfield-bucketclass
154-
namespace: container-object-storage-system
155149
driverName: cosi.scality.com
156150
deletionPolicy: Delete
157151
parameters:
@@ -172,7 +166,6 @@ Brownfield provisioning allows you to manage an **already-existing** S3 bucket i
172166
kind: Bucket
173167
metadata:
174168
name: "<EXISTING_BUCKET_NAME>"
175-
namespace: container-object-storage-system
176169
spec:
177170
bucketClaim: {}
178171
driverName: cosi.scality.com
@@ -199,9 +192,8 @@ Brownfield provisioning allows you to manage an **already-existing** S3 bucket i
199192
kind: BucketClaim
200193
metadata:
201194
name: my-brownfield-bucketclaim
202-
namespace: container-object-storage-system
203195
spec:
204-
bucketClassName: brownfield-bucket-class
196+
bucketClassName: brownfield-bucketclass
205197
existingBucketName: "<EXISTING_BUCKET_NAME>"
206198
protocols:
207199
- S3
@@ -244,17 +236,15 @@ A `BucketAccessClass` defines how access (IAM policy or S3 keys) is granted:
244236
245237
```bash
246238
cat <<EOF | kubectl apply -f -
247-
apiVersion: objectstorage.k8s.io/v1alpha1
248239
kind: BucketAccessClass
240+
apiVersion: objectstorage.k8s.io/v1alpha1
249241
metadata:
250-
name: bucketaccessclass
251-
namespace: container-object-storage-system
252-
spec:
253-
driverName: cosi.scality.com
254-
authenticationType: KEY
255-
parameters:
256-
objectStorageSecretName: s3-secret-for-cosi
257-
objectStorageSecretNamespace: default
242+
name: bucket-access-class
243+
driverName: cosi.scality.com
244+
authenticationType: KEY
245+
parameters:
246+
objectStorageSecretName: s3-secret-for-cosi
247+
objectStorageSecretNamespace: default
258248
EOF
259249
```
260250
@@ -273,10 +263,9 @@ apiVersion: objectstorage.k8s.io/v1alpha1
273263
kind: BucketAccess
274264
metadata:
275265
name: my-bucketaccess
276-
namespace: container-object-storage-system
277266
spec:
278267
bucketClaimName: my-greenfield-bucketclaim # or my-brownfield-bucketclaim
279-
bucketAccessClassName: bucketaccessclass
268+
bucketAccessClassName: bucket-access-class
280269
credentialsSecretName: my-s3-credentials
281270
protocol: S3
282271
EOF

0 commit comments

Comments
 (0)