Skip to content

Commit 26af777

Browse files
2xlhe2xlhe
authored
Merge branch 'main' into boto3
2 parents 548bdec + dfd1964 commit 26af777

8 files changed

+393
-3
lines changed

‎.github/workflows/full-docker-image.yml

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Build test OCI Image
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
env:
8+
REGISTRY: ghcr.io
9+
IMAGE_NAME: magalucloud/s3-specs
10+
jobs:
11+
build:
12+
runs-on: ubuntu-latest
13+
permissions:
14+
contents: read
15+
packages: write
16+
steps:
17+
- name: Checkout Repository
18+
uses: actions/checkout@v4
19+
20+
- name: Login to Github Container Registry
21+
uses: docker/login-action@v3
22+
with:
23+
registry: ${{ env.REGISTRY }}
24+
username: ${{ github.repository_owner }}
25+
password: ${{ secrets.GITHUB_TOKEN }}
26+
27+
- name: "Build and push OCI image, tag: full"
28+
uses: docker/[email protected]
29+
with:
30+
context: .
31+
push: true
32+
file: oci/full.Containerfile
33+
tags: |
34+
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:full
35+
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:full-${{github.sha}}

‎bin/configure_profiles.py

+17-2
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ def configure_profiles(profiles):
7878
if not (endpoint and access_key and secret_key and region):
7979
print(f"Perfil {profile_name} está incompleto. Ignorando...")
8080
continue
81+
82+
if (access_key == "YOUR-KEY-ID-HERE" or secret_key == "YOUR-SECRET-KEY-HERE" ):
83+
print(f"Perfil {profile_name} está incompleto. Ignorando...")
84+
continue
8185

8286
set_aws_profiles(profile_name=profile_name, data=profile_data)
8387
set_rclone_profiles(profile_name=profile_name, data=profile_data)
@@ -91,7 +95,18 @@ def configure_profiles(profiles):
9195
if __name__ == "__main__":
9296
if len(sys.argv) > 1:
9397
with open(sys.argv[1], 'r') as file:
94-
profiles = yaml.safe_load(file)
98+
profiles_list = yaml.safe_load(file)
99+
profiles = {}
100+
for profile in list(profiles_list.items())[2][1]:
101+
if not isinstance(profile, dict):
102+
profiles = profiles_list
103+
break
104+
profiles[profile.get("profile_name")] = {}
105+
profiles[profile.get("profile_name")]["endpoint"] = profile.get("endpoint_url")
106+
profiles[profile.get("profile_name")]["access_key"] = profile.get("aws_access_key_id")
107+
profiles[profile.get("profile_name")]["secret_key"] = profile.get("aws_secret_access_key")
108+
profiles[profile.get("profile_name")]["region"] = profile.get("region_name")
109+
95110
else:
96111
profiles_data = os.getenv("PROFILES")
97112
if not profiles_data:
@@ -101,4 +116,4 @@ def configure_profiles(profiles):
101116

102117
print(f"Number of profiles - {len(profiles)}")
103118
configure_profiles(profiles)
104-
print(f"Profile Configurations Done!")
119+
print("Profile Configurations Done!")

‎docs/conftest.py

+30-1
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,9 @@ def bucket_with_one_object_and_cold_storage_class(s3_client):
206206
delete_object_and_wait(s3_client, bucket_name, object_key)
207207
delete_bucket_and_wait(s3_client, bucket_name)
208208

209-
@pytest.fixture(params=[{'object_key': 'test-object.txt'}])
209+
@pytest.fixture(params=[{
210+
'object_key': 'test-object.txt'
211+
}])
210212
def bucket_with_one_object(request, s3_client):
211213
# this fixture accepts an optional request.param['object_key'] if you
212214
# need a custom specific key name for your test
@@ -227,6 +229,33 @@ def bucket_with_one_object(request, s3_client):
227229
delete_object_and_wait(s3_client, bucket_name, object_key)
228230
delete_bucket_and_wait(s3_client, bucket_name)
229231

232+
@pytest.fixture(params=[{
233+
'object_prefix': "",
234+
'object_key_list': ['test-object-1.txt', 'test-object-2.txt']
235+
}])
236+
def bucket_with_many_objects(request, s3_client):
237+
# this fixture accepts an optional request.param['object_key_list'] with a list of custom key names
238+
object_key_list = request.param['object_key_list']
239+
# and a string prefix to prepend on all objects
240+
object_prefix = request.param.get('object_prefix', "")
241+
242+
bucket_name = generate_unique_bucket_name(base_name='fixture-bucket-with-many-objects')
243+
create_bucket_and_wait(s3_client, bucket_name)
244+
245+
content = b"Sample content for testing presigned URLs."
246+
for object_key in object_key_list:
247+
put_object_and_wait(s3_client, bucket_name, f"{object_prefix}{object_key}", content)
248+
249+
# Yield the bucket name and object details to the test
250+
yield bucket_name, object_prefix, content
251+
252+
# Teardown: Delete the object and bucket after the test
253+
for object_key in object_key_list:
254+
delete_object_and_wait(s3_client, bucket_name, object_key)
255+
delete_bucket_and_wait(s3_client, bucket_name)
256+
257+
258+
230259
@pytest.fixture
231260
def bucket_with_one_storage_class_cold_object(s3_client, bucket_with_one_object):
232261
# Generate a unique bucket name and ensure it exists

‎docs/object_download_cli.py

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# ---
2+
# jupyter:
3+
# kernelspec:
4+
# name: s3-specs
5+
# display_name: S3 Specs
6+
# language_info:
7+
# name: python
8+
# ---
9+
10+
# # Download de Objeto utilizando a CLI MGC
11+
#
12+
# O comando para o download de um objeto via mgc cli é o `object-storage objects download`
13+
# como exemplificado nos exemplos abaixo:
14+
15+
commands = [
16+
"{mgc_path} object-storage objects download '{bucket_name}/{object_key}' '{local_path}'",
17+
"{mgc_path} os objects download '{bucket_name}/{object_key}' '{local_path}'",
18+
]
19+
20+
# + {"jupyter": {"source_hidden": true}}
21+
import pytest
22+
import logging
23+
import subprocess
24+
from shlex import split
25+
from s3_helpers import (
26+
run_example,
27+
)
28+
pytestmark = [pytest.mark.basic, pytest.mark.cli]
29+
# -
30+
31+
# + tags=["parameters"]
32+
config = "../params/br-ne1.yaml"
33+
# -
34+
35+
# +
36+
object_keys = [
37+
"test-object.txt",
38+
"test/object/sub/folder/😘 Arquivo com espaço e acentuação 🍕.txt",
39+
]
40+
test_cases = [
41+
(command, {'object_key': object_key})
42+
for command in commands
43+
for object_key in object_keys
44+
]
45+
46+
@pytest.mark.parametrize(
47+
"cmd_template, bucket_with_one_object",
48+
test_cases,
49+
indirect=["bucket_with_one_object"]
50+
)
51+
def test_download_object_cli(cmd_template, bucket_with_one_object, active_mgc_workspace, mgc_path):
52+
local_path = "/tmp/downloaded test file.txt"
53+
bucket_name, object_key, content = bucket_with_one_object
54+
cmd = split(cmd_template.format(mgc_path=mgc_path, bucket_name=bucket_name, object_key=object_key, local_path=local_path))
55+
56+
result = subprocess.run(cmd, capture_output=True, text=True)
57+
58+
assert result.returncode == 0, f"Command failed with error: {result.stderr}"
59+
logging.info(f"Output from {cmd_template}: {result.stdout}")
60+
61+
run_example(__name__, "test_download_object_cli", config=config)
62+
# -

‎docs/objects_list_cli.py

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# ---
2+
# jupyter:
3+
# kernelspec:
4+
# name: s3-specs
5+
# display_name: S3 Specs
6+
# language_info:
7+
# name: python
8+
# ---
9+
10+
# # Listagem de Objeto utilizando a CLI MGC
11+
#
12+
# O comando para a listagem de objetos em um bucket via mgc cli é o `object-storage objects list`
13+
# junto com o nome do bucket é possivel incluir um prefixo, considerando que é comum ferramentas
14+
# de sync de diretórios usarem nomes de objetos em um formato de "path"
15+
# como nos exemplos abaixo:
16+
17+
commands = [
18+
"{mgc_path} object-storage objects list '{bucket_name}'",
19+
"{mgc_path} os objects list '{bucket_name}/{object_prefix}'",
20+
]
21+
22+
# + {"jupyter": {"source_hidden": true}}
23+
import pytest
24+
import logging
25+
import subprocess
26+
from shlex import split
27+
from s3_helpers import (
28+
run_example,
29+
)
30+
pytestmark = [pytest.mark.basic, pytest.mark.cli]
31+
32+
# -
33+
34+
# + tags=["parameters"]
35+
config = "../params/br-ne1.yaml"
36+
# -
37+
38+
# +
39+
40+
test_buckets = [
41+
{
42+
"object_prefix": "",
43+
"object_key_list": ["simple-object-key.txt"],
44+
},
45+
{
46+
"object_prefix": "simple-prefix/",
47+
"object_key_list": [ "sufix-1.txt", "sufix-2.txt", ]
48+
},
49+
{
50+
"object_prefix": "prefix/with/multiple/slashes/",
51+
"object_key_list": [
52+
"acentuação e emojis 🎉 🥳 😘-1.txt",
53+
"acentuação e emojis 😘 🎉 🥳-2.txt",
54+
]
55+
},
56+
]
57+
58+
test_cases = [
59+
(command, test_bucket)
60+
for command in commands
61+
for test_bucket in test_buckets
62+
]
63+
64+
@pytest.mark.parametrize(
65+
"cmd_template, bucket_with_many_objects",
66+
test_cases,
67+
indirect=["bucket_with_many_objects"]
68+
)
69+
def test_list_objects_cli(cmd_template, bucket_with_many_objects, active_mgc_workspace, mgc_path):
70+
bucket_name, object_prefix, _content = bucket_with_many_objects
71+
cmd = split(cmd_template.format(mgc_path=mgc_path, bucket_name=bucket_name, object_prefix=object_prefix))
72+
73+
result = subprocess.run(cmd, capture_output=True, text=True)
74+
75+
assert result.returncode == 0, f"Command failed with error: {result.stderr}"
76+
logging.info(f"Output from {cmd_template}: {result.stdout}")
77+
78+
run_example(__name__, "test_list_objects_cli", config=config)
79+
# -

‎justfile

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
set shell := ["bash", "-cu"]
2+
3+
@default:
4+
just --list
5+
6+
#Configure profiles in the CLI's
7+
setup-profiles:
8+
uv run ./bin/configure_profiles.py ./params.example.yaml
9+
10+
#Execute the tests of s3-specs
11+
tests *pytest_params:
12+
just setup-profiles
13+
uv run pytest ./docs/*_test.py --config ./params.example.yaml {{pytest_params}}
14+
15+
#Execute the tests of s3-specs and generate a report of the tests after running.
16+
report category:
17+
just setup-profiles
18+
bash -c "cd reports && ./run.sh {{category}} ../params.example.yaml ../docs/ && mv report_*.pdf outputs/"

‎oci/README.md

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Running S3 Specs from OCI image (Docker or Podman)
2+
3+
## 1. Create a config file with your endpoints and credentials,
4+
following the yaml format in profiles.example.yaml
5+
6+
```bash
7+
wget https://raw.githubusercontent.com/MagaluCloud/s4-specs/refs/heads/main/params.example.yaml -O params.yaml
8+
vim params.yaml
9+
# Edit to add this values on the number of profiles your tests needs
10+
# (policy and acl tests for example needs 2 different profiles):
11+
# region_name: "br-ne1"
12+
# endpoint_url: "https://br-se1.magaluobjects.com/"
13+
# aws_access_key_id: "YOUR-KEY-ID-HERE"
14+
# aws_secret_access_key: "YOUR-SECRET-KEY-HERE"
15+
```
16+
17+
### 1.1. Optional: build the image locally
18+
19+
If you cloned the repo and want to build the image locally instead of using the published ones,
20+
use the Containerfiles under the `oci` folder:
21+
22+
```bash
23+
IMAGE_NAME=myimage podman build -t "$IMAGE_NAME" -f ./oci/full.Containerfile .
24+
```
25+
26+
## 2. Run all tests (this might take several minutes)
27+
28+
```bash
29+
# $IMAGE_NAME is the local tag name, if you build the Containerfile yourself,
30+
# or the published one like https://ghcr.io/magalucloud/s3-specs:full
31+
#
32+
# all examples can be run with docker as well
33+
podman run -t \
34+
-v $(realpath params.yaml):/app/params.example.yaml \
35+
"$IMAGE_NAME" tests \
36+
-n auto
37+
```
38+
39+
> ::note:: Whatever comes after the image name (`-n auto` in the example above)
40+
are pytest arguments, see `oci/entrypoint.sh` for details. The next examples are
41+
variations of test filters using the markers feature of pytest (`-m` argument).
42+
43+
## 3. Run all tests, excluding specific categories
44+
45+
In following example we use the `-m` arg with the `not` operator to exclude two categories:
46+
47+
```bash
48+
podman run -t \
49+
-v $(realpath params.yaml):/app/params.example.yaml \
50+
"$IMAGE_NAME" tests \
51+
-n auto -m "not locking not slow"
52+
```
53+
54+
## 4. Run only some specific categories
55+
56+
You can pass pytest arguments to the image entrypoint, for example, you can filter tests of
57+
specific categories, using the `-m` arg for markers, like:
58+
59+
```bash
60+
podman run -t \
61+
-v $(realpath params.yaml):/app/params.example.yaml \
62+
"$IMAGE_NAME" tests \
63+
-n auto -m "basic cold_storage"
64+
```
65+
66+
## 5. Run generating reports
67+
68+
You can generate reports of the tests using the following command:
69+
70+
```bash
71+
podman run -t \
72+
-v $(realpath .):/app/reports/outputs \
73+
-v $(realpath params.yaml):/app/params.example.yaml \
74+
"$IMAGE_NAME" reports \
75+
category_of_tests
76+
```
77+
78+
To execute this command you need to pass a category of tests you want to running. The possibles categories are:
79+
80+
- full: execute all the tests
81+
- versioning: execute versioning tests
82+
- basic: execute basic tests (included acl, list, presigned url and bucketname)
83+
- policy: execute policies tests
84+
- cold: execute cold storage tests
85+
- locking: execute locking tests
86+
- big-objects: execute big-objects tests

0 commit comments

Comments
 (0)