Skip to content

tests: add integration tests for s3 #215

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+270 −204
Open

tests: add integration tests for s3 #215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
35bd5be
tests: add integration tests for s3
quitrk Jun 30, 2025
254f019
chore: update deps
quitrk Jun 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions integration/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import asyncio

from .s3 import run as s3_run

async def main():
success = True

try:
# this will test integration with aioboto3
await s3_run()
except Exception as e:
print(f'Error: {e}')
success = False

if not success:
raise Exception('Integration tests failed')


asyncio.run(main())
34 changes: 34 additions & 0 deletions integration/s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import os

async def run():
from skynet.modules.ttt.s3 import S3

s3 = S3()

file = './test_upload.txt'
# create a test file
with open(file, 'w') as f:
f.write('This is a test file for S3 integration.')

# # # Test upload
await s3.upload_file(file)

# # delete the local test file
os.remove(file)

# # # Test download
await s3.download_file(file)

# # # Check if the file exists after download
if not os.path.exists(file):
raise Exception('Downloaded file does not exist')

# # # Test delete
await s3.delete_file(file)
os.remove(file)

# # # Check if the file is deleted from S3
await s3.download_file(file)

if os.path.exists(file):
raise Exception('File was not deleted from S3')
331 changes: 171 additions & 160 deletions poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ optional = true
vllm = "0.7.3"

[tool.poetry.dependencies]
aioboto3 = "14.3.0"
aioboto3 = "15.0.0"
aiofiles = "24.1.0"
aiohttp = "3.11.18"
aiohttp = "3.12.13"
async-lru = "2.0.5"
av = "^12.3.0"
beautifulsoup4 = "4.13.4"
Expand All @@ -42,19 +42,19 @@ faster-whisper = "1.1.1"
flashrank = "^0.2.10"
kreuzberg = "3.1.4"
langchain = "0.3.25"
langchain-community = "0.3.22"
langchain-huggingface = "0.2.0"
langchain-openai = "0.3.16"
oci = "2.151.0"
prometheus-client = "0.21.1"
langchain-community = "0.3.25"
langchain-huggingface = "0.3.0"
langchain-openai = "0.3.27"
oci = "2.154.3"
prometheus-client = "0.22.1"
prometheus-fastapi-instrumentator = "7.1.0"
pybase64 = "1.4.1"
pydantic = "2.11.4"
pydantic = "2.11.7"
pyjwt = {version = "2.10.1", extras = ["crypto"]}
pypdf = "5.4.0"
pypdf = "5.7.0"
python = "~3.11"
python-multipart = "0.0.20"
redis = "6.1.0"
redis = "6.2.0"
sentence-transformers = "^3.4.1"
silero-vad = "^5.1.2"
torch = "2.5.1"
Expand Down
34 changes: 17 additions & 17 deletions requirements-vllm.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
aioboto3==14.3.0 ; python_version == "3.11"
aiobotocore==2.22.0 ; python_version == "3.11"
aioboto3==15.0.0 ; python_version == "3.11"
aiobotocore==2.23.0 ; python_version == "3.11"
aiofiles==24.1.0 ; python_version == "3.11"
aiohappyeyeballs==2.6.1 ; python_version == "3.11"
aiohttp==3.11.18 ; python_version == "3.11"
aiohttp==3.12.13 ; python_version == "3.11"
aioitertools==0.12.0 ; python_version == "3.11"
aiosignal==1.3.2 ; python_version == "3.11"
airportsdata==20250224 ; python_version == "3.11"
Expand All @@ -15,8 +15,8 @@ attrs==25.3.0 ; python_version == "3.11"
av==12.3.0 ; python_version == "3.11"
beautifulsoup4==4.13.4 ; python_version == "3.11"
blake3==1.0.5 ; python_version == "3.11"
boto3==1.37.3 ; python_version == "3.11"
botocore==1.37.3 ; python_version == "3.11"
boto3==1.38.27 ; python_version == "3.11"
botocore==1.38.27 ; python_version == "3.11"
certifi==2025.4.26 ; python_version == "3.11"
cffi==1.17.1 ; python_version == "3.11"
charset-normalizer==3.4.2 ; python_version == "3.11"
Expand Down Expand Up @@ -72,13 +72,13 @@ jsonpointer==3.0.0 ; python_version == "3.11"
jsonschema-specifications==2025.4.1 ; python_version == "3.11"
jsonschema==4.23.0 ; python_version == "3.11"
kreuzberg==3.1.4 ; python_version == "3.11"
langchain-community==0.3.22 ; python_version == "3.11"
langchain-core==0.3.60 ; python_version == "3.11"
langchain-huggingface==0.2.0 ; python_version == "3.11"
langchain-openai==0.3.16 ; python_version == "3.11"
langchain-community==0.3.25 ; python_version == "3.11"
langchain-core==0.3.66 ; python_version == "3.11"
langchain-huggingface==0.3.0 ; python_version == "3.11"
langchain-openai==0.3.27 ; python_version == "3.11"
langchain-text-splitters==0.3.8 ; python_version == "3.11"
langchain==0.3.25 ; python_version == "3.11"
langsmith==0.3.42 ; python_version == "3.11"
langsmith==0.3.45 ; python_version == "3.11"
lark==1.2.2 ; python_version == "3.11"
llvmlite==0.43.0 ; python_version == "3.11"
lm-format-enforcer==0.10.11 ; python_version == "3.11"
Expand Down Expand Up @@ -110,9 +110,9 @@ nvidia-cusparse-cu12==12.3.1.170 ; platform_system == "Linux" and platform_machi
nvidia-nccl-cu12==2.21.5 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
nvidia-nvjitlink-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
nvidia-nvtx-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
oci==2.151.0 ; python_version == "3.11"
oci==2.154.3 ; python_version == "3.11"
onnxruntime==1.22.0 ; python_version == "3.11"
openai==1.81.0 ; python_version == "3.11"
openai==1.93.0 ; python_version == "3.11"
opencv-python-headless==4.11.0.86 ; python_version == "3.11"
orjson==3.10.18 ; python_version == "3.11" and platform_python_implementation != "PyPy"
outlines-core==0.1.26 ; python_version == "3.11"
Expand All @@ -122,7 +122,7 @@ partial-json-parser==0.2.1.1.post5 ; python_version == "3.11"
pillow==11.2.1 ; python_version == "3.11"
playa-pdf==0.5.0 ; python_version == "3.11"
pluggy==1.6.0 ; python_version == "3.11"
prometheus-client==0.21.1 ; python_version == "3.11"
prometheus-client==0.22.1 ; python_version == "3.11"
prometheus-fastapi-instrumentator==7.1.0 ; python_version == "3.11"
propcache==0.3.1 ; python_version == "3.11"
protobuf==6.31.0 ; python_version == "3.11"
Expand All @@ -134,11 +134,11 @@ pycountry==24.6.1 ; python_version == "3.11"
pycparser==2.22 ; python_version == "3.11"
pydantic-core==2.33.2 ; python_version == "3.11"
pydantic-settings==2.9.1 ; python_version == "3.11"
pydantic==2.11.4 ; python_version == "3.11"
pydantic==2.11.7 ; python_version == "3.11"
pygments==2.19.1 ; python_version == "3.11"
pyjwt==2.10.1 ; python_version == "3.11"
pyopenssl==24.3.0 ; python_version == "3.11"
pypdf==5.4.0 ; python_version == "3.11"
pypdf==5.7.0 ; python_version == "3.11"
pypdfium2==4.30.0 ; python_version == "3.11"
pyreadline3==3.5.4 ; python_version == "3.11" and sys_platform == "win32"
pytest==7.4.4 ; python_version == "3.11"
Expand All @@ -151,15 +151,15 @@ pytz==2025.2 ; python_version == "3.11"
pyyaml==6.0.2 ; python_version == "3.11"
pyzmq==26.4.0 ; python_version == "3.11"
ray==2.40.0 ; python_version == "3.11"
redis==6.1.0 ; python_version == "3.11"
redis==6.2.0 ; python_version == "3.11"
referencing==0.36.2 ; python_version == "3.11"
regex==2024.11.6 ; python_version == "3.11"
requests-toolbelt==1.0.0 ; python_version == "3.11"
requests==2.32.3 ; python_version == "3.11"
rich-toolkit==0.14.6 ; python_version == "3.11"
rich==14.0.0 ; python_version == "3.11"
rpds-py==0.25.1 ; python_version == "3.11"
s3transfer==0.11.3 ; python_version == "3.11"
s3transfer==0.13.0 ; python_version == "3.11"
safetensors==0.5.3 ; python_version == "3.11"
scikit-learn==1.6.1 ; python_version == "3.11"
scipy==1.15.3 ; python_version == "3.11"
Expand Down
34 changes: 17 additions & 17 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
aioboto3==14.3.0 ; python_version == "3.11"
aiobotocore==2.22.0 ; python_version == "3.11"
aioboto3==15.0.0 ; python_version == "3.11"
aiobotocore==2.23.0 ; python_version == "3.11"
aiofiles==24.1.0 ; python_version == "3.11"
aiohappyeyeballs==2.6.1 ; python_version == "3.11"
aiohttp==3.11.18 ; python_version == "3.11"
aiohttp==3.12.13 ; python_version == "3.11"
aioitertools==0.12.0 ; python_version == "3.11"
aiosignal==1.3.2 ; python_version == "3.11"
annotated-types==0.7.0 ; python_version == "3.11"
Expand All @@ -12,8 +12,8 @@ async-timeout==5.0.1 ; python_version == "3.11" and python_full_version < "3.11.
attrs==25.3.0 ; python_version == "3.11"
av==12.3.0 ; python_version == "3.11"
beautifulsoup4==4.13.4 ; python_version == "3.11"
boto3==1.37.3 ; python_version == "3.11"
botocore==1.37.3 ; python_version == "3.11"
boto3==1.38.27 ; python_version == "3.11"
botocore==1.38.27 ; python_version == "3.11"
certifi==2025.4.26 ; python_version == "3.11"
cffi==1.17.1 ; python_version == "3.11"
charset-normalizer==3.4.2 ; python_version == "3.11"
Expand Down Expand Up @@ -52,13 +52,13 @@ joblib==1.5.0 ; python_version == "3.11"
jsonpatch==1.33 ; python_version == "3.11"
jsonpointer==3.0.0 ; python_version == "3.11"
kreuzberg==3.1.4 ; python_version == "3.11"
langchain-community==0.3.22 ; python_version == "3.11"
langchain-core==0.3.60 ; python_version == "3.11"
langchain-huggingface==0.2.0 ; python_version == "3.11"
langchain-openai==0.3.16 ; python_version == "3.11"
langchain-community==0.3.25 ; python_version == "3.11"
langchain-core==0.3.66 ; python_version == "3.11"
langchain-huggingface==0.3.0 ; python_version == "3.11"
langchain-openai==0.3.27 ; python_version == "3.11"
langchain-text-splitters==0.3.8 ; python_version == "3.11"
langchain==0.3.25 ; python_version == "3.11"
langsmith==0.3.42 ; python_version == "3.11"
langsmith==0.3.45 ; python_version == "3.11"
lxml==5.4.0 ; python_version == "3.11"
markupsafe==3.0.2 ; python_version == "3.11"
marshmallow==3.26.1 ; python_version == "3.11"
Expand All @@ -80,25 +80,25 @@ nvidia-cusparse-cu12==12.3.1.170 ; platform_system == "Linux" and platform_machi
nvidia-nccl-cu12==2.21.5 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
nvidia-nvjitlink-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
nvidia-nvtx-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
oci==2.151.0 ; python_version == "3.11"
oci==2.154.3 ; python_version == "3.11"
onnxruntime==1.22.0 ; python_version == "3.11"
openai==1.81.0 ; python_version == "3.11"
openai==1.93.0 ; python_version == "3.11"
orjson==3.10.18 ; python_version == "3.11" and platform_python_implementation != "PyPy"
packaging==24.2 ; python_version == "3.11"
pillow==11.2.1 ; python_version == "3.11"
playa-pdf==0.5.0 ; python_version == "3.11"
prometheus-client==0.21.1 ; python_version == "3.11"
prometheus-client==0.22.1 ; python_version == "3.11"
prometheus-fastapi-instrumentator==7.1.0 ; python_version == "3.11"
propcache==0.3.1 ; python_version == "3.11"
protobuf==6.31.0 ; python_version == "3.11"
pybase64==1.4.1 ; python_version == "3.11"
pycparser==2.22 ; python_version == "3.11"
pydantic-core==2.33.2 ; python_version == "3.11"
pydantic-settings==2.9.1 ; python_version == "3.11"
pydantic==2.11.4 ; python_version == "3.11"
pydantic==2.11.7 ; python_version == "3.11"
pyjwt==2.10.1 ; python_version == "3.11"
pyopenssl==24.3.0 ; python_version == "3.11"
pypdf==5.4.0 ; python_version == "3.11"
pypdf==5.7.0 ; python_version == "3.11"
pypdfium2==4.30.0 ; python_version == "3.11"
pyreadline3==3.5.4 ; python_version == "3.11" and sys_platform == "win32"
python-calamine==0.3.2 ; python_version == "3.11"
Expand All @@ -108,11 +108,11 @@ python-multipart==0.0.20 ; python_version == "3.11"
python-pptx==1.0.2 ; python_version == "3.11"
pytz==2025.2 ; python_version == "3.11"
pyyaml==6.0.2 ; python_version == "3.11"
redis==6.1.0 ; python_version == "3.11"
redis==6.2.0 ; python_version == "3.11"
regex==2024.11.6 ; python_version == "3.11"
requests-toolbelt==1.0.0 ; python_version == "3.11"
requests==2.32.3 ; python_version == "3.11"
s3transfer==0.11.3 ; python_version == "3.11"
s3transfer==0.13.0 ; python_version == "3.11"
safetensors==0.5.3 ; python_version == "3.11"
scikit-learn==1.6.1 ; python_version == "3.11"
scipy==1.15.3 ; python_version == "3.11"
Expand Down
2 changes: 2 additions & 0 deletions skynet/modules/ttt/s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import aiofiles.os
from botocore.config import Config

from skynet.env import (
Expand Down Expand Up @@ -32,6 +33,7 @@ async def download_file(self, filename):
log.info(f'Downloaded file from S3: {filename}')
except Exception as e:
log.error(f'Failed to download file {filename} from S3: {e}')
await aiofiles.os.remove(filename)

async def upload_file(self, filename):
try:
Expand Down