Skip to content

Commit b187feb

Browse files
White4ShadowWhite4Shadow
committed
* enhance cleanup and optimize performance
1 parent 260f5ea commit b187feb

File tree

9 files changed

+110
-102
lines changed

9 files changed

+110
-102
lines changed

‎service/backend/Dockerfile‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,26 @@
1-
21
FROM python:3.11-slim
32

43
WORKDIR /app
54

65
COPY requirements.txt .
76
RUN pip install --no-cache-dir -r requirements.txt
8-
RUN apt-get clean && apt-get update && apt-get install -y netcat-openbsd sqlite3 && rm -rf /var/lib/apt/lists/*
7+
RUN apt-get clean && apt-get update && apt-get install -y cron netcat-openbsd sqlite3 && rm -rf /var/lib/apt/lists/*
98
RUN pip install psycopg2-binary
10-
RUN pip install py-spy
119

1210
COPY generate_secret.sh .
1311

14-
15-
1612
COPY . .
1713

1814
COPY start.sh .
1915
RUN chmod +x start.sh
2016

17+
COPY cleanup/cleanup.sh /service/cleanup.sh
18+
RUN chmod +x /service/cleanup.sh
19+
20+
COPY cleanup/cron /etc/cron.d/cl-cron
21+
RUN chmod 0644 /etc/cron.d/cl-cron
22+
23+
2124
EXPOSE 2626
2225

2326
ENTRYPOINT ["./start.sh"]

‎service/backend/cleanup/cleanup.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
UPLOADS_DIR="/app/uploads"
3+
AGE_MINUTES=15
4+
5+
echo "$(date): [cleanup] Starting cleanup in $UPLOADS_DIR (files older than $AGE_MINUTES minutes)"
6+
7+
files_to_delete=$(find "$UPLOADS_DIR" -mindepth 1 -type f ! -path "$UPLOADS_DIR/default/*" -mmin +"$AGE_MINUTES")
8+
count_files=$(echo "$files_to_delete" | grep -c . || true)
9+
if [ "$count_files" -gt 0 ]; then
10+
echo "$(date): [cleanup] Deleting $count_files files:"
11+
echo "$files_to_delete"
12+
echo "$files_to_delete" | xargs -r rm -f
13+
else
14+
echo "$(date): [cleanup] No files to delete."
15+
fi
16+
17+
dirs_to_delete=$(find "$UPLOADS_DIR" -mindepth 1 -type d -empty ! -name "default")
18+
count_dirs=$(echo "$dirs_to_delete" | grep -c . || true)
19+
if [ "$count_dirs" -gt 0 ]; then
20+
echo "$(date): [cleanup] Deleting $count_dirs empty directories:"
21+
echo "$dirs_to_delete"
22+
echo "$dirs_to_delete" | xargs -r rmdir
23+
else
24+
echo "$(date):[cleanup] No empty directories to delete."
25+
fi
26+
27+
echo "$(date): [cleanup] Cleanup finished."

‎service/backend/cleanup/cron‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
MAILTO=""
2+
*/1 * * * * root /service/cleanup.sh >> /proc/1/fd/1 2>&1

‎service/backend/main.py‎

Lines changed: 0 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,6 @@ async def lifespan(app: FastAPI):
6868
finally:
6969
await db_gen.aclose()
7070

71-
task = asyncio.create_task(cleanup_old_records())
72-
7371
try:
7472
yield
7573
finally:
@@ -578,99 +576,7 @@ async def enter_dungeon(dungeon_id: int, current_user: schemas.User = Depends(ge
578576
winner=current_user.username if user_power_with_items >= dungeon.required_power else "No Winner",
579577
xp_gained=xp_gained, message=message, new_level=new_level)
580578

581-
async def cleanup_old_records():
582-
"""
583-
Periodically cleans up old records from the database and deletes associated
584-
user upload folders asynchronously.
585-
"""
586-
while True:
587-
print("Cleaning up old records...")
588-
db_gen = get_async_db()
589-
db = await anext(db_gen)
590-
try:
591-
ten_minutes_ago = datetime.utcnow() - timedelta(minutes=10)
592-
593-
users_to_delete_query = await db.execute(
594-
select(models.User.id, models.User.username).where(models.User.created_at < ten_minutes_ago)
595-
)
596-
deleted_users = users_to_delete_query.all()
597-
deleted_user_ids = [u[0] for u in deleted_users]
598-
deleted_usernames = [u[1] for u in deleted_users]
599-
600-
if deleted_user_ids:
601-
await db.execute(
602-
models.Inventory.__table__.delete().where(
603-
models.Inventory.user_id.in_(deleted_user_ids)
604-
)
605-
)
606-
607-
await db.execute(
608-
models.CompletedDungeon.__table__.delete().where(
609-
models.CompletedDungeon.user_id.in_(deleted_user_ids)
610-
)
611-
)
612-
613-
await db.execute(
614-
models.Fight.__table__.delete().where(
615-
(models.Fight.attacker_id.in_(deleted_user_ids)) |
616-
(models.Fight.defender_id.in_(deleted_user_ids))
617-
)
618-
)
619-
620-
await db.execute(
621-
models.Fight.__table__.delete().where(
622-
models.Fight.created_at < ten_minutes_ago
623-
)
624-
)
625-
626-
dungeons_subquery = select(models.Dungeon.id).where(
627-
(models.Dungeon.one_time_clear == False) | (models.Dungeon.one_time_clear == None)
628-
)
629-
await db.execute(
630-
models.CompletedDungeon.__table__.delete().where(
631-
models.CompletedDungeon.completion_timestamp < ten_minutes_ago,
632-
models.CompletedDungeon.dungeon_id.in_(dungeons_subquery)
633-
)
634-
)
635-
636-
await db.execute(
637-
models.User.__table__.delete().where(
638-
models.User.created_at < ten_minutes_ago
639-
)
640-
)
641-
642-
await db.execute(
643-
models.Item.__table__.delete().where(
644-
models.Item.created_at < ten_minutes_ago
645-
)
646-
)
647-
648-
await db.execute(
649-
models.Dungeon.__table__.delete().where(
650-
models.Dungeon.created_at < ten_minutes_ago,
651-
models.Dungeon.owner != 'all'
652-
)
653-
)
654-
655-
await db.commit()
656579

657-
loop = asyncio.get_event_loop()
658-
for username in deleted_usernames:
659-
user_dir = os.path.join("uploads", username)
660-
if os.path.exists(user_dir):
661-
try:
662-
await loop.run_in_executor(None, shutil.rmtree, user_dir)
663-
print(f"Deleted upload folder for user {username}")
664-
except Exception as e:
665-
print(f"Failed to delete upload folder for user {username}: {e}")
666-
667-
except Exception as e:
668-
print(f"Error during cleanup: {e}")
669-
await db.rollback()
670-
finally:
671-
await db_gen.aclose()
672-
673-
await asyncio.sleep(240)
674580

675581
@app.post("/images/upload", status_code=status.HTTP_201_CREATED)
676582
async def upload_dungeon_image(file: UploadFile = File(...),

‎service/backend/start.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
2+
crond &
33
bash generate_secret.sh && \
44
SECRET_VALUE=$(cat .env | cut -d'=' -f2) && \
55
echo "export SECRET_KEY=$SECRET_VALUE" >> /etc/profile.d/secret.sh

‎service/cleanup/Dockerfile‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM postgres:15-alpine
2+
3+
RUN apk add --no-cache bash cronie
4+
5+
COPY cleanup.sh /service/cleanup.sh
6+
RUN chmod +x /service/cleanup.sh
7+
8+
COPY cron /etc/cron.d/cl-cron
9+
RUN chmod 0644 /etc/cron.d/cl-cron && crontab /etc/cron.d/cl-cron
10+
11+
CMD ["crond", "-f", "-x", "sch,proc,pars,load,misc"]

‎service/cleanup/cleanup.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/sh
2+
echo "=== cleanup start: $(date) ==="
3+
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
4+
5+
PGUSER=backend
6+
PGPASSWORD=backend
7+
PGDATABASE=backenddb
8+
PGHOST=backend-db
9+
INTERVAL="10 minutes"
10+
11+
export PGUSER PGPASSWORD PGDATABASE PGHOST
12+
## psql -t -c "SELECT COUNT(*) FROM inventory WHERE user_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '10');"
13+
count=$(psql -t -c "SELECT COUNT(*) FROM inventory WHERE user_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL');" | xargs)
14+
echo "$(date): Deleting $count rows from inventory ..."
15+
psql -c "DELETE FROM inventory WHERE user_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL');"
16+
17+
count=$(psql -t -c "SELECT COUNT(*) FROM completed_dungeons WHERE user_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL');" | xargs)
18+
echo "$(date): Deleting $count rows from completed_dungeons ..."
19+
psql -c "DELETE FROM completed_dungeons WHERE user_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL');"
20+
21+
count=$(psql -t -c "SELECT COUNT(*) FROM fights WHERE attacker_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL') OR defender_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL');" | xargs)
22+
echo "$(date): Deleting $count rows from fights (user links) ..."
23+
psql -c "DELETE FROM fights WHERE attacker_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL') OR defender_id IN (SELECT id FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL');"
24+
25+
count=$(psql -t -c "SELECT COUNT(*) FROM fights WHERE created_at < NOW() - INTERVAL '$INTERVAL';" | xargs)
26+
echo "$(date): Deleting $count rows from fights (old fights) ..."
27+
psql -c "DELETE FROM fights WHERE created_at < NOW() - INTERVAL '$INTERVAL';"
28+
29+
count=$(psql -t -c "SELECT COUNT(*) FROM completed_dungeons WHERE completion_timestamp < NOW() - INTERVAL '$INTERVAL' AND dungeon_id IN (SELECT id FROM dungeons WHERE one_time_clear IS FALSE OR one_time_clear IS NULL);" | xargs)
30+
echo "$(date): Deleting $count rows from completed_dungeons (old dungeons) ..."
31+
psql -c "DELETE FROM completed_dungeons WHERE completion_timestamp < NOW() - INTERVAL '$INTERVAL' AND dungeon_id IN (SELECT id FROM dungeons WHERE one_time_clear IS FALSE OR one_time_clear IS NULL);"
32+
33+
count=$(psql -t -c "SELECT COUNT(*) FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL';" | xargs)
34+
echo "$(date): Deleting $count rows from users ..."
35+
psql -c "DELETE FROM users WHERE created_at < NOW() - INTERVAL '$INTERVAL';"
36+
37+
count=$(psql -t -c "SELECT COUNT(*) FROM inventory WHERE item_id IN (SELECT id FROM items WHERE created_at < NOW() - INTERVAL '$INTERVAL');" | xargs)
38+
echo "$(date): Deleting $count rows from inventory (old items) ..."
39+
psql -c "DELETE FROM inventory WHERE item_id IN (SELECT id FROM items WHERE created_at < NOW() - INTERVAL '$INTERVAL');"
40+
41+
count=$(psql -t -c "SELECT COUNT(*) FROM items WHERE created_at < NOW() - INTERVAL '$INTERVAL';" | xargs)
42+
echo "$(date): Deleting $count rows from items ..."
43+
psql -c "DELETE FROM items WHERE created_at < NOW() - INTERVAL '$INTERVAL';"
44+
45+
46+
count=$(psql -t -c "SELECT COUNT(*) FROM dungeons WHERE created_at < NOW() - INTERVAL '$INTERVAL' AND owner != 'all';" | xargs)
47+
echo "$(date): Deleting $count rows from dungeons ..."
48+
psql -c "DELETE FROM dungeons WHERE created_at < NOW() - INTERVAL '$INTERVAL' AND owner != 'all';"

‎service/cleanup/cron‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
MAILTO=""
2+
* * * * * root /service/cleanup.sh > /proc/1/fd/1 2>&1

‎service/docker-compose.yml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,17 @@ services:
4949
timeout: 5s
5050
retries: 5
5151
start_period: 5s
52-
mem_limit: 1G
53-
cpus: 1
52+
db-cleanup:
53+
build:
54+
context: ./cleanup
55+
environment:
56+
PGUSER: backend
57+
PGPASSWORD: backend
58+
PGDATABASE: backenddb
59+
PGHOST: backend-db
60+
depends_on:
61+
- backend-db
62+
restart: unless-stopped
5463

5564

5665
volumes:

0 commit comments

Comments
 (0)