Skip to content

Commit 0039b3c

Browse files
authored
Merge pull request #100 from SavageAim/return-to-celery
Prep for VM deployment
2 parents c9d6cb1 + 99da824 commit 0039b3c

35 files changed

Lines changed: 1377 additions & 2163 deletions

.github/workflows/docker-build.yml

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -80,43 +80,6 @@ jobs:
8080
docker image tag ghcr.io/savageaim/app/ws-backend:${{ steps.get_release.outputs.tag_name }} freyamade/savageaim:websockets-${{ steps.get_release.outputs.tag_name }}
8181
docker push --all-tags freyamade/savageaim
8282
83-
build-task-backend:
84-
runs-on: ubuntu-latest
85-
defaults:
86-
run:
87-
working-directory: backend
88-
89-
steps:
90-
- uses: actions/checkout@v3
91-
92-
- name: Log in to Docker Hub
93-
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
94-
with:
95-
username: ${{ secrets.DOCKER_USERNAME }}
96-
password: ${{ secrets.DOCKER_PASSWORD }}
97-
98-
- name: Log in to the Container registry
99-
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
100-
with:
101-
registry: ghcr.io
102-
username: ${{ github.actor }}
103-
password: ${{ secrets.GITHUB_TOKEN }}
104-
105-
- name: Get Release Info
106-
id: get_release
107-
uses: bruceadams/get-release@v1.3.2
108-
env:
109-
GITHUB_TOKEN: ${{ github.token }}
110-
111-
- name: Build and Push Docker Images
112-
run: |
113-
docker build . --file deployment/tasks.Dockerfile -t ghcr.io/savageaim/app/task-backend:latest -t ghcr.io/savageaim/app/task-backend:${{ steps.get_release.outputs.tag_name }}
114-
docker push --all-tags ghcr.io/savageaim/app/task-backend
115-
# Rename images to dockerhub and push there too
116-
docker image tag ghcr.io/savageaim/app/task-backend:latest freyamade/savageaim:tasks
117-
docker image tag ghcr.io/savageaim/app/task-backend:${{ steps.get_release.outputs.tag_name }} freyamade/savageaim:tasks-${{ steps.get_release.outputs.tag_name }}
118-
docker push --all-tags freyamade/savageaim
119-
12083
build-frontend:
12184
runs-on: ubuntu-latest
12285
defaults:

backend/api/apps.py

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,3 @@
44
class ApiConfig(AppConfig):
55
name = 'api'
66
default_auto_field = 'django.db.models.AutoField'
7-
8-
def ready(self):
9-
super().ready()
10-
# Import all our cloud tasks in here so they can be discovered
11-
from api.tasks import ( # noqa
12-
CheckGameVersionTask,
13-
DBCleanupTask,
14-
RefreshTokensTask,
15-
SeedTask,
16-
VerificationReminderTask,
17-
VerifyCharacterTask,
18-
)
Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,81 @@
1+
# stdlib
2+
from os import scandir
3+
from pathlib import Path
14
# lib
5+
from django.db import IntegrityError
6+
import yaml
7+
from django.conf import settings
28
from django.core.management.base import BaseCommand
39
# local
4-
from api.tasks import SeedTask
10+
from api import models
511

612

713
class Command(BaseCommand):
814
help = 'Seed the DB with static data for Gear, Tier and Job information.'
915

1016
def handle(self, *args, **options):
11-
SeedTask.sync({}, {'attributes': 'required'})
17+
self.stdout.write(self.style.HTTP_REDIRECT('Beginning Seed of DB'))
18+
seed_data_dir = settings.BASE_DIR / 'seed_data'
19+
gear_data_dir = seed_data_dir / 'gear'
20+
21+
# Get the Tier and Gear data and import them
22+
with open(seed_data_dir / 'tiers.yml', 'r') as f:
23+
self.stdout.write(self.style.HTTP_REDIRECT('Seeding Tiers'))
24+
self.import_file(f, models.Tier)
25+
26+
with scandir(gear_data_dir) as expac_dirs:
27+
for expac_dir in expac_dirs:
28+
if not expac_dir.is_dir():
29+
continue
30+
31+
with scandir(expac_dir.path) as gear_files:
32+
for file in gear_files:
33+
version = Path(file.path).stem
34+
self.stdout.write(self.style.HTTP_REDIRECT(f'Seeding Gear from {version}'))
35+
36+
# Store the version for the file in the DB
37+
try:
38+
models.XIVVersion.objects.create(version=version)
39+
except IntegrityError:
40+
pass
41+
42+
with open(file.path, 'r') as f:
43+
self.import_file(f, models.Gear)
44+
45+
# Lastly we import the Job data.
46+
# This is handled *slightly* differently because the 'ordering' key in this file will most likely change
47+
# between expansions, especially for dps
48+
# So this Integrity Error will be handled slightly differently
49+
50+
with open(seed_data_dir / 'jobs.yml', 'r') as f:
51+
self.stdout.write(self.style.HTTP_REDIRECT('Seeding Jobs'))
52+
self.import_jobs(f)
53+
54+
def import_file(self, file, model):
55+
data = yaml.safe_load(file)
56+
for item in data:
57+
self.stdout.write(f'\t{item["name"]}')
58+
_, created = model.objects.get_or_create(**item)
59+
if not created:
60+
self.stdout.write('\t\tSkipping, as it is already in the DB.')
61+
62+
def import_jobs(self, file):
63+
"""
64+
Import Job data.
65+
If Job exists, ensure the ordering value is up to date
66+
"""
67+
data = yaml.safe_load(file)
68+
for job in data:
69+
self.stdout.write(f'\t{job["id"]}')
70+
71+
# Check if the Job is already in the Database
72+
try:
73+
obj = models.Job.objects.get(pk=job['id'])
74+
self.stdout.write(
75+
f'\t\tAlready exists, ensuring correct ordering ({obj.ordering} -> {job["ordering"]})',
76+
)
77+
obj.ordering = job['ordering']
78+
obj.save()
79+
except models.Job.DoesNotExist:
80+
# If it doesn't exist, just create it!
81+
models.Job.objects.create(**job)

backend/api/tasks.py

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
"""
2+
Set up our tasks for celery to run
3+
4+
Task to verify accounts on XIVAPI.
5+
"""
6+
# stdlib
7+
from datetime import timedelta
8+
# lib
9+
from asgiref.sync import async_to_sync
10+
from celery import shared_task
11+
from celery.utils.log import get_task_logger
12+
from channels.layers import get_channel_layer
13+
from django.core.management import call_command
14+
from django.db.models import Q
15+
from django.utils import timezone
16+
# local
17+
from . import notifier
18+
from .lodestone_scraper import LodestoneScraper
19+
from .models import Character, Notification, Team
20+
21+
logger = get_task_logger(__name__)
22+
23+
24+
def assimilate_proxies(real_char: Character):
25+
# Find all Proxy characters that have the same lodestone ID as this one
26+
proxies = Character.objects.filter(
27+
Q(user__isnull=True) | Q(user_id=real_char.user_id),
28+
lodestone_id=real_char.lodestone_id,
29+
)
30+
31+
# For each Character (which should only ever be in one team each);
32+
# - Notify the Team Leader that the claim has happened
33+
# - Move the BIS List to the real Character, name it using the Team's name
34+
# - Update the TeamMember object to point to this character
35+
for char in proxies:
36+
for tm in char.teammember_set.all():
37+
if char.user is None:
38+
notifier.team_proxy_claim(tm)
39+
40+
bis = tm.bis_list
41+
bis.owner = real_char
42+
bis.name = f'BIS From {tm.team.name}'
43+
bis.save()
44+
45+
tm.character = real_char
46+
tm.save()
47+
48+
49+
@shared_task(name='verify_character')
50+
def verify_character(pk: int):
51+
"""
52+
Verify the character has the expected token in the bio on xivapi.
53+
54+
If so, update the flag to True, and delete all other unverified characters with the same lodestone id
55+
"""
56+
# Check that the character is unverified and exists
57+
logger.info(f'Commencing verification attempt for Character #{pk}.')
58+
try:
59+
obj = Character.objects.get(pk=pk, verified=False)
60+
except Character.DoesNotExist:
61+
logger.warn(f'Character #{pk} either does not exist or is verified. Exiting.')
62+
return
63+
64+
# Call the xivapi function in a sync context
65+
logger.debug('calling lookup function')
66+
err = LodestoneScraper.get_instance().check_token(obj.lodestone_id, obj.token)
67+
logger.debug('finished lookup function')
68+
69+
if err is not None:
70+
notifier.verify_fail(obj, err)
71+
logger.info(f'Character #{pk} could not be verified. Exiting. ({err})')
72+
return
73+
74+
logger.info(f'Character #{pk} verified. Updating DB.')
75+
# First we update the flag on the object specified
76+
obj.verified = True
77+
obj.save()
78+
79+
# Before we go deleting any Characters, we need to sort all the Proxies that share the lodestone ID
80+
assimilate_proxies(obj)
81+
82+
# Next delete all unverified instances of the character (this includes proxies)
83+
logger.info(f'Deleting unverified instances of Character #{obj.lodestone_id} (#{pk}) owned by {obj.user_id}.')
84+
objs = Character.objects.filter(
85+
Q(user__isnull=True) | Q(user_id=obj.user_id),
86+
verified=False,
87+
lodestone_id=obj.lodestone_id,
88+
).exclude(pk=pk)
89+
ids_to_delete = [o.pk for o in objs]
90+
logger.info(f'Found {objs.count()} instances of Character #{obj.lodestone_id} to delete.\n{ids_to_delete}')
91+
objs.delete()
92+
# Then we're done!
93+
notifier.verify_success(obj)
94+
# Also send websocket details
95+
channel_layer = get_channel_layer()
96+
if channel_layer is not None:
97+
async_to_sync(channel_layer.group_send)(f'user-updates-{obj.user.id}', {'type': 'character', 'id': obj.pk})
98+
99+
100+
@shared_task(name='verify_reminder')
101+
def remind_users_to_verify():
102+
"""
103+
Find non-verified Characters that are 5 days old.
104+
Send Notifications to remind the User to verify.
105+
"""
106+
logger.debug(f'Running at: {timezone.now()}')
107+
older_than = timezone.now() - timedelta(days=5)
108+
logger.debug(f'Reminding unverified characters older than {older_than}.')
109+
110+
characters = Character.objects.filter(verified=False, user__isnull=False, created__lt=older_than)
111+
logger.debug(f'Found {characters.count()} characters. Reminding their Users.')
112+
for char in characters:
113+
# Check that there wasn't already a reminder sent about this Character
114+
if not Notification.objects.filter(type='verify_reminder', link=f'/characters/{char.id}/').exists():
115+
notifier.verify_reminder(char)
116+
117+
118+
@shared_task(name='cleanup')
119+
def cleanup():
120+
"""
121+
Cleanup the DB of all unverified (non-proxy) characters made more than 7 days ago
122+
"""
123+
logger.debug(f'Running at: {timezone.now()}')
124+
older_than = timezone.now() - timedelta(days=7)
125+
logger.debug(f'Deleting unverified characters older than {older_than}.')
126+
127+
objs = Character.objects.filter(verified=False, user__isnull=False, created__lt=older_than)
128+
logger.debug(f'Found {objs.count()} characters. Deleting them.')
129+
for char in objs:
130+
# Remove them from every team they are a member of
131+
teams = Team.objects.filter(members__character=char).distinct()
132+
for team in teams:
133+
team.remove_character(char, False)
134+
135+
char.bis_lists.all().delete()
136+
char.delete()
137+
138+
139+
@shared_task(name='refresh_tokens')
140+
def refresh_tokens():
141+
"""
142+
Refresh any tokens that are about to expire
143+
"""
144+
call_command('refresh_tokens')
145+
146+
147+
@shared_task(name='check_game_version')
148+
def check_game_version():
149+
"""
150+
Check if a new, unseeded, game version has been added to xivapi that we don't have yet
151+
"""
152+
call_command('check_game_version', '--latest', '--notify')

backend/api/tasks/__init__.py

Lines changed: 0 additions & 17 deletions
This file was deleted.

backend/api/tasks/base.py

Lines changed: 0 additions & 25 deletions
This file was deleted.

backend/api/tasks/check_game_version.py

Lines changed: 0 additions & 12 deletions
This file was deleted.

backend/api/tasks/db_cleanup.py

Lines changed: 0 additions & 32 deletions
This file was deleted.

backend/api/tasks/refresh_tokens.py

Lines changed: 0 additions & 12 deletions
This file was deleted.

0 commit comments

Comments
 (0)