Skip to content

Commit 6dbaae9

Browse files
committed
test23
1 parent 342d6cf commit 6dbaae9

225 files changed

Lines changed: 57193 additions & 465 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -162,10 +162,8 @@ cython_debug/
162162

163163
**/settings/secret.py
164164

165-
# Pycharm tailwind autocomplete workaround folder
166-
167-
pyc-autocomplete/
168-
169165
/docker/
170166
**/docker/tailwindcss
171167
**/tailwind/build.css
168+
169+
**/media/

.pre-commit-config.yaml

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,18 +28,27 @@ repos:
2828
types: [markdown]
2929
additional_dependencies:
3030
- mdformat-myst
31-
- repo: https://github.com/biomejs/pre-commit
32-
rev: v0.6.1
33-
hooks:
34-
- id: biome-check
35-
additional_dependencies: ["@biomejs/biome@1.9.2"]
31+
3632
- repo: local
3733
hooks:
38-
- id: rustywind # Remove rustywind once biomejs tailwind sorting is competent: https://github.com/biomejs/biome/issues/1274
39-
name: Tailwind CSS Class sorter
34+
- id: rustywind
35+
name: rustywind Tailwind CSS class linter # The Prettier tailwind plugin doesn't like Django templates
4036
language: node
4137
additional_dependencies:
4238
- rustywind@latest
4339
entry: rustywind
44-
args: [--write]
45-
types_or: [html]
40+
args: ["--write"]
41+
types_or: ["html"]
42+
43+
- repo: https://github.com/JoC0de/pre-commit-prettier
44+
on: push # Prettier can be slow (5-10s) so only run on push to not be annoying every commit
45+
# We need to run prettier-tailwind and prettier-jinja separately since prettier-jinja overrides tailwind
46+
hooks:
47+
- id: prettier-jinja
48+
name: Prettier + Jinja/DTL 🥳 (push)
49+
language: node
50+
additional_dependencies:
51+
- prettier@3.6.2
52+
- prettier-plugin-jinja-template@2.1.0
53+
entry: prettier
54+
args: ["--write", "--plugin=prettier-plugin-jinja-template"]

.prettierignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Ignore artifacts:
2+
build
3+
coverage
4+
input.css
5+
build.css

.prettierrc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{}

biome.json

Lines changed: 0 additions & 20 deletions
This file was deleted.

dev/docker/compose.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ services:
3030
- watchfiles
3131
- --filter
3232
- python
33-
- "uv run celery -A director worker"
33+
- "uv run celery -A director worker --concurrency=4"
3434
- /director5/manager/director
3535
working_dir: /director5/manager
3636
networks:
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from django.urls import path
2+
3+
from director.apps.sites.consumers.site_status import SiteStatusConsumer
4+
5+
websocket_urlpatterns = [
6+
path("sites/<int:site_id>/websocket/status", SiteStatusConsumer.as_asgi()),
7+
]
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import logging
2+
from typing import Literal
3+
4+
from asgiref.sync import async_to_sync, sync_to_async
5+
from channels.db import database_sync_to_async
6+
from channels.layers import get_channel_layer
7+
from django.template.loader import render_to_string
8+
9+
from director.apps.sites.models import Site, Operation
10+
from director.apps.users.models import User
11+
from redis import asyncio as aioredis
12+
13+
from channels.generic.websocket import AsyncWebsocketConsumer
14+
import json
15+
16+
from director.settings import REDIS_URL
17+
18+
logger = logging.getLogger(__name__)
19+
20+
_redis = None
21+
22+
async def _get_redis() -> aioredis.Redis:
23+
"""
24+
Returns:
25+
An :class:`aioredis.Redis` instance, and avoids making a new connection if one already exists
26+
This helps save resources
27+
"""
28+
global _redis
29+
if _redis is None:
30+
_redis = await aioredis.from_url(REDIS_URL)
31+
return _redis
32+
33+
34+
class SiteStatusConsumer(AsyncWebsocketConsumer):
35+
async def connect(self):
36+
self.site_id = self.scope["url_route"]["kwargs"]["site_id"]
37+
self.group_name = f"site_{self.site_id}"
38+
self.redis = await _get_redis()
39+
40+
self.user = self.scope["user"]
41+
if not self.user.is_authenticated:
42+
await self.close()
43+
return
44+
45+
try:
46+
if self.user.is_superuser:
47+
self.site = await Site.objects.aget(id=self.site_id)
48+
else:
49+
self.site = await Site.objects.aget(id=self.site_id, users=self.user)
50+
except Site.DoesNotExist:
51+
await self.close()
52+
return
53+
54+
await self.channel_layer.group_add(self.group_name, self.channel_name)
55+
await self.accept()
56+
57+
# Add this user to a redis set of active users
58+
await self.redis.sadd(self.group_name, self.user.id)
59+
# Let all connected user's know this user is now connected
60+
await self.channel_layer.group_send(
61+
self.group_name,
62+
{"type": "send_active_users"}
63+
)
64+
65+
async def disconnect(self, close_code):
66+
await self.channel_layer.group_discard(self.group_name, self.channel_name)
67+
68+
# Remove this user from the redis set of active users
69+
await self.redis.srem(self.group_name, self.user.id)
70+
# Make sure other users know this user's connection is going away
71+
await self.channel_layer.group_send(
72+
self.group_name,
73+
{"type": "send_active_users"}
74+
)
75+
76+
async def send_power_event(self, event):
77+
power_event_template = render_to_string("base_with_site_header_and_nav.html.partials/status_indicator.html", {"site": self.site, "status": event["status"]})
78+
await self.send(text_data=power_event_template.strip())
79+
80+
async def send_operation_updated_event(self, event):
81+
operation = await Operation.objects.filter(site=self.site).afirst()
82+
power_event_template = await sync_to_async(render_to_string)("site_dashboard.html.partials/operations_and_process_log.html", {"operation": operation})
83+
await self.send(text_data=power_event_template.strip())
84+
85+
async def send_process_log_data_event(self, event):
86+
pass
87+
88+
async def send_active_users(self, event):
89+
members = await self.redis.smembers(self.group_name)
90+
users = []
91+
for member_id in members:
92+
try:
93+
user = await User.objects.aget(id=member_id.decode("utf-8"))
94+
users.append(user)
95+
except User.DoesNotExist:
96+
logger.exception("Non critical error: a user does not exist when parsing the redis active users set"
97+
f"User id was {member_id}, group was {self.group_name}")
98+
99+
active_users_template = render_to_string("base_with_site_header_and_nav.html.partials/active_users.html", {"active_users": users, "user": self.user})
100+
await self.send(text_data=active_users_template.strip())
101+
102+
def trigger_power_event(site: Site, status: Literal["Online", "Offline", "Restarting", "Starting", "Stopping"]):
103+
async_to_sync(get_channel_layer().group_send)(
104+
f"site_{site.id}",
105+
{
106+
"type": "send_power_event",
107+
"status": status,
108+
}
109+
)
110+
111+
def trigger_operation_updated_event(site: Site):
112+
async_to_sync(get_channel_layer().group_send)(
113+
f"site_{site.id}",
114+
{
115+
"type": "send_operation_updated_event",
116+
}
117+
)
118+
119+
def trigger_process_log_data_event(site: Site):
120+
async_to_sync(get_channel_layer().group_send)(
121+
f"site_{site.id}",
122+
{
123+
"type": "send_operation_updated_event",
124+
}
125+
)

manager/director/apps/sites/models.py

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ class Availabilities(models.TextChoices):
119119

120120
id: int
121121
domain_set: models.QuerySet["Domain"]
122+
operation: Operation
122123

123124
def __str__(self):
124125
return self.name
@@ -133,11 +134,16 @@ def sites_url(self) -> str:
133134
default = settings.SITE_URL_FORMATS[None]
134135
return settings.SITE_URL_FORMATS.get(self.purpose, default).format(self.name)
135136

137+
@property
138+
def channels_group_name(self) -> str:
139+
"""Return the group name used to specify differentiate on a site-by-site basis in a consumer"""
140+
return f"site_{self.id}"
141+
136142
def start_operation(self, ty: str) -> "Operation":
137143
from . import operations
138144

139145
op = Operation.objects.create(site=self, ty=ty)
140-
operations.send_operation_updated_message(self)
146+
operations.trigger_operation_updated_event(self)
141147
return op
142148

143149
def list_domains(self) -> list[str]:
@@ -208,7 +214,7 @@ class Database(models.Model):
208214
host = models.ForeignKey(DatabaseHost, on_delete=models.CASCADE)
209215
password = models.CharField(max_length=255, null=False, blank=False)
210216

211-
site: Site
217+
site: Site # reverse OneToOneField from Site.database, this is automatically done by Django, we just type annotate it to be clearer
212218

213219
def __str__(self) -> str:
214220
return self.redacted_db_url
@@ -219,7 +225,13 @@ def username(self) -> str:
219225

220226
@property
221227
def redacted_db_url(self) -> str:
222-
return f"{self.host.dbms}://{self.username}:***@{self.host.hostname}:{self.host.port}/{self.username}"
228+
# We check if the reverse relation to a Site has been created, if not, don't use the variables
229+
# or else anything relying on Database's __str__ method will fail, i.e. django admin, since this database is not always
230+
# guaranteed to have a reverse relation
231+
if hasattr(self, "site"):
232+
return f"{self.host.dbms}://{self.username}:***@{self.host.hostname}:{self.host.port}/{self.username}"
233+
else:
234+
return "No site linked to this database (database url does not exist)"
223235

224236
def serialize_for_appserver(self) -> dict[str, str]:
225237
return {
@@ -327,12 +339,21 @@ class Operation(models.Model):
327339
started_time = models.DateTimeField(null=True)
328340

329341
def __str__(self) -> str:
330-
return f"{type(self).__name__}: {self.ty}"
342+
return f"{type(self).__name__}: {self.ty} - {self.site.name}"
331343

332344
@property
333345
def has_started(self) -> bool:
334346
return self.started_time is not None
335347

348+
@property
349+
def progress(self) -> float:
350+
try:
351+
return round((1 - self.action_set.filter(result=None).count() / self.action_set.count()) * 100)
352+
except ZeroDivisionError:
353+
return 100.0
354+
355+
def list_actions_in_order(self) -> models.QuerySet[Action]:
356+
return self.action_set.order_by("id")
336357

337358
class Action(models.Model):
338359
"""An individual task in an operation.
@@ -390,6 +411,10 @@ class Action(models.Model):
390411
def __str__(self) -> str:
391412
return f"{type(self).__name__}: {self.name}"
392413

414+
@property
415+
def has_started(self) -> bool:
416+
return self.started_time is not None
417+
393418
def start_action(self) -> None:
394419
self.started_time = timezone.localtime()
395420
self.save(update_fields=["started_time"])

manager/director/apps/sites/operations.py

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@
44
from functools import wraps
55
from typing import overload
66

7+
from asgiref.sync import async_to_sync
8+
from channels.layers import get_channel_layer
9+
710
from .appserver import Appserver
11+
from .consumers.site_status import trigger_operation_updated_event
812
from .models import Action, Operation, Site
913

1014
type ActionCallback = Callable[[Site, list[Appserver]], Iterator[str]]
@@ -154,28 +158,20 @@ def auto_run_operation_wrapper(operation_id: int) -> Iterator[OperationWrapper]:
154158
#. Runs the :class:`OperationWrapper` with the given scope when the with statement has finished.
155159
#. Deletes the :class:`.Operation` if it was successful.
156160
"""
157-
operation = Operation.objects.get(id=operation_id)
161+
operation = Operation.objects.select_related("site").get(id=operation_id)
158162
wrapper = OperationWrapper(operation)
159163

160164
yield wrapper
161165

162-
send_operation_updated_message(operation.site)
166+
trigger_operation_updated_event(operation.site)
163167

164168
def action_started(_: Action) -> None:
165-
send_operation_updated_message(operation.site)
169+
trigger_operation_updated_event(operation.site)
166170

167171
result = wrapper.execute_operation(new_action_callback=action_started)
168172

169173
if result:
170174
operation.action_set.all().delete()
171175
operation.delete()
172176

173-
send_operation_updated_message(operation.site)
174-
175-
176-
def send_operation_updated_message(_site: Site) -> None:
177-
pass
178-
179-
180-
def send_site_updated_message(_site: Site) -> None:
181-
pass
177+
trigger_operation_updated_event(operation.site)

0 commit comments

Comments
 (0)