Skip to content

Commit da93d07

Browse files
filips123filips123
authored
Merge pull request #100 from filips123/backend-improvements
Implement some backend improvements
2 parents 35cc44d + 5a0ee05 commit da93d07

17 files changed

+1021
-929
lines changed

‎.github/workflows/api.yaml

+14-13
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,12 @@ jobs:
2424

2525
steps:
2626
- name: Checkout repository
27-
uses: actions/checkout@v3
27+
uses: actions/checkout@v4
2828

2929
- name: Configure Poetry cache
30-
uses: actions/cache@v3
30+
uses: actions/cache@v4
3131
with:
3232
path: |
33-
~/.cache/black
3433
~/.cache/pip
3534
~/.cache/pypoetry
3635
~/.local/share/pypoetry
@@ -40,9 +39,9 @@ jobs:
4039
restore-keys: ${{ runner.os }}-poetry-lint-
4140

4241
- name: Install Python
43-
uses: actions/setup-python@v4
42+
uses: actions/setup-python@v5
4443
with:
45-
python-version: "3.11"
44+
python-version: "3.12"
4645

4746
- name: Install Poetry
4847
run: |
@@ -53,22 +52,24 @@ jobs:
5352
- name: Install dependencies
5453
run: poetry install --extras sentry
5554

56-
- name: Lint the project with ruff
57-
run: ruff gimvicurnik
55+
- name: Lint the project with ruff check
56+
if: always()
57+
run: ruff check --output-format=github
5858

59-
- name: Lint the project with black
60-
run: black gimvicurnik --check
59+
- name: Lint the project with ruff format
60+
if: always()
61+
run: ruff format --check
6162

6263
typecheck:
6364
name: Typechecking
6465
runs-on: ubuntu-latest
6566

6667
steps:
6768
- name: Checkout repository
68-
uses: actions/checkout@v3
69+
uses: actions/checkout@v4
6970

7071
- name: Configure Poetry cache
71-
uses: actions/cache@v3
72+
uses: actions/cache@v4
7273
with:
7374
path: |
7475
~/.cache/pip
@@ -80,9 +81,9 @@ jobs:
8081
restore-keys: ${{ runner.os }}-poetry-typecheck-
8182

8283
- name: Install Python
83-
uses: actions/setup-python@v4
84+
uses: actions/setup-python@v5
8485
with:
85-
python-version: "3.11"
86+
python-version: "3.12"
8687

8788
- name: Install Poetry
8889
run: |

‎.github/workflows/deploy.yaml

+4-2
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,16 @@ jobs:
3939
with:
4040
path: |
4141
~/.cache/yarn
42+
~/.yarn/berry/cache
4243
./website/node_modules/.cache
44+
./website/node_modules/.vite
4345
key: ${{ runner.os }}-yarn-deploy-${{ hashFiles('**/yarn.lock') }}
4446
restore-keys: ${{ runner.os }}-yarn-
4547

4648
- name: Install Python
47-
uses: actions/setup-python@v4
49+
uses: actions/setup-python@v5
4850
with:
49-
python-version: "3.11"
51+
python-version: "3.12"
5052

5153
- name: Install Poetry
5254
run: |

‎.github/workflows/website.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,11 @@ jobs:
5252
run: yarn install --immutable
5353

5454
- name: Lint the project with ESLint
55+
if: always()
5556
run: yarn lint
5657

5758
- name: Lint the project with Prettier
59+
if: always()
5860
run: yarn format
5961

6062
typecheck:

‎API/README.md

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
GimVičUrnik
2-
===========
1+
GimVičUrnik - API
2+
=================
33

44
An API for a school timetable, substitutions and menus at Gimnazija Vič.
55

@@ -58,10 +58,10 @@ You can retrieve all API routes using the `gimvicurnik routes` commands. The off
5858

5959
## Contributing
6060

61-
The API uses ruff, black and mypy for linting the code. They are included in project's development dependencies.
61+
The API uses ruff for linting and formatting the code, and mypy for typechecking. They are included in the project's development dependencies.
6262

6363
Please make sure that your changes are formatted correctly according to the code style:
6464

65-
* Linting: `ruff gimvicurnik`
65+
* Linting: `ruff check`
66+
* Formatting: `ruff format`
6667
* Typechecking: `mypy gimvicurnik`
67-
* Formatting: `black gimvicurnik`

‎API/config.yaml.sample

+3-3
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ sentry:
3030
requests: 1.0
3131
other: 1.0
3232
profilerSampleRate:
33-
commands: false
34-
requests: false
35-
other: false
33+
commands: false
34+
requests: false
35+
other: false
3636

3737
logging:
3838
version: 1

‎API/gimvicurnik/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ def _format_date(date: datetime.date) -> str:
235235
return date.strftime("%d. %m. %Y")
236236

237237
def _format_week(date: datetime.date) -> str:
238-
return f"{_format_date(date)} — {_format_date((date + datetime.timedelta(days=4)))}"
238+
return f"{_format_date(date)} \u2013 {_format_date(date + datetime.timedelta(days=4))}"
239239

240240
filters = self.app.jinja_env.filters
241241
filters["date"] = _format_date

‎API/gimvicurnik/__main__.py

+4
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ def _get_version(ctx: click.Context, _param: str, value: str) -> None:
4141
flask_version = metadata.version("flask")
4242
werkzeug_version = metadata.version("werkzeug")
4343
pdfplumber_version = metadata.version("pdfplumber")
44+
pdfminer_version = metadata.version("pdfminer.six")
4445
openpyxl_version = metadata.version("openpyxl")
46+
mammoth_version = metadata.version("mammoth")
4547

4648
try:
4749
sentry_version = metadata.version("sentry-sdk")
@@ -56,7 +58,9 @@ def _get_version(ctx: click.Context, _param: str, value: str) -> None:
5658
f"Flask: {flask_version}\n"
5759
f"Werkzeug: {werkzeug_version}\n"
5860
f"pdfplumber: {pdfplumber_version}\n"
61+
f"pdfminer: {pdfminer_version}\n"
5962
f"openpyxl: {openpyxl_version}\n"
63+
f"mammoth: {mammoth_version}\n"
6064
f"Sentry SDK: {sentry_version}",
6165
color=ctx.color,
6266
)

‎API/gimvicurnik/blueprints/calendar.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ def get_combined_calendar_for_classes(classes: list[str]) -> Response:
264264
Class.get_substitutions(None, classes),
265265
Class.get_lessons(classes),
266266
config.lessonTimes,
267-
f"Koledar - {', '.join(classes)} - Gimnazija Vič",
267+
f"Koledar \u2013 {', '.join(classes)} \u2013 Gimnazija Vič",
268268
config.urls.api + request.path,
269269
)
270270

@@ -274,7 +274,7 @@ def get_timetable_calendar_for_classes(classes: list[str]) -> Response:
274274
Class.get_substitutions(None, classes),
275275
Class.get_lessons(classes),
276276
config.lessonTimes,
277-
f"Urnik - {', '.join(classes)} - Gimnazija Vič",
277+
f"Urnik \u2013 {', '.join(classes)} \u2013 Gimnazija Vič",
278278
config.urls.api + request.path,
279279
include_substitutions=False,
280280
)
@@ -285,7 +285,7 @@ def get_substitutions_calendar_for_classes(classes: list[str]) -> Response:
285285
Class.get_substitutions(None, classes),
286286
Class.get_lessons(classes),
287287
config.lessonTimes,
288-
f"Nadomeščanja - {', '.join(classes)} - Gimnazija Vič",
288+
f"Nadomeščanja \u2013 {', '.join(classes)} \u2013 Gimnazija Vič",
289289
config.urls.api + request.path,
290290
include_timetable=False,
291291
)
@@ -297,6 +297,6 @@ def get_schedules_calendar_for_classes(classes: list[str]) -> Response:
297297
.join(Class)
298298
.filter(Class.name.in_(classes))
299299
.order_by(LunchSchedule.time, LunchSchedule.class_),
300-
f"Razporedi kosila - {', '.join(classes)} - Gimnazija Vič",
300+
f"Razporedi kosila \u2013 {', '.join(classes)} \u2013 Gimnazija Vič",
301301
config.urls.api + request.path,
302302
)

‎API/gimvicurnik/blueprints/feed.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ class DateDisplay(enum.Enum):
3838
def get_mime_type(url: str) -> str:
3939
"""Get MIME type for a few extensions we know documents use."""
4040

41-
if re.match(r"\.pdf(?:\?[\w=]*)?$", url):
41+
if re.search(r"\.pdf(?:\?[\w=]*)?$", url):
4242
return "application/pdf"
4343
if url.endswith(".docx"):
4444
return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
@@ -81,12 +81,16 @@ def _create_feed(
8181
last_updated = Session.query(func.max(Document.modified)).filter(query_filter).scalar()
8282
last_updated = last_updated or date.fromtimestamp(0)
8383

84+
# Get the frontend page based on the feed type
85+
feed_page = "circulars" if feed_type == FeedType.CIRCULARS else "sources"
86+
8487
# Render the feed from Atom/RSS template
8588
content = render_template(
8689
f"{feed_format.value}.xml",
8790
urls=config.urls,
8891
name=feed_name,
8992
type=feed_type.value,
93+
page=feed_page,
9094
entries=query,
9195
last_updated=last_updated,
9296
date_display=date_display,

‎API/gimvicurnik/database/__init__.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,8 @@ def get_empty(cls) -> Iterator[dict[str, Any]]:
186186
times = Session.query(func.min(Lesson.time), func.max(Lesson.time))[0]
187187

188188
if times[0] is None or times[1] is None:
189-
return []
189+
yield from ()
190+
return
190191

191192
classrooms = Session.query(Classroom.name).order_by(Classroom.name).distinct().all()
192193
occupied = set(Session.query(Lesson.day, Lesson.time, Classroom.name).join(Classroom).distinct())

‎API/gimvicurnik/templates/atom.xml

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
<?xml version="1.0" encoding="UTF-8" ?>
22

33
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:webfeeds="http://webfeeds.org/rss/1.0">
4-
<title>{{ name }} - Gimnazija Vič</title>
5-
<subtitle>{{ name }} - Gimnazija Vič</subtitle>
4+
<title>{{ name }} – Gimnazija Vič</title>
5+
<subtitle>{{ name }} – Gimnazija Vič</subtitle>
66
<id>{{ urls.api }}/feed/{{ type }}.atom</id>
77
<updated>{{ last_updated.strftime("%Y-%m-%dT%H:%M:%SZ") }}</updated>
88

99
<link href="{{ urls.api }}/feed/{{ type }}.atom" rel="self" type="application/atom+xml" />
1010
<link href="{{ urls.api }}/feed/{{ type }}.rss" rel="alternate" type="application/rss+xml" />
11-
<link href="{{ urls.website }}/documents" rel="alternate" type="text/html" />
11+
<link href="{{ urls.website }}/{{ page }}" rel="alternate" type="text/html" />
1212

1313
<icon>{{ urls.website }}/img/icons/android-chrome-192x192.png</icon>
1414
<webfeeds:icon>{{ urls.website }}/img/icons/android-chrome-192x192.png</webfeeds:icon>

‎API/gimvicurnik/templates/rss.xml

+6-6
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,18 @@
22

33
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:webfeeds="http://webfeeds.org/rss/1.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
44
<channel>
5-
<title>{{ name }} - Gimnazija Vič</title>
6-
<description>{{ name }} - Gimnazija Vič</description>
7-
<link>{{ urls.website }}/documents</link>
5+
<title>{{ name }} – Gimnazija Vič</title>
6+
<description>{{ name }} – Gimnazija Vič</description>
7+
<link>{{ urls.website }}/{{ page }}</link>
88
<lastBuildDate>{{ last_updated.strftime("%a, %d %b %Y %H:%M:%S GMT") }}</lastBuildDate>
99

1010
<atom:link href="{{ urls.api }}/feed/{{ type }}.rss" rel="self" type="application/rss+xml" />
1111
<atom:link href="{{ urls.api }}/feed/{{ type }}.atom" rel="alternate" type="application/atom+xml" />
12-
<atom:link href="{{ urls.website }}/documents" rel="alternate" type="text/html" />
12+
<atom:link href="{{ urls.website }}/{{ page }}" rel="alternate" type="text/html" />
1313

1414
<image>
15-
<title>{{ name }} - Gimnazija Vič</title>
16-
<link>{{ urls.website }}/documents</link>
15+
<title>{{ name }} – Gimnazija Vič</title>
16+
<link>{{ urls.website }}/{{ page }}</link>
1717
<url>{{ urls.website }}/img/icons/android-chrome-192x192.png</url>
1818
</image>
1919
<webfeeds:icon>{{ urls.website }}/img/icons/android-chrome-192x192.png</webfeeds:icon>

‎API/gimvicurnik/updaters/eclassroom.py

+21-4
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from itertools import product
1010
from urllib.parse import urlparse
1111

12-
from mammoth import convert_to_html # type: ignore
12+
import mammoth # type: ignore
1313
from openpyxl import load_workbook
1414
from sqlalchemy import insert
1515

@@ -37,7 +37,7 @@
3737
from typing import Any
3838
from collections.abc import Iterator
3939
from io import BytesIO
40-
from mammoth.documents import Image # type: ignore
40+
from mammoth.documents import Image, Hyperlink # type: ignore
4141
from sqlalchemy.orm import Session
4242
from sentry_sdk.tracing import Span
4343
from ..config import ConfigSourcesEClassroom
@@ -278,7 +278,13 @@ def document_needs_parsing(self, document: DocumentInfo) -> bool:
278278
return False
279279

280280
@with_span(op="parse", pass_span=True)
281-
def parse_document(self, document: DocumentInfo, stream: BytesIO, effective: date, span: Span) -> None: # type: ignore[override]
281+
def parse_document( # type: ignore[override]
282+
self,
283+
document: DocumentInfo,
284+
stream: BytesIO,
285+
effective: date,
286+
span: Span,
287+
) -> None:
282288
"""Parse the document and store extracted data."""
283289

284290
span.set_tag("document.source", self.source)
@@ -322,8 +328,19 @@ def extract_document(self, document: DocumentInfo, content: bytes, span: Span) -
322328
def ignore_images(_image: Image) -> dict:
323329
return {}
324330

331+
def transform_hyperlinks(hyperlink: Hyperlink) -> Hyperlink:
332+
hyperlink.target_frame = "_blank"
333+
return hyperlink
334+
325335
# Convert DOCX to HTML
326-
result = convert_to_html(content, convert_image=ignore_images)
336+
result = mammoth.convert_to_html(
337+
content,
338+
convert_image=ignore_images,
339+
transform_document=mammoth.transforms.element_of_type(
340+
mammoth.documents.Hyperlink,
341+
transform_hyperlinks,
342+
),
343+
)
327344
return typing.cast(str, result.value)
328345

329346
def _parse_substitutions_pdf(self, stream: BytesIO, effective: date) -> None:

‎API/gimvicurnik/updaters/menu.py

+9-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ def get_documents(self) -> Iterator[DocumentInfo]:
5555

5656
if not menus:
5757
self.logger.info("No menus found")
58-
return iter(())
58+
yield from ()
59+
return
5960

6061
for menu in menus:
6162
for link in menu.find_all("a", href=True):
@@ -115,7 +116,13 @@ def document_needs_parsing(self, document: DocumentInfo) -> bool:
115116
return True
116117

117118
@with_span(op="parse", pass_span=True)
118-
def parse_document(self, document: DocumentInfo, stream: BytesIO, effective: datetime.date, span: Span) -> None: # type: ignore[override]
119+
def parse_document( # type: ignore[override]
120+
self,
121+
document: DocumentInfo,
122+
stream: BytesIO,
123+
effective: datetime.date,
124+
span: Span,
125+
) -> None:
119126
"""Parse the document and store extracted data."""
120127

121128
span.set_tag("document.source", self.source)

‎API/gimvicurnik/updaters/solsis.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ def _download_substitutions(self, date: date_) -> TypeRoot:
322322
"""Download and parse the Solsis JSON file."""
323323

324324
# Every request needs a different nonsense
325-
nonsense = "%032x" % getrandbits(128)
325+
nonsense = f"{getrandbits(128):032x}"
326326

327327
# Compose the URL
328328
params = f"func=gateway&call=suplence&datum={date.strftime('%Y-%m-%d')}&nonsense={nonsense}"

0 commit comments

Comments
 (0)