Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Catalog: Optimize Querying and Restrict Beta Packages to DSM < 7.0 #145

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Catalog: Optimize Querying and Restrict Beta Packages to DSM < 7.0 #145

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1b78b63
Optimise queries for major version
mreid-tt Feb 21, 2025
3b1a574
Limit betas to < DSM 7.0
mreid-tt Feb 21, 2025
0f16063
Optimize catalog generation and improve stability
mreid-tt Feb 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
227 changes: 123 additions & 104 deletions spkrepo/views/nas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,55 +43,41 @@ def is_valid_language(language):


@cache.memoize(timeout=600)
def get_catalog(arch, build, language, beta):
# Find the closest matching firmware for the provided build
closest_firmware = (
Firmware.query.filter(Firmware.build <= build, Firmware.type == "dsm")
.order_by(Firmware.build.desc())
.first()
)

# Extract major version from the closest matching firmware
major_version = (
int(closest_firmware.version.split(".")[0])
if closest_firmware and closest_firmware.version
else None
)

# latest version per package and major version
def get_catalog(arch, build, major, language, beta):
# Step 1: Get the latest version for each package
latest_version = db.session.query(
Version.package_id, db.func.max(Version.version).label("latest_version")
).select_from(Version)

if not beta:
latest_version = latest_version.filter(
db.or_(Version.report_url.is_(None), Version.report_url == "")
db.or_(
Version.report_url.is_(None), Version.report_url == ""
) # Exclude beta
)

latest_version = (
latest_version.join(Build)
.filter(Build.active)
.join(Build.architectures)
.filter(Architecture.code.in_(["noarch", arch]))
.filter(db.or_(Architecture.code == arch, Architecture.code == "noarch"))
.join(Build.firmware)
.filter(Firmware.build <= build)
.filter(
db.or_(
# Check if major_version is not None before applying the filter
(major_version is not None)
and Firmware.version.startswith(f"{major_version}."),
# Include earlier "noarch" version when major_version < 6
Firmware.version.startswith(f"{major}."),
db.and_(
Architecture.code == "noarch",
(major_version is not None) and (major_version < 6),
major < 6,
Firmware.version.startswith("3."),
),
)
)
.group_by(Version.package_id)
).subquery()
.subquery()
)

# latest firmware on given version
# Step 2: Get the latest firmware for each version
latest_firmware = (
db.session.query(
Version.package_id,
Expand All @@ -102,7 +88,7 @@ def get_catalog(arch, build, language, beta):
.join(Build)
.filter(Build.active)
.join(Build.architectures)
.filter(Architecture.code.in_(["noarch", arch]))
.filter(db.or_(Architecture.code == arch, Architecture.code == "noarch"))
.join(Build.firmware)
.filter(Firmware.build <= build)
.join(
Expand All @@ -113,9 +99,10 @@ def get_catalog(arch, build, language, beta):
),
)
.group_by(Version.package_id, latest_version.c.latest_version)
).subquery()
.subquery()
)

# build on given version and firmware
# Step 3: Get the latest builds for versions
latest_build = (
Build.query.options(
db.joinedload(Build.architectures),
Expand All @@ -130,7 +117,7 @@ def get_catalog(arch, build, language, beta):
db.joinedload(Build.version, Version.package, Package.screenshots),
)
.join(Build.architectures)
.filter(Architecture.code.in_(["noarch", arch]))
.filter(db.or_(Architecture.code == arch, Architecture.code == "noarch"))
.join(Firmware)
.join(Version)
.join(
Expand All @@ -143,85 +130,96 @@ def get_catalog(arch, build, language, beta):
)
)

# fill the catalog
entries = []
for b in latest_build.all():
entry = {
"package": b.version.package.name,
"version": b.version.version_string,
"dname": b.version.displaynames.get(
language, b.version.displaynames["enu"]
).displayname,
"desc": b.version.descriptions.get(
language, b.version.descriptions["enu"]
).description,
"link": url_for(
".data", path=b.path, arch=arch, build=build, _external=True
),
"thumbnail": [
url_for(".data", path=icon.path, _external=True)
for icon in b.version.icons.values()
],
"qinst": b.version.license is None and b.version.install_wizard is False,
"qupgrade": b.version.license is None and b.version.upgrade_wizard is False,
"qstart": (
b.version.license is None
and b.version.install_wizard is False
and b.version.startable is not False
),
"deppkgs": b.version.dependencies,
"conflictpkgs": b.version.conflicts,
"download_count": b.version.package.download_count,
"recent_download_count": b.version.package.recent_download_count,
}

if b.version.package.screenshots:
entry["snapshot"] = [
url_for(".data", path=screenshot.path, _external=True)
for screenshot in b.version.package.screenshots
]
if b.version.report_url:
entry["report_url"] = b.version.report_url
entry["beta"] = True
if b.version.changelog:
entry["changelog"] = b.version.changelog
if b.version.distributor:
entry["distributor"] = b.version.distributor
if b.version.distributor_url:
entry["distributor_url"] = b.version.distributor_url
if b.version.maintainer:
entry["maintainer"] = b.version.maintainer
if b.version.maintainer_url:
entry["maintainer_url"] = b.version.maintainer_url
if b.version.service_dependencies:
entry["depsers"] = " ".join(
[service.code for service in b.version.service_dependencies]
)
if b.md5:
entry["md5"] = b.md5
if b.version.conf_dependencies:
entry["conf_deppkgs"] = b.version.conf_dependencies
if b.version.conf_conflicts:
entry["conf_conxpkgs"] = b.version.conf_conflicts
if b.version.conf_privilege:
entry["conf_privilege"] = b.version.conf_privilege
if b.version.conf_resource:
entry["conf_resource"] = b.version.conf_resource
entries.append(entry)
# Step 4: Construct response with "packages"
packages = []
for b in latest_build:
packages.append(build_package_entry(b, language, arch, build))

# DSM 5.1
if build >= 5004:
keyrings = []
if current_app.config["GNUPG_PATH"] is not None: # pragma: no cover
gpg = gnupg.GPG(gnupghome=current_app.config["GNUPG_PATH"])
return {
"packages": entries,
"keyrings": [
gpg.export_keys(current_app.config["GNUPG_FINGERPRINT"]).strip()
],
}
return {"packages": entries, "keyrings": []}
keyrings.append(
gpg.export_keys(current_app.config["GNUPG_FINGERPRINT"]).strip()
)

return {
"packages": packages,
"keyrings": keyrings,
}

return packages

return entries

def build_package_entry(b, language, arch, build):
entry = {
"package": b.version.package.name,
"version": b.version.version_string,
"dname": b.version.displaynames.get(
language, b.version.displaynames["enu"]
).displayname,
"desc": b.version.descriptions.get(
language, b.version.descriptions["enu"]
).description,
"link": url_for(
".data",
path=b.path,
arch=arch,
build=build,
_external=True,
),
"thumbnail": [
url_for(".data", path=icon.path, _external=True)
for icon in b.version.icons.values()
],
"qinst": b.version.license is None and b.version.install_wizard is False,
"qupgrade": b.version.license is None and b.version.upgrade_wizard is False,
"qstart": (
b.version.license is None
and b.version.install_wizard is False
and b.version.startable is not False
),
"deppkgs": b.version.dependencies,
"conflictpkgs": b.version.conflicts,
"download_count": b.version.package.download_count,
"recent_download_count": b.version.package.recent_download_count,
}

if b.version.package.screenshots:
entry["snapshot"] = [
url_for(".data", path=screenshot.path, _external=True)
for screenshot in b.version.package.screenshots
]
if b.version.report_url:
entry["report_url"] = b.version.report_url
entry["beta"] = True
if b.version.changelog:
entry["changelog"] = b.version.changelog
if b.version.distributor:
entry["distributor"] = b.version.distributor
if b.version.distributor_url:
entry["distributor_url"] = b.version.distributor_url
if b.version.maintainer:
entry["maintainer"] = b.version.maintainer
if b.version.maintainer_url:
entry["maintainer_url"] = b.version.maintainer_url
if b.version.service_dependencies:
entry["depsers"] = " ".join(
[service.code for service in b.version.service_dependencies]
)
if b.md5:
entry["md5"] = b.md5
if b.version.conf_dependencies:
entry["conf_deppkgs"] = b.version.conf_dependencies
if b.version.conf_conflicts:
entry["conf_conxpkgs"] = b.version.conf_conflicts
if b.version.conf_privilege:
entry["conf_privilege"] = b.version.conf_privilege
if b.version.conf_resource:
entry["conf_resource"] = b.version.conf_resource

return entry


@nas.route("/", methods=["POST", "GET"])
Expand All @@ -245,10 +243,31 @@ def catalog():
build = int(request.values["build"])
except ValueError:
abort(422)
beta = request.values.get("package_update_channel") == "beta"
# DSM 7.0
if build < 40000:
beta = request.values.get("package_update_channel") == "beta"
else:
beta = False # Ensure no beta packages are returned for DSM 7+
# Check if "major" is provided
if "major" in request.values:
try:
major = int(request.values["major"]) # Use provided major version
except ValueError:
abort(422)
else:
# Find major version from firmware table (if not provided)
closest_firmware = (
Firmware.query.filter(Firmware.build <= build, Firmware.type == "dsm")
.order_by(Firmware.build.desc())
.first()
)
if not closest_firmware or not closest_firmware.version:
abort(422)
# Extract major version from firmware.version (e.g., "7.2" → "7")
major = int(closest_firmware.version.split(".")[0])

# get the catalog
catalog = get_catalog(arch, build, language, beta)
catalog = get_catalog(arch, build, major, language, beta)

return Response(json.dumps(catalog), mimetype="application/json")

Expand Down