From 0f8319f376c0763980cbb3aa935867ce862fd8f7 Mon Sep 17 00:00:00 2001 From: slashburygin Date: Fri, 10 Jul 2026 10:03:57 +0300 Subject: [PATCH] add quota --- AGENTS.md | 4 +- exordos_core/compute/dm/models.py | 3 + exordos_core/quota/__init__.py | 15 + exordos_core/quota/dm/__init__.py | 0 exordos_core/quota/dm/models.py | 202 +++++++++++ exordos_core/secret/dm/models.py | 5 + exordos_core/secret/service.py | 10 +- exordos_core/tests/functional/conftest.py | 33 ++ .../functional/restapi/quota/__init__.py | 15 + .../restapi/quota/test_quota_api.py | 199 +++++++++++ .../tests/functional/service/test_quota.py | 322 ++++++++++++++++++ .../unit/compute/pool/drivers/test_libvirt.py | 10 +- .../tests/unit/compute/test_models.py | 5 +- exordos_core/user_api/api/routes.py | 10 +- exordos_core/user_api/network/dm/models.py | 2 + exordos_core/user_api/quota/__init__.py | 15 + exordos_core/user_api/quota/api/__init__.py | 15 + .../user_api/quota/api/controllers.py | 69 ++++ exordos_core/user_api/quota/api/routes.py | 34 ++ migrations/0069-add-quota-tables-f8778e.py | 68 ++++ 20 files changed, 1020 insertions(+), 16 deletions(-) create mode 100644 exordos_core/quota/__init__.py create mode 100644 exordos_core/quota/dm/__init__.py create mode 100644 exordos_core/quota/dm/models.py create mode 100644 exordos_core/tests/functional/restapi/quota/__init__.py create mode 100644 exordos_core/tests/functional/restapi/quota/test_quota_api.py create mode 100644 exordos_core/tests/functional/service/test_quota.py create mode 100644 exordos_core/user_api/quota/__init__.py create mode 100644 exordos_core/user_api/quota/api/__init__.py create mode 100644 exordos_core/user_api/quota/api/controllers.py create mode 100644 exordos_core/user_api/quota/api/routes.py create mode 100644 migrations/0069-add-quota-tables-f8778e.py diff --git a/AGENTS.md b/AGENTS.md index 6f69c904..468ec761 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,10 +89,10 @@ exordos_core/ tox -e py310,py312,py314 # Run unit tests only -tox -e py310 +tox -e py314 # Run functional tests -tox -e py310-functional +tox -e py314-functional # Run linters tox -e ruff-check # Check code style diff --git a/exordos_core/compute/dm/models.py b/exordos_core/compute/dm/models.py index 06118918..f6aff45a 100644 --- a/exordos_core/compute/dm/models.py +++ b/exordos_core/compute/dm/models.py @@ -35,6 +35,7 @@ from exordos_core.common import utils from exordos_core.common.dm import models as cm from exordos_core.compute import constants as nc +from exordos_core.quota.dm.models import QuotaModelMixin if tp.TYPE_CHECKING: from exordos_core.compute.pool.drivers.base import AbstractPoolDriver @@ -301,6 +302,7 @@ class UnscheduledVolume(models.ModelWithUUID, orm.SQLStorableMixin): class NodeSet( infra_models.NodeSet, ua_models.InstanceWithDerivativesMixin, + QuotaModelMixin, orm.SQLStorableMixin, ): __tablename__ = "compute_sets" @@ -340,6 +342,7 @@ def set_active(self): class Node( infra_models.Node, + QuotaModelMixin, orm.SQLStorableWithJSONFieldsMixin, ): __tablename__ = "nodes" diff --git a/exordos_core/quota/__init__.py b/exordos_core/quota/__init__.py new file mode 100644 index 00000000..ba779e95 --- /dev/null +++ b/exordos_core/quota/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/exordos_core/quota/dm/__init__.py b/exordos_core/quota/dm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/exordos_core/quota/dm/models.py b/exordos_core/quota/dm/models.py new file mode 100644 index 00000000..178c758d --- /dev/null +++ b/exordos_core/quota/dm/models.py @@ -0,0 +1,202 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import importlib +import logging +import typing as tp +import uuid as sys_uuid + +from restalchemy.common import exceptions as ra_e +from restalchemy.dm import filters as dm_filters +from restalchemy.dm import models +from restalchemy.dm import properties +from restalchemy.dm import types +from restalchemy.storage.sql import orm + +LOG = logging.getLogger(__name__) + + +class QuotaExceededError(ra_e.ValidationErrorException): + message = "Quota exceeded for resource '%(resource_name)s' in project %(project_id)s: %(current)s > %(limit)s" + + def __init__( + self, resource_name: str, limit: int, current: int, project_id: sys_uuid.UUID + ): + super().__init__( + resource_name=resource_name, + limit=limit, + current=current, + project_id=project_id, + ) + self.resource_name = resource_name + self.limit = limit + self.current = current + self.project_id = project_id + + +DEFAULT_QUOTA_LIMIT = 1000 +DEFAULT_QUOTA_LIMITS: tp.Dict[str, int] = { + "net_lb": DEFAULT_QUOTA_LIMIT, + "compute_sets": DEFAULT_QUOTA_LIMIT, + "nodes": DEFAULT_QUOTA_LIMIT, + "secret_passwords": DEFAULT_QUOTA_LIMIT, + "secret_certificates": DEFAULT_QUOTA_LIMIT, + "secret_rsa_keys": DEFAULT_QUOTA_LIMIT, + "secret_ssh_keys": DEFAULT_QUOTA_LIMIT, +} +DEFAULT_QUOTA_FIELD_LIMITS: tp.Dict[str, tp.Dict[str, int]] = { + "nodes": {"cores": 10000}, +} +QUOTA_RESOURCE_MODELS = { + "net_lb": "exordos_core.user_api.network.dm.models:LB", + "compute_sets": "exordos_core.compute.dm.models:NodeSet", + "nodes": "exordos_core.compute.dm.models:Node", + "secret_passwords": "exordos_core.secret.dm.models:Password", + "secret_certificates": "exordos_core.secret.dm.models:Certificate", + "secret_rsa_keys": "exordos_core.secret.dm.models:RSAKey", + "secret_ssh_keys": "exordos_core.secret.dm.models:SSHKey", +} + + +def get_quota_resource_model(resource_name: str) -> type: + try: + module_name, class_name = QUOTA_RESOURCE_MODELS[resource_name].split(":") + except KeyError: + raise ValueError(f"Unknown quota resource: {resource_name}") + + module = importlib.import_module(module_name) + return getattr(module, class_name) + + +class QuotaModelMixin: + def _quota_limits(self, session) -> tp.Collection["QuotaLimit"]: + limits = list( + QuotaLimit.objects.get_all( + session=session, + filters={ + "project_id": dm_filters.EQ(self.project_id), + "resource_name": dm_filters.EQ(self.__tablename__), + }, + ) + ) + limit_fields = {limit.field_name for limit in limits} + if "" not in limit_fields: + default_limit = DEFAULT_QUOTA_LIMITS.get(self.__tablename__) + if default_limit is not None: + limits.append( + QuotaLimit( + project_id=self.project_id, + resource_name=self.__tablename__, + field_name="", + limit=default_limit, + ) + ) + + for field_name, default_limit in DEFAULT_QUOTA_FIELD_LIMITS.get( + self.__tablename__, {} + ).items(): + if field_name not in limit_fields: + limits.append( + QuotaLimit( + project_id=self.project_id, + resource_name=self.__tablename__, + field_name=field_name, + limit=default_limit, + ) + ) + return limits + + def _quota_check(self, session) -> None: + # Check entity-count and aggregate-field limits before inserting. + try: + limits = self._quota_limits(session) + except ValueError: + LOG.exception("Invalid quota configuration for %s", self.__tablename__) + return + + if not limits: + return + + field_limits = [limit for limit in limits if limit.field_name] + count_limits = [limit for limit in limits if not limit.field_name] + aggregate_fields = ", ".join( + f"SUM({field_name}) AS {field_name}" + for field_name in sorted({limit.field_name for limit in field_limits}) + ) + selected_fields = ", ".join( + field + for field in ("COUNT(uuid) AS entity_count", aggregate_fields) + if field + ) + result = session.execute( + f"SELECT {selected_fields} FROM {self.__tablename__} WHERE project_id = %s", + (self.project_id,), + ) + row = result.fetchone() + + for quota_limit in field_limits: + current = (row[quota_limit.field_name] or 0) + getattr( + self, quota_limit.field_name + ) + if current > quota_limit.limit: + raise QuotaExceededError( + resource_name=f"{self.__tablename__}.{quota_limit.field_name}", + limit=quota_limit.limit, + current=current, + project_id=self.project_id, + ) + + current_count = row["entity_count"] + 1 + for quota_limit in count_limits: + if current_count > quota_limit.limit: + raise QuotaExceededError( + resource_name=self.__tablename__, + limit=quota_limit.limit, + current=current_count, + project_id=self.project_id, + ) + + def insert(self, session=None): + # Reserve quota capacity by checking entity-count and field totals. + if session is None: + with self._get_engine().session_manager(session=session) as s: + self._quota_check(s) + super().insert(session=s) + else: + self._quota_check(session) + super().insert(session=session) + + +class QuotaLimit( + models.ModelWithUUID, + models.ModelWithTimestamp, + models.ModelWithProject, + orm.SQLStorableMixin, +): + __tablename__ = "quota_limits" + + resource_name = properties.property( + types.String(max_length=255), + required=True, + ) + field_name = properties.property( + types.String(max_length=255), + default="", + ) + limit = properties.property( + types.Integer(min_value=0), + required=True, + ) diff --git a/exordos_core/secret/dm/models.py b/exordos_core/secret/dm/models.py index 78a46a0a..518a692f 100644 --- a/exordos_core/secret/dm/models.py +++ b/exordos_core/secret/dm/models.py @@ -29,6 +29,7 @@ from exordos_core.common import constants as c from exordos_core.common.dm import models as cm from exordos_core.common.dm import targets as ct +from exordos_core.quota.dm.models import QuotaModelMixin from exordos_core.secret import constants as sc @@ -65,6 +66,7 @@ class Secret( class Password( Secret, + QuotaModelMixin, orm.SQLStorableMixin, ua_models.TargetResourceSQLStorableMixin, ): @@ -136,6 +138,7 @@ class DNSCoreCertificateMethod(AbstractCertificateMethod): class Certificate( Secret, + QuotaModelMixin, orm.SQLStorableWithJSONFieldsMixin, ua_models.TargetResourceSQLStorableMixin, ): @@ -217,6 +220,7 @@ def get_deleted_certificates( class RSAKey( Secret, + QuotaModelMixin, orm.SQLStorableMixin, ua_models.TargetResourceSQLStorableMixin, ): @@ -293,6 +297,7 @@ def get_resource_target_fields(self) -> tp.Set[str]: class SSHKey( Secret, + QuotaModelMixin, orm.SQLStorableMixin, ua_models.TargetResourceSQLStorableMixin, ): diff --git a/exordos_core/secret/service.py b/exordos_core/secret/service.py index 57fe141d..ba970e0f 100644 --- a/exordos_core/secret/service.py +++ b/exordos_core/secret/service.py @@ -22,6 +22,7 @@ from gcl_sdk.agents.universal.dm import models as ua_models from restalchemy.common import contexts from restalchemy.dm import filters as dm_filters +from restalchemy.storage import exceptions as ra_exceptions from exordos_core.common import constants as c from exordos_core.compute.dm import models as nm @@ -358,7 +359,14 @@ def _actualize_new_ssh_key( node=node.uuid, status=sc.SecretStatus.IN_PROGRESS, ) - key_host_resource.insert() + try: + key_host_resource.insert() + except ra_exceptions.ConflictRecords: + LOG.debug( + "SSH key resource %s for node %s already exists", + key_resource.uuid, + node.uuid, + ) key.status = sc.SecretStatus.IN_PROGRESS.value key.save() diff --git a/exordos_core/tests/functional/conftest.py b/exordos_core/tests/functional/conftest.py index 71360029..0f27927d 100644 --- a/exordos_core/tests/functional/conftest.py +++ b/exordos_core/tests/functional/conftest.py @@ -513,6 +513,39 @@ def factory( return factory +@pytest.fixture +def node_factory_with_model(): + def factory( + uuid: tp.Optional[sys_uuid.UUID] = None, + name: str = "node", + cores: int = 1, + ram: int = 1024, + image: str = "ubuntu_24.04", + project_id: sys_uuid.UUID = c.SERVICE_PROJECT_ID, + status: tp.Optional[str] = None, + **kwargs, + ) -> tp.Tuple[tp.Dict[str, tp.Any], node_models.Node]: + uuid = uuid or _make_uuid() + status_value = nc.NodeStatus.NEW.value if status is None else status.value + node = node_models.Node( + uuid=uuid, + name=name, + cores=cores, + ram=ram, + project_id=project_id, + status=status_value, + disk_spec=sdk_infra_models.RootDiskSpec(image=image), + **kwargs, + ) + view = node.dump_to_simple_view() + if status is None: + view.pop("status") + view.pop("node_set") + return view, node + + return factory + + @pytest.fixture def node_set_factory(): def factory( diff --git a/exordos_core/tests/functional/restapi/quota/__init__.py b/exordos_core/tests/functional/restapi/quota/__init__.py new file mode 100644 index 00000000..416aabcf --- /dev/null +++ b/exordos_core/tests/functional/restapi/quota/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/exordos_core/tests/functional/restapi/quota/test_quota_api.py b/exordos_core/tests/functional/restapi/quota/test_quota_api.py new file mode 100644 index 00000000..a50dd638 --- /dev/null +++ b/exordos_core/tests/functional/restapi/quota/test_quota_api.py @@ -0,0 +1,199 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import typing as tp +import uuid as sys_uuid + +from bazooka import exceptions as bazooka_exc +from gcl_iam.tests.functional import clients as iam_clients +import pytest + +from exordos_core.common import constants as c + +_API_PREFIX = ["quota"] + + +@pytest.fixture +def project_id(): + return sys_uuid.uuid4() + + +class TestQuotaLimitsUserApi: + @staticmethod + def _limit_cmp_shallow( + a: tp.Dict[str, tp.Any], + b: tp.Dict[str, tp.Any], + ) -> bool: + return all( + a.get(key, "") == b[key] + for key in ( + "uuid", + "project_id", + "resource_name", + "field_name", + "limit", + ) + ) + + def test_limits_list( + self, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + url = client.build_collection_uri(_API_PREFIX + ["limits"]) + + response = client.get(url) + + assert response.status_code == 200 + assert len(response.json()) == 0 + + def test_limits_add( + self, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + url = client.build_collection_uri(_API_PREFIX + ["limits"]) + + limit_data = { + "uuid": str(sys_uuid.uuid4()), + "project_id": str(c.SERVICE_PROJECT_ID), + "resource_name": "nodes", + "field_name": "cores", + "limit": 5, + } + response = client.post(url, json=limit_data) + output = response.json() + + assert response.status_code == 201 + assert self._limit_cmp_shallow(limit_data, output) + + url = client.build_resource_uri(_API_PREFIX + ["limits", output["uuid"]]) + client.delete(url) + + def test_limits_add_several( + self, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + url = client.build_collection_uri(_API_PREFIX + ["limits"]) + urls = [] + + for i, resource_name in enumerate( + ("net_lb", "compute_sets", "secret_passwords") + ): + limit_data = { + "uuid": str(sys_uuid.uuid4()), + "project_id": str(c.SERVICE_PROJECT_ID), + "resource_name": resource_name, + "limit": i + 1, + } + response = client.post(url, json=limit_data) + output = response.json() + + assert response.status_code == 201 + assert self._limit_cmp_shallow(limit_data, output) + urls.append( + client.build_resource_uri(_API_PREFIX + ["limits", output["uuid"]]) + ) + + for u in urls: + response = client.get(u) + assert response.status_code == 200 + + for u in urls: + client.delete(u) + + def test_limits_get( + self, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + url = client.build_collection_uri(_API_PREFIX + ["limits"]) + + limit_data = { + "uuid": str(sys_uuid.uuid4()), + "project_id": str(c.SERVICE_PROJECT_ID), + "resource_name": "net_lb", + "limit": 5, + } + response = client.post(url, json=limit_data) + output = response.json() + assert response.status_code == 201 + + url = client.build_resource_uri(_API_PREFIX + ["limits", output["uuid"]]) + response = client.get(url) + assert response.status_code == 200 + assert self._limit_cmp_shallow(limit_data, response.json()) + + client.delete(url) + + def test_limits_update( + self, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + url = client.build_collection_uri(_API_PREFIX + ["limits"]) + + limit_data = { + "uuid": str(sys_uuid.uuid4()), + "project_id": str(c.SERVICE_PROJECT_ID), + "resource_name": "net_lb", + "limit": 5, + } + response = client.post(url, json=limit_data) + output = response.json() + assert response.status_code == 201 + + update = {"limit": 10} + url = client.build_resource_uri(_API_PREFIX + ["limits", output["uuid"]]) + response = client.put(url, json=update) + output = response.json() + + assert response.status_code == 200 + assert output["limit"] == 10 + assert output["resource_name"] == "net_lb" + + client.delete(url) + + def test_limits_delete( + self, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + url = client.build_collection_uri(_API_PREFIX + ["limits"]) + + limit_data = { + "uuid": str(sys_uuid.uuid4()), + "project_id": str(c.SERVICE_PROJECT_ID), + "resource_name": "net_lb", + "limit": 5, + } + response = client.post(url, json=limit_data) + output = response.json() + assert response.status_code == 201 + + url = client.build_resource_uri(_API_PREFIX + ["limits", output["uuid"]]) + response = client.delete(url) + assert response.status_code == 204 + + with pytest.raises(bazooka_exc.NotFoundError): + client.get(url) diff --git a/exordos_core/tests/functional/service/test_quota.py b/exordos_core/tests/functional/service/test_quota.py new file mode 100644 index 00000000..8aa0a237 --- /dev/null +++ b/exordos_core/tests/functional/service/test_quota.py @@ -0,0 +1,322 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid as sys_uuid + +import pytest + +from exordos_core.common import constants as c +from exordos_core.quota.dm.models import QuotaExceededError +from exordos_core.quota.dm.models import QuotaLimit +from exordos_core.user_api.network.dm.models import LB +from exordos_core.user_api.quota.api.controllers import QuotaLimitController + + +@pytest.fixture +def project_id(): + return sys_uuid.uuid4() + + +_TABLENAME = "net_lb" + + +@pytest.fixture +def _quota_limit_2(user_api): + obj = QuotaLimit( + uuid=sys_uuid.uuid4(), + project_id=c.SERVICE_PROJECT_ID, + resource_name=_TABLENAME, + limit=2, + ) + obj.insert() + yield obj + try: + obj.delete() + except Exception: + pass + + +class TestQuotaNoLimit: + def test_creates_entities_with_default_limit(self, user_api, lb_factory_with_model): + _, lb1 = lb_factory_with_model() + _, lb2 = lb_factory_with_model() + + lb1.insert() + lb2.insert() + + entities_count = LB.objects.count() + assert entities_count == 2 + + lb1.delete() + lb2.delete() + + +class TestQuotaAggregateFieldLimit: + @pytest.fixture + def _quota_limits(self, user_api): + limits = [ + QuotaLimit( + uuid=sys_uuid.uuid4(), + project_id=c.SERVICE_PROJECT_ID, + resource_name="nodes", + field_name="cores", + limit=4, + ), + QuotaLimit( + uuid=sys_uuid.uuid4(), + project_id=c.SERVICE_PROJECT_ID, + resource_name="nodes", + field_name="ram", + limit=4096, + ), + ] + for limit in limits: + limit.insert() + yield limits + for limit in limits: + try: + limit.delete() + except Exception: + pass + + def test_rejects_unknown_quota_resource(self): + with pytest.raises(ValueError, match="Unknown quota resource: unknown"): + QuotaLimitController._validate_quota_field("unknown", "") + + def test_rejects_unknown_quota_field(self): + with pytest.raises(ValueError, match="Unknown quota field: unknown"): + QuotaLimitController._validate_quota_field("nodes", "unknown") + + def test_rejects_non_integer_quota_field(self): + with pytest.raises(ValueError, match="Quota field must be an integer: name"): + QuotaLimitController._validate_quota_field("nodes", "name") + + def test_blocks_nodes_when_cores_limit_is_exceeded( + self, + _quota_limits, + node_factory_with_model, + ): + _, first_node = node_factory_with_model(cores=2, ram=1024) + _, second_node = node_factory_with_model(cores=3, ram=1024) + + first_node.insert() + with pytest.raises(QuotaExceededError) as exc_info: + second_node.insert() + + assert exc_info.value.resource_name == "nodes.cores" + assert exc_info.value.limit == 4 + assert exc_info.value.current == 5 + + first_node.delete() + + def test_node_field_quotas_are_isolated_per_project( + self, + _quota_limits, + node_factory_with_model, + ): + # Assume the quota limits fixture sets per-project limits: + # cores limit: 4, ram limit: 4096 for each project. + project_b_uuid = sys_uuid.uuid4() + project_b_limits = [ + QuotaLimit( + uuid=sys_uuid.uuid4(), + project_id=project_b_uuid, + resource_name="nodes", + field_name="cores", + limit=4, + ), + QuotaLimit( + uuid=sys_uuid.uuid4(), + project_id=project_b_uuid, + resource_name="nodes", + field_name="ram", + limit=4096, + ), + ] + for limit in project_b_limits: + limit.insert() + + # Project A: exceed cores/ram limits and ensure quota is enforced + _, a_node1 = node_factory_with_model( + project_id=c.SERVICE_PROJECT_ID, + cores=2, + ram=2048, + ) + _, a_node2 = node_factory_with_model( + project_id=c.SERVICE_PROJECT_ID, + cores=2, + ram=2048, + ) + _, a_node3 = node_factory_with_model( + project_id=c.SERVICE_PROJECT_ID, + cores=1, + ram=1024, + ) + + a_node1.insert() + a_node2.insert() + + # Exceeding the limit in project A should raise, and the error + # values should only reflect usage in project A. + with pytest.raises(QuotaExceededError) as exc_info: + a_node3.insert() + + assert exc_info.value.resource_name in {"nodes.cores", "nodes.ram"} + assert exc_info.value.limit in {4, 4096} + assert exc_info.value.current in {5, 5120} + + # Project B: usage should be independent of project A. + # Staying within limits in project B must not raise. + _, b_node1 = node_factory_with_model( + project_id=project_b_uuid, + cores=2, + ram=2048, + ) + _, b_node2 = node_factory_with_model( + project_id=project_b_uuid, + cores=2, + ram=2048, + ) + + b_node1.insert() + b_node2.insert() + + # Clean up nodes + a_node1.delete() + a_node2.delete() + a_node3.delete() + b_node1.delete() + b_node2.delete() + for limit in project_b_limits: + limit.delete() + + def test_blocks_nodes_when_ram_limit_is_exceeded( + self, + _quota_limits, + node_factory_with_model, + ): + _, first_node = node_factory_with_model(cores=1, ram=2048) + _, second_node = node_factory_with_model(cores=1, ram=3072) + + first_node.insert() + with pytest.raises(QuotaExceededError) as exc_info: + second_node.insert() + + assert exc_info.value.resource_name == "nodes.ram" + assert exc_info.value.limit == 4096 + assert exc_info.value.current == 5120 + + first_node.delete() + + +class TestQuotaWithLimit: + def test_creates_entities( + self, + _quota_limit_2, + lb_factory_with_model, + ): + _, lb = lb_factory_with_model() + lb.insert() + + entities_count = LB.objects.count() + assert entities_count == 1 + + lb.delete() + + def test_blocks_creation_when_exceeded( + self, + _quota_limit_2, + lb_factory_with_model, + ): + _, lb1 = lb_factory_with_model() + _, lb2 = lb_factory_with_model() + _, lb3 = lb_factory_with_model() + + lb1.insert() + lb2.insert() + with pytest.raises(QuotaExceededError): + lb3.insert() + + lb1.delete() + lb2.delete() + + def test_delete_releases_entity( + self, + _quota_limit_2, + lb_factory_with_model, + ): + _, lb = lb_factory_with_model() + lb.insert() + + entities_count_before = LB.objects.count() + assert entities_count_before == 1 + + lb.delete() + + entities_count_after = LB.objects.count() + assert entities_count_after == 0 + + def test_create_after_delete_respects_limit( + self, + _quota_limit_2, + lb_factory_with_model, + ): + _, lb1 = lb_factory_with_model() + _, lb2 = lb_factory_with_model() + + lb1.insert() + lb1.delete() + lb2.insert() + + lb2.delete() + + def test_limit_isolated_per_project( + self, + _quota_limit_2, + lb_factory_with_model, + project_id, + ): + + _, lb_a = lb_factory_with_model(project_id=project_id) + _, lb_b = lb_factory_with_model() # default: SERVICE_PROJECT_ID + + lb_a.insert() + lb_b.insert() # different project, should succeed even though limit=2 + + lb_a.delete() + lb_b.delete() + + def test_exceeded_error_details( + self, + _quota_limit_2, + lb_factory_with_model, + ): + _, lb1 = lb_factory_with_model() + _, lb2 = lb_factory_with_model() + _, lb3 = lb_factory_with_model() + + lb1.insert() + lb2.insert() + with pytest.raises(QuotaExceededError) as exc_info: + lb3.insert() + + assert exc_info.value.resource_name == _TABLENAME + assert exc_info.value.limit == 2 + assert exc_info.value.current == 3 + assert exc_info.value.project_id == c.SERVICE_PROJECT_ID + + lb1.delete() + lb2.delete() diff --git a/exordos_core/tests/unit/compute/pool/drivers/test_libvirt.py b/exordos_core/tests/unit/compute/pool/drivers/test_libvirt.py index 93795eb7..4a2ab658 100644 --- a/exordos_core/tests/unit/compute/pool/drivers/test_libvirt.py +++ b/exordos_core/tests/unit/compute/pool/drivers/test_libvirt.py @@ -14,8 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import uuid as sys_uuid from unittest import mock +import uuid as sys_uuid from xml.dom import minidom from xml.etree import ElementTree as ET @@ -37,9 +37,7 @@ def _local_driver() -> LibvirtPoolDriver: # no real virtualization or daemon needed, so real libvirt calls # (lookupByUUIDString, etc.) can be exercised end-to-end. spec = models.LibvirtPoolDriverSpec(connection_uri="test:///default") - pool = models.MachinePool( - uuid=sys_uuid.uuid4(), name="test-pool", driver_spec=spec - ) + pool = models.MachinePool(uuid=sys_uuid.uuid4(), name="test-pool", driver_spec=spec) return LibvirtPoolDriver(pool) @@ -65,9 +63,7 @@ def test_removes_only_direct_children_leaving_nested_matches_alone(self): # getElementsByTagName searches the whole subtree recursively - # a naive removeChild(node) on a match found deeper in the tree # (not a direct child of root) raises NotFoundErr. - doc = minidom.parseString( - "directnested" - ) + doc = minidom.parseString("directnested") root = doc.firstChild XMLLibvirtInstance._remove_direct_children(root, "a") diff --git a/exordos_core/tests/unit/compute/test_models.py b/exordos_core/tests/unit/compute/test_models.py index aae41e5f..8200373c 100644 --- a/exordos_core/tests/unit/compute/test_models.py +++ b/exordos_core/tests/unit/compute/test_models.py @@ -13,8 +13,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import uuid as sys_uuid from unittest.mock import patch +import uuid as sys_uuid from gcl_sdk.agents.universal.dm import models as ua_models from gcl_sdk.infra.dm import models as infra_models @@ -36,7 +36,8 @@ def test_reuses_an_existing_key(self): ) with ( - patch.object(models.orm.SQLStorableMixin, "insert"), + patch.object(models.QuotaModelMixin, "insert"), + patch.object(models.Volume, "insert"), patch.object(ua_models.NodeEncryptionKey, "get_or_create") as get_or_create, ): node.insert() diff --git a/exordos_core/user_api/api/routes.py b/exordos_core/user_api/api/routes.py index 980e2811..3d0cc47a 100644 --- a/exordos_core/user_api/api/routes.py +++ b/exordos_core/user_api/api/routes.py @@ -23,6 +23,7 @@ from exordos_core.user_api.em.api import routes as em_routes from exordos_core.user_api.iam.api import routes as iam_routes from exordos_core.user_api.network.api import routes as network_routes +from exordos_core.user_api.quota.api import routes as quota_routes from exordos_core.user_api.repo.api import routes as repo_routes from exordos_core.user_api.secret.api import routes as secret_routes from exordos_core.user_api.security.api import routes as security_routes @@ -44,15 +45,16 @@ class ApiEndpointRoute(routes.Route): __controller__ = controllers.ApiEndpointController __allow_methods__ = [routes.FILTER] + compute = routes.route(compute_routes.ComputeRoute) + config = routes.route(config_routes.ConfigRoute) dns = routes.route(dns_routes.DnsRoute) health = routes.route(HealthRoute) iam = routes.route(iam_routes.IamRoute) em = routes.route(em_routes.ElementManagerRoute) - vs = routes.route(vs_routers.VSRoute) - config = routes.route(config_routes.ConfigRoute) + network = routes.route(network_routes.NetworkRoute) + quota = routes.route(quota_routes.QuotaRoute) secret = routes.route(secret_routes.SecretRoute) security = routes.route(security_routes.SecurityRoute) - compute = routes.route(compute_routes.ComputeRoute) - network = routes.route(network_routes.NetworkRoute) ua = routes.route(ua_routes.UaRoute) + vs = routes.route(vs_routers.VSRoute) repo = routes.route(repo_routes.RepoRoute) diff --git a/exordos_core/user_api/network/dm/models.py b/exordos_core/user_api/network/dm/models.py index f19c9708..5950d2cd 100644 --- a/exordos_core/user_api/network/dm/models.py +++ b/exordos_core/user_api/network/dm/models.py @@ -32,6 +32,7 @@ from exordos_core.common import exceptions as ex_exceptions from exordos_core.common import utils as u +from exordos_core.quota.dm.models import QuotaModelMixin from exordos_core.secret import utils as su @@ -68,6 +69,7 @@ class LB( models.ModelWithNameDesc, models.ModelWithTimestamp, models.ModelWithProject, + QuotaModelMixin, orm.SQLStorableMixin, ua_models.TargetResourceMixin, ): diff --git a/exordos_core/user_api/quota/__init__.py b/exordos_core/user_api/quota/__init__.py new file mode 100644 index 00000000..ba779e95 --- /dev/null +++ b/exordos_core/user_api/quota/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/exordos_core/user_api/quota/api/__init__.py b/exordos_core/user_api/quota/api/__init__.py new file mode 100644 index 00000000..ba779e95 --- /dev/null +++ b/exordos_core/user_api/quota/api/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/exordos_core/user_api/quota/api/controllers.py b/exordos_core/user_api/quota/api/controllers.py new file mode 100644 index 00000000..45ac05cd --- /dev/null +++ b/exordos_core/user_api/quota/api/controllers.py @@ -0,0 +1,69 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gcl_iam.api import controllers as iam_controllers +from restalchemy.api import controllers as ra_controllers +from restalchemy.api import resources +from restalchemy.dm import types + +from exordos_core.quota.dm import models + + +class QuotaController(ra_controllers.RoutesListController): + __TARGET_PATH__ = "/v1/quota/" + + +class QuotaLimitController( + iam_controllers.PolicyBasedController, + ra_controllers.BaseResourceControllerPaginated, +): + __policy_service_name__ = "quota" + __policy_name__ = "limit" + + __resource__ = resources.ResourceByRAModel( + models.QuotaLimit, + process_filters=True, + convert_underscore=False, + ) + + @staticmethod + def _validate_quota_field(resource_name, field_name): + resource_model = models.get_quota_resource_model(resource_name) + if not field_name: + return + + quota_property = resource_model.properties.properties.get(field_name) + if quota_property is None: + raise ValueError(f"Unknown quota field: {field_name}") + if not isinstance( + quota_property.get_property_type(), (types.Integer, types.Float) + ): + raise ValueError(f"Quota field must be an integer: {field_name}") + + def create(self, **kwargs): + self._validate_quota_field( + kwargs["resource_name"], + kwargs.get("field_name", ""), + ) + return super().create(**kwargs) + + def update(self, uuid, **kwargs): + quota_limit = self.get(uuid=uuid) + self._validate_quota_field( + kwargs.get("resource_name", quota_limit.resource_name), + kwargs.get("field_name", quota_limit.field_name), + ) + return super().update(uuid, **kwargs) diff --git a/exordos_core/user_api/quota/api/routes.py b/exordos_core/user_api/quota/api/routes.py new file mode 100644 index 00000000..6c491891 --- /dev/null +++ b/exordos_core/user_api/quota/api/routes.py @@ -0,0 +1,34 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from restalchemy.api import routes + +from exordos_core.user_api.quota.api import controllers + + +class QuotaLimitsRoute(routes.Route): + """Handler for /v1/quota/limits/ endpoint""" + + __controller__ = controllers.QuotaLimitController + + +class QuotaRoute(routes.Route): + """Handler for /v1/quota/ endpoint""" + + __allow_methods__ = [routes.FILTER] + __controller__ = controllers.QuotaController + + limits = routes.route(QuotaLimitsRoute) diff --git a/migrations/0069-add-quota-tables-f8778e.py b/migrations/0069-add-quota-tables-f8778e.py new file mode 100644 index 00000000..4d99f52d --- /dev/null +++ b/migrations/0069-add-quota-tables-f8778e.py @@ -0,0 +1,68 @@ +# Copyright 2026 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from restalchemy.storage.sql import migrations + +LOG = logging.getLogger(__name__) + + +class MigrationStep(migrations.AbstractMigrationStep): + def __init__(self): + self._depends = ["0068-fix-resource-status-hash-check-437c89.py"] + + @property + def migration_id(self): + return "f8778ebb-ce15-4b72-9860-305b84f3f7ae" + + @property + def is_manual(self): + return False + + def upgrade(self, session): + sql_expressions = [ + # quota_limits + """ + CREATE TABLE IF NOT EXISTS quota_limits ( + uuid UUID NOT NULL PRIMARY KEY, + project_id UUID NOT NULL, + resource_name VARCHAR(255) NOT NULL, + field_name VARCHAR(255) NOT NULL DEFAULT '', + "limit" INTEGER NOT NULL, + "created_at" TIMESTAMP(6) NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP(6) NOT NULL DEFAULT NOW() + ); + """, + """ + CREATE UNIQUE INDEX IF NOT EXISTS quota_limits_project_resource_field_name_idx + ON quota_limits (project_id, resource_name, field_name); + """, + ] + + for expr in sql_expressions: + session.execute(expr, None) + + def downgrade(self, session): + tables = [ + "quota_limits", + ] + + for table_name in tables: + self._delete_table_if_exists(session, table_name) + + +migration_step = MigrationStep()