move certs to universal builder (#482)

Author: slashburygin

exordos/manifests/examples/certificates.yaml

@@ -0,0 +1,54 @@
+name: "certificates-example"
+uuid: "b8a4de1c-2f56-4bc1-a3b8-4fe40124cccc"
+description: "Exordos certificates example"
+schema_version: 1
+version: "0.0.1"
+api_version: "v1"
+
+requirements:
+  core:
+    from_version: "0.0.0"
+
+resources:
+  $core.secret.certificates:
+    # DNS Core certificate (ACME via Let's Encrypt with DNS-01 challenge)
+    # Uses DNS CoreDNS plugin to automatically fulfill ACME challenges.
+    # After provisioning, cert and key are stored on the agent.
+    wildcard_cert:
+      name: "wildcard-cert"
+      project_id: "12345678-c625-4fee-81d5-f691897b8142"
+      email: "admin@example.com"
+      method:
+        kind: dns_core
+      domains:
+        - "*.example.com"
+        - "example.com"
+      expiration_threshold: 30
+      overcome_threshold: false
+
+    # Multi-domain SAN certificate
+    # Subject Alternative Names for multiple service domains.
+    san_cert:
+      name: "san-cert"
+      project_id: "12345678-c625-4fee-81d5-f691897b8142"
+      email: "devops@example.com"
+      method:
+        kind: dns_core
+      domains:
+        - "api.example.com"
+        - "auth.example.com"
+        - "admin.example.com"
+      expiration_threshold: 14
+
+    # Internal service certificate with no wildcards
+    internal_cert:
+      name: "internal-cert"
+      project_id: "12345678-c625-4fee-81d5-f691897b8142"
+      email: "it@example.com"
+      method:
+        kind: dns_core
+      domains:
+        - "gitlab.internal.example.com"
+        - "jenkins.internal.example.com"
+      expiration_threshold: 7
+      overcome_threshold: true

exordos_core/agent/universal/drivers/secret/backend/cert.py

@@ -36,13 +36,11 @@
 class CertBotBackendClient(base.AbstractBackendClient):
     """Cert bot backend client."""
 
-    DEFAULT_PRIVATE_KEY_PATH = "/etc/exordos_core/certbot/privkey.pem"
-
     def __init__(
         self,
         dns_client: dns_clients.TinyDNSCoreClient,
         admin_email: str,
-        private_key_path: str = DEFAULT_PRIVATE_KEY_PATH,
+        private_key_path: str,
     ) -> None:
         self._dns_client = dns_client
         self._admin_email = admin_email
@@ -89,7 +87,7 @@ def create(self, resource: models.Resource) -> tp.Dict[str, tp.Any]:
         cert_x509 = x509.load_pem_x509_certificate(fullchain_pem.encode())
         expiration_at = cert_x509.not_valid_after_utc
 
-        # Build storagable password and save
+        # Build storage_cert and save
         driver_cert = driver_dm.Certificate.from_cert_resource(
             resource, pkey_pem, csr_pem, fullchain_pem, expiration_at
         )

…t/universal/drivers/secret/backend/db.py …ersal/drivers/secret/backend/password.pyexordos_core/agent/universal/drivers/secret/backend/db.py renamed to exordos_core/agent/universal/drivers/secret/backend/password.py exordos_core/agent/universal/drivers/secret/backend/db.py renamed to exordos_core/agent/universal/drivers/secret/backend/password.py

@@ -31,7 +31,7 @@
 LOG = logging.getLogger(__name__)
 
 
-class DatabaseSecretBackendClient(base.AbstractBackendClient):
+class DatabasePasswordBackendClient(base.AbstractBackendClient):
     """Secret Backend client based on SQL database."""
 
     def get(self, resource: models.Resource) -> tp.Dict[str, tp.Any]:

exordos_core/agent/universal/drivers/secret/cert.py

@@ -28,6 +28,7 @@
 
 CERT_TARGET_FIELDS_STORAGE = "/var/lib/exordos/exordos_core/cert_target_fields.json"
 DEFAULT_ADMIN_EMAIL = "infra@exordos.com"
+DEFAULT_PRIVATE_KEY_PATH = "/etc/exordos_core/certbot/privkey.pem"
 
 
 class CoreDNSCertificateCapabilityDriver(direct.DirectAgentDriver):
@@ -38,9 +39,11 @@ def __init__(
         user_api_base_url: str,
         username: str,
         password: str,
+        private_key_path: str = DEFAULT_PRIVATE_KEY_PATH,
         admin_email: str = DEFAULT_ADMIN_EMAIL,
+        storage_path: str = CERT_TARGET_FIELDS_STORAGE,
     ) -> None:
-        storage = fs.TargetFieldsFileStorage(CERT_TARGET_FIELDS_STORAGE)
+        storage = fs.TargetFieldsFileStorage(storage_path)
 
         auth = core_client_base.CoreIamAuthenticator(
             base_url=user_api_base_url, username=username, password=password
@@ -49,7 +52,9 @@ def __init__(
             base_url=user_api_base_url, auth=auth
         )
 
-        client = cert_back.CertBotBackendClient(dns_client, admin_email)
+        client = cert_back.CertBotBackendClient(
+            dns_client, admin_email, private_key_path
+        )
 
         super().__init__(storage=storage, client=client)
 

exordos_core/agent/universal/drivers/secret/password.py

@@ -20,7 +20,7 @@
 from gcl_sdk.agents.universal.drivers import direct
 from gcl_sdk.agents.universal.storage import fs
 
-from exordos_core.agent.universal.drivers.secret.backend import db as db_back
+from exordos_core.agent.universal.drivers.secret.backend import password as db_back
 
 LOG = logging.getLogger(__name__)
 
@@ -34,7 +34,7 @@ class PasswordCapabilityDriver(direct.DirectAgentDriver):
 
     def __init__(self, storage_path: str = PASSWORD_TARGET_FIELDS_STORAGE):
         storage = fs.TargetFieldsFileStorage(storage_path)
-        client = db_back.DatabaseSecretBackendClient()
+        client = db_back.DatabasePasswordBackendClient()
 
         super().__init__(storage=storage, client=client)
 

exordos_core/common/constants.py

@@ -41,8 +41,13 @@
 NODE_UUID_PATH = os.path.join(WORK_DIR, "node-id")
 
 DEFAULT_USER_API_HOST = "127.0.0.1"
-DEFAULT_USER_API_PORT = 11010
-DEFAULT_ROOT_ENDPOINT = f"http://{DEFAULT_USER_API_HOST}:{DEFAULT_USER_API_PORT}/v1/"
+DEFAULT_USER_API_PORT = 80
+DEFAULT_ROOT_ENDPOINT = (
+    f"http://{DEFAULT_USER_API_HOST}:{DEFAULT_USER_API_PORT}/api/core/v1/"
+)
+
+DEFAULT_DNS_CERT_USERNAME = "admin"
+DEFAULT_DNS_CERT_PASSWORD = "admin"
 
 DEFAULT_GLOBAL_SALT = "FOy/2kwwdn0ig1QOq7cestqe"
 DEFAULT_ADMIN_SALT = "d4JJ9QYuEEJxHCFja9FZskG4"

exordos_core/gservice/service.py

@@ -55,7 +55,7 @@
 from exordos_core.network.lb.builders import paas as net_lb_paas
 from exordos_core.network.lb.dm import models as lb_models
 from exordos_core.secret import service as secret_service
-from exordos_core.secret.builders import service as password_builder_svc
+from exordos_core.secret.builders import service as secret_builder_svc
 from exordos_core.telemetry import service as telemetry_service
 from exordos_core.vs.builders import service as vs_builder_svc
 
@@ -195,7 +195,10 @@ def __init__(self, iter_min_period=3, iter_pause=0.1):
         secret_svc = secret_service.SecretServiceBuilder(
             iter_min_period=iter_min_period,
         )
-        password_builder = password_builder_svc.PasswordBuilder(
+        password_builder = secret_builder_svc.PasswordBuilder(
+            iter_min_period=iter_min_period
+        )
+        cert_builder = secret_builder_svc.CertificateBuilder(
             iter_min_period=iter_min_period
         )
         # Build the event sender only when event delivery is enabled (the
@@ -240,6 +243,7 @@ def __init__(self, iter_min_period=3, iter_pause=0.1):
             net_border_paas_builder,
             secret_svc,
             password_builder,
+            cert_builder,
             em_builder,
             dns_sync,
             # non-essential services should be last

exordos_core/secret/builders/service.py

@@ -96,3 +96,38 @@ def actualize_outdated_instance(
 
         if status_updated or password_updated:
             current_instance.save()
+
+
+class CertificateBuilder(sdk_builder.UniversalBuilderService):
+    def __init__(
+        self,
+        iter_min_period: int = 1,
+        iter_pause: float = 0.1,
+    ) -> None:
+        super().__init__(
+            instance_model=Certificate,
+            iter_min_period=iter_min_period,
+            iter_pause=iter_pause,
+        )
+
+    def actualize_outdated_instance(
+        self,
+        current_instance: Certificate,
+        actual_instance: Certificate,
+    ) -> None:
+        status_updated = current_instance.status != actual_instance.status
+        if status_updated:
+            current_instance.status = actual_instance.status
+
+        cert_updated = (
+            current_instance.key != actual_instance.key
+            or current_instance.cert != actual_instance.cert
+            or current_instance.expiration_at != actual_instance.expiration_at
+        )
+        if cert_updated:
+            current_instance.key = actual_instance.key
+            current_instance.cert = actual_instance.cert
+            current_instance.expiration_at = actual_instance.expiration_at
+
+        if status_updated or cert_updated:
+            current_instance.save()

exordos_core/secret/service.py

@@ -547,5 +547,5 @@ def _actualize_ssh_keys(self) -> None:
 
     def _iteration(self) -> None:
         with contexts.Context().session_manager():
-            self._actualize_certificates()
+            # self._actualize_certificates()
             self._actualize_ssh_keys()

exordos_core/tests/functional/conftest.py

@@ -37,6 +37,7 @@
 from restalchemy.dm import filters as dm_filters
 from restalchemy.storage.sql import engines
 
+from exordos_core.agent.universal.drivers.secret import cert as cert_driver
 from exordos_core.agent.universal.drivers.secret import password as password_driver
 from exordos_core.common import constants as c
 from exordos_core.common import utils
@@ -1433,19 +1434,18 @@ def factory(hostname, ip_addresses=None, key=None):
 
 
 @pytest.fixture()
-def agent_service(
+def password_agent_service(
     default_node: tp.Dict[str, tp.Any],
     user_api_client: iam_clients.GenesisCoreTestRESTClient,
 ):
     agent_uuid = sys_uuid.UUID(default_node["uuid"])
     orch_client = orch_db.DatabaseOrchClient()
     with tempfile.TemporaryDirectory() as temp_dir:
-        temp_file_path = os.path.join(temp_dir, "password_target_fields.json")
-
-        with open(temp_file_path, "w", encoding="utf-8") as f:
+        passwords_temp_file_path = os.path.join(temp_dir, "password_target_fields.json")
+        with open(passwords_temp_file_path, "w", encoding="utf-8") as f:
             json.dump({}, f, indent=4)
 
-        p_d = password_driver.PasswordCapabilityDriver(temp_file_path)
+        p_d = password_driver.PasswordCapabilityDriver(passwords_temp_file_path)
         caps_drivers = [
             p_d,
         ]
@@ -1464,11 +1464,53 @@ def agent_service(
         yield agent
 
 
+@pytest.fixture()
+def cert_agent_service(
+    default_node: tp.Dict[str, tp.Any],
+    user_api_client: iam_clients.GenesisCoreTestRESTClient,
+):
+    agent_uuid = sys_uuid.UUID(default_node["uuid"])
+    orch_client = orch_db.DatabaseOrchClient()
+    with tempfile.TemporaryDirectory() as temp_dir:
+        cert_temp_file_path = os.path.join(temp_dir, "cert_target_fields.json")
+        with open(cert_temp_file_path, "w", encoding="utf-8") as f:
+            json.dump({}, f, indent=4)
+        privkey_temp_file_path = os.path.join(temp_dir, "privkey.pem")
+        c_d = cert_driver.CoreDNSCertificateCapabilityDriver(
+            user_api_base_url=c.DEFAULT_ROOT_ENDPOINT,
+            username=c.DEFAULT_DNS_CERT_USERNAME,
+            password=c.DEFAULT_DNS_CERT_PASSWORD,
+            storage_path=cert_temp_file_path,
+            private_key_path=privkey_temp_file_path,
+        )
+        caps_drivers = [
+            c_d,
+        ]
+        agent = ua_agent_service.UniversalAgentService(
+            system_uuid=agent_uuid,
+            agent_uuid=agent_uuid,
+            orch_client=orch_client,
+            caps_drivers=caps_drivers,
+            facts_drivers=[],
+            iter_min_period=3,
+            payload_path=None,
+            verify_node_on_register=False,
+        )
+        agent._setup()
+        agent._register_agent()
+        yield agent
+
+
 @pytest.fixture()
 def password_builder():
     yield secret_service.PasswordBuilder()
 
 
+@pytest.fixture()
+def certificate_builder():
+    yield secret_service.CertificateBuilder()
+
+
 @pytest.fixture()
 def universal_scheduler():
     yield ua_scheduler_service.UniversalAgentSchedulerService(capabilities=["*"])