build, salt: Generate OLMv1 manifest

Author: eg-ayoub

buildchain/buildchain/codegen.py

@@ -332,6 +332,26 @@ def codegen_chart_cert_manager() -> types.TaskDict:
         "task_dep": ["check_for:tox", "check_for:helm"],
     }
 
+def codegen_olm() -> types.TaskDict:
+    """Generate the SLS file for OLMv1 using the OLM render script."""
+    target_sls = constants.ROOT / "salt/metalk8s/addons/olm/deployed/chart.sls"
+    cmd = (
+        f"{constants.OLM_RENDER_CMD} "
+        f"--output {target_sls}"
+    )
+
+    file_dep = [ constants.OLM_RENDER_SCRIPT ]
+
+    return {
+        "name": "olm",
+        "title": utils.title_with_subtask_name("CODEGEN"),
+        "doc": codegen_olm.__doc__,
+        "actions": [doit.action.CmdAction(cmd, cwd=constants.ROOT)],
+        "file_dep": file_dep,
+        "task_dep": ["check_for:tox"],
+    }
+
+
 
 # List of available code generation tasks.
 CODEGEN: Tuple[Callable[[], types.TaskDict], ...] = (
@@ -345,6 +365,7 @@ def codegen_chart_cert_manager() -> types.TaskDict:
     codegen_chart_prometheus_adapter,
     codegen_chart_thanos,
     codegen_chart_cert_manager,
+    codegen_olm,
 )
 
 

buildchain/buildchain/constants.py

@@ -32,6 +32,7 @@
 PROMETHEUS_REPOSITORY: str = "quay.io/prometheus"
 THANOS_REPOSITORY: str = "quay.io/thanos"
 CERT_MANAGER_REPOSITORY: str = "quay.io/jetstack"
+OPERATOR_FRAMEWORK_REPOSITORYT: str = "quay.io/operator-framework"
 
 # Paths {{{
 
@@ -76,6 +77,8 @@
 CHART_ROOT: Path = ROOT / "charts"
 CHART_RENDER_SCRIPT: Path = CHART_ROOT / "render.py"
 
+OLM_RENDER_SCRIPT: Path = ROOT / "olm/render.py"
+
 # }}}
 # Vagrant parameters {{{
 
@@ -145,6 +148,7 @@ def git_ref() -> Optional[str]:
 ]
 
 CHART_RENDER_CMD: str = f"tox -e chart-render -- --kube-version {versions.K8S_VERSION}"
+OLM_RENDER_CMD: str = f"tox -e olm-render -- -v v{versions.OLM_VERSION}"
 
 # For mypy, see `--no-implicit-reexport` documentation.
 __all__ = ["ROOT"]

buildchain/buildchain/image.py

@@ -217,6 +217,10 @@ def _local_image(name: str, **kwargs: Any) -> targets.LocalImage:
         "cert-manager-cainjector",
         "cert-manager-acmesolver",
     ],
+    constants.OPERATOR_FRAMEWORK_REPOSITORYT: [
+        "catalogd",
+        "operator-controller",
+    ],
 }
 
 REMOTE_NAMES: Dict[str, str] = {

buildchain/buildchain/salt_tree.py

@@ -344,6 +344,8 @@ def task(self) -> types.TaskDict:
         file_dep=[METALK8S_OPERATOR_MANIFESTS],
     ),
     Path("salt/metalk8s/addons/metalk8s-operator/deployed/init.sls"),
+    Path("salt/metalk8s/addons/olm/deployed/chart.sls"),
+    Path("salt/metalk8s/addons/olm/deployed/init.sls"),
     Path("salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls"),
     Path("salt/metalk8s/addons/prometheus-adapter/deployed/init.sls"),
     Path("salt/metalk8s/addons/prometheus-operator/macros.j2"),

buildchain/buildchain/versions.py

@@ -32,6 +32,8 @@
 CONTAINERD_RELEASE: str = "1"
 SOSREPORT_RELEASE: str = "2"
 
+OLM_VERSION: str = "1.1.0"
+
 
 def load_version_information() -> None:
     """Load version information from `VERSION`."""
@@ -225,6 +227,16 @@ def _version_prefix(version: str, prefix: str = "v") -> str:
         version="v0.36.1",
         digest="sha256:e542959e1b36d5046083d1b64a7049c356b68a44a173c58b3ae7c0c9ada932d5",
     ),
+    Image(
+        name="catalogd",
+        version=_version_prefix(OLM_VERSION),
+        digest="sha256:95477a136772765fa2cfb02a6e5fb52bcc167ef7b5333f9e238b0b13b9e72f7b",
+    ),
+    Image(
+        name="operator-controller",
+        version=_version_prefix(OLM_VERSION),
+        digest="sha256:6272919257e695fcdadf0b57cc0a272084ebf2caca7571e2e5d79d6a56a788fa",
+    ),
     # Local images
     Image(
         name="metalk8s-alert-logger",

olm/.gitignore

@@ -0,0 +1 @@
+*.yaml

olm/render.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python
+
+"""
+This script downloads operator controller and catalogd manifests and `renders` them into a useable
+salt chart.
+
+To do so, it:
+  - downloads manifests from github releases (optional)
+  - merges duplicate objects
+  - adds appropriate labels
+  - changes names and namespaces of cert-manager resources
+  - swaps image values
+"""
+
+import argparse
+import io
+import pathlib
+import re
+import requests
+import sys
+import yaml
+
+CONTROLLER_MANIFEST_URL = "https://github.com/operator-framework/operator-controller/releases/download/{version}/operator-controller.yaml"
+
+CATALOGD_MANIFEST_URL = "https://github.com/operator-framework/catalogd/releases/download/{version}/catalogd.yaml"
+
+REGISTRIES_CONF = """
+[[registry]]
+prefix = "{% endraw -%}{{ repo.registry_endpoint }}{%- raw %}"
+insecure = true
+location = "{% endraw -%}{{ repo.registry_endpoint }}{%- raw %}:80"
+[[registry]]
+prefix = "registry.metalk8s.lan"
+insecure = true
+location = "{% endraw -%}{{ repo.registry_endpoint }}{%- raw %}:80"
+"""
+
+
+START_BLOCK = """
+#!jinja | metalk8s_kubernetes
+
+{%- from "metalk8s/map.jinja" import repo with context %}
+{%- from "metalk8s/repo/macro.sls" import build_image_name with context %}
+
+{% raw %}
+"""
+
+END_BLOCK = """
+{% endraw %}
+"""
+
+class DownloadError(Exception):
+    pass
+
+def semver_regex_type(value):
+    pat = re.compile(r"^v\d+\.\d+\.\d+$")
+    if not pat.match(value):
+        raise argparse.ArgumentTypeError("invalid value, must be ~v1.1.2")
+    return value
+
+def download_source_manifest(version):
+    controller_response = requests.get(CONTROLLER_MANIFEST_URL.format(version=version))
+    catalogd_response = requests.get(CATALOGD_MANIFEST_URL.format(version=version))
+    if controller_response.status_code == 200 and catalogd_response.status_code == 200:
+        return controller_response.content, catalogd_response.content
+    raise DownloadError("Problem fetching catalogd or operator controller manifests")
+
+"""
+Merge source manifests on top of destination
+"""
+def merge(source, destination):
+    def _dict_merge(src, dst):
+        for k, v in src.items():
+            if k in dst and isinstance(dst[k], dict) and isinstance(v, dict):
+                _dict_merge(v, dst[k])
+            else:
+                dst[k] = v
+    to_add = []
+    # check for each doc in source
+    for sdoc in source:
+        # does it exist in the destination docs ?
+        found = False
+        for ddoc in destination:
+            if ddoc['kind'] == sdoc['kind'] and ddoc['metadata']['name'] == sdoc['metadata']['name']:
+                found = True
+                _dict_merge(sdoc, ddoc)
+        if not found:
+            to_add.append(sdoc)
+    destination.extend(to_add)
+    return destination
+
+def add_labels(manifest, version):
+    for doc in manifest:
+        doc['metadata'].setdefault('labels', {}).update({
+                "app.kubernetes.io/instance": "olm",
+                "app.kubernetes.io/managed-by": "salt",
+                "app.kubernetes.io/part-of": "metalk8s",
+                "app.kubernetes.io/version": str(version),
+                "heritage": "metalk8s",
+            })
+    return manifest
+
+def add_tolerations(manifest):
+    for doc in manifest:
+        if doc['kind'] == "Deployment":
+            doc['spec']['template']['spec'].setdefault('tolerations', []).extend([
+                {
+                    "key": "node-role.kubernetes.io/bootstrap",
+                    "operator": "Exists",
+                    "effect": "NoSchedule",
+                },
+                {
+                    "key": "node-role.kubernetes.io/infra",
+                    "operator": "Exists",
+                    "effect": "NoSchedule",
+                },
+            ])
+    return manifest
+
+def add_registries_conf(manifest):
+    manifest.append({
+        "apiVersion": "v1",
+        "kind": "ConfigMap",
+        "metadata": {
+            "name": "registries-conf",
+            "namespace": "olmv1-system",
+        },
+        "data": {
+            "registries.conf": REGISTRIES_CONF,
+        },
+    })
+    for doc in manifest:
+        if doc['kind'] == "Deployment":
+            doc['spec']['template']['spec']['volumes'].append({
+                "name": "registries-conf",
+                "configMap": {
+                    "name": "registries-conf",
+                },
+            })
+            doc['spec']['template']['spec']['containers'][0]['volumeMounts'].append({
+                "name": "registries-conf",
+                "mountPath": "/etc/containers/",
+            })
+    return manifest
+
+def add_node_selector(manifest):
+    for doc in manifest:
+        if doc['kind'] == "Deployment":
+            doc['spec']['template']['spec'].setdefault('nodeSelector', {}).update({
+                "kubernetes.io/os": "linux",
+                "node-role.kubernetes.io/infra": "",
+            })
+    return manifest
+
+def fixup_certmanager(manifest):
+    for doc in manifest:
+        if doc['apiVersion'] == "cert-manager.io/v1" and \
+            doc['metadata'].get('namespace') == 'cert-manager':
+                doc['metadata']['namespace'] = 'metalk8s-certs'
+        if doc['kind'] == "MutatingWebhookConfiguration":
+            doc['metadata']['annotations']['cert-manager.io/inject-ca-from-secret']\
+                = "metalk8s-certs/olmv1-ca"
+    return manifest
+
+class multiline_string(str):
+    pass
+
+def represent_multiline_string(dumper, data):
+    scalar = yaml.SafeDumper.represent_str(dumper, data)
+    scalar.style = "|"
+    return scalar
+
+yaml.SafeDumper.add_representer(multiline_string, represent_multiline_string)
+
+def render(manifest):
+    def _fix_strings(obj):
+        if isinstance(obj, dict):
+            return dict((k, _fix_strings(v)) for (k, v) in obj.items())
+        elif isinstance(obj, list):
+            return [ _fix_strings(elem) for elem in obj ]
+        elif isinstance(obj, str):
+            if "\n" in obj:
+                value = "\n".join(
+                    line for line in obj.splitlines() if not re.match(r"^\s*$", line)
+                )
+                return multiline_string(value)
+            return obj
+        else:
+            return obj
+    manifest = _fix_strings(manifest)
+    out = START_BLOCK.lstrip()
+    stream = io.StringIO()
+    yaml.safe_dump_all(
+        manifest,
+        stream,
+        default_flow_style=False,
+    )
+    stream.seek(0)
+    out += re.sub(
+            r"image: quay.io/operator-framework/(?P<image>.*):(?P<tag>.*)",
+            r'image: {% endraw -%}{{ build_image_name("\g<image>", False) }}{%- raw %}:\g<tag>',
+            stream.read(),
+            )
+    out += END_BLOCK
+    return out
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--version', type=semver_regex_type)
+    parser.add_argument('-o', '--output', type=pathlib.Path)
+    args = parser.parse_args()
+
+    # 1 - download the manifests
+    controller_manifest, catalogd_manifest = download_source_manifest(version=args.version)
+
+    # 1.5 - interpret yaml
+    controller = list(yaml.safe_load_all(controller_manifest))
+    catalogd = list(yaml.safe_load_all(catalogd_manifest))
+
+    # 2 - merge manifests
+    manifest = merge(catalogd, controller)
+
+    # 3- add labels, tolerations, nodeSelector
+    manifest = add_labels(manifest, args.version)
+    manifest = add_tolerations(manifest)
+    manifest = add_node_selector(manifest)
+    manifest = add_registries_conf(manifest)
+
+    # 4- Fix cert-manager objects
+    manifest = fixup_certmanager(manifest)
+
+    # 5- render yaml with new images
+    rendered = render(manifest)
+
+    if args.output:
+        with open(args.output, "w") as fd:
+            fd.write(rendered)
+    else:
+        sys.stdout.write(rendered)
+
+
+if __name__ == "__main__":
+    main()