feat(workflow-engine): make temporal worker a rock

.gitignore

@@ -1,10 +1,12 @@
 __pycache__
 .env
-tools/workflow-engine/charms/temporal-worker/config.yml
+tools/workflow-engine/charms/config.yml
+tools/workflow-engine/charms/environment.yml
+tools/workflow-engine/charms/registryconfig.yml
 tools/workflow-engine/charms/temporal-worker/dist
 *.rock
 .terraform
 *terraform.tfstate*
 *.tfvars*
 .vscode
-venv
+venv

tools/workflow-engine/charms/temporal-worker/config.yml.template → tools/workflow-engine/charms/config.yml.template

@@ -46,14 +46,3 @@ temporal-worker-k8s:
     mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
     emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
     -----END CERTIFICATE-----
-  # Only needed if deploying under a proxy
-  # http-proxy: "http://squid.internal:3128"
-  # https-proxy: "http://squid.internal:3128"
-  # no-proxy: ".canonical.com"
-
-  # Charmed workflows
-  # The "workflows-file-name" must match the wheel file created with poetry
-  workflows-file-name: "oci_factory_workflows-0.0.1-py3-none-any.whl"
-  # To support all defined workflows and activities, use the 'all' keyword
-  supported-workflows: "all"
-  supported-activities: "all"

tools/workflow-engine/charms/environment.yml.template

@@ -0,0 +1,27 @@
+env:
+  - name: ROCKS_EVENTBUS_USERNAME
+    value:
+  - name: ROCKS_EVENTBUS_PASSWORD
+    value:
+  - name: ROCKS_EVENTBUS_KAFKA_ADDRESS
+    value:
+  - name: ROCKS_EVENTBUS_KARAPACE_URL
+    value:
+  - name: OS_AUTH_URL
+    value:
+  - name: OS_USERNAME
+    value:
+  - name: OS_PASSWORD
+    value:
+  - name: OS_PROJECT_NAME
+    value:
+  - name: OS_STORAGE_URL
+    value:
+  - name: GITHUB_TOKEN
+    value:
+  - name: MATTERMOST_TOKEN
+    value:
+  - name: MATTERMOST_SERVER
+    value: https://chat.canonical.com
+  - name: MATTERMOST_CHANNEL_ID
+    value:

tools/workflow-engine/charms/registryconfig.yml.template

@@ -0,0 +1,3 @@
+registrypath:
+username:
+password:

tools/workflow-engine/charms/temporal-worker/Makefile

@@ -0,0 +1,104 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Makefile to help automate tasks
+
+# The name of the python package/project
+PY_PACKAGE := resource_sample
+
+# ROCK build parameters
+ROCK_NAME := temporal-worker_1.0_amd64.rock
+IMAGE_NAME := temporal-worker-rock:latest
+
+# build and dist folders
+BUILD := build
+DIST := dist
+
+# Paths to venv executables
+POETRY := poetry
+PY := python3
+PYTEST := pytest
+BANDIT := bandit
+BLACK := black
+FLAKE8 := flake8
+ISORT := isort
+MYPY := mypy
+PYLINT := pylint
+PYDOCSTYLE := pydocstyle
+
+.PHONY: help
+help:  ## Print help about available targets
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: run
+run: ## Run the application
+	$(POETRY) run $(PY) -m $(PY_PACKAGE)
+
+.PHONY: install
+install:
+	$(POETRY) install --only main --no-root
+
+# Development tools.
+.PHONY: install-dev
+install-dev:
+	$(POETRY) install --with dev --no-root
+
+.PHONY: lint
+lint: ## Run linter
+	$(POETRY) run $(FLAKE8) $(PY_PACKAGE) tests
+	$(POETRY) run $(ISORT) --check $(PY_PACKAGE) tests
+	$(POETRY) run $(BLACK) --check $(PY_PACKAGE) tests
+	$(POETRY) run $(BANDIT) --configfile pyproject.toml --quiet --recursive $(PY_PACKAGE) tests
+
+.PHONY: fmt
+fmt: ## Reformat code for linter
+	$(POETRY) run $(ISORT) $(PY_PACKAGE) tests
+	$(POETRY) run $(BLACK) $(PY_PACKAGE) tests
+
+.PHONY: test
+test: ## Run tests
+	$(POETRY) run $(PYTEST) --cov=$(PY_PACKAGE) tests
+
+.PHONY: check
+check: clean install-dev lint test  ## Runs linter and tests from a clean directory
+
+
+# Release management.
+
+.PHONY: changelog
+changelog:  ## Add a new entry to the Changelog and bump the package version
+	./scripts/update_changelog.sh
+
+.PHONY: build
+build: ## Create a Python source distribution and a wheel in dist
+	$(POETRY) build
+
+.PHONY: build_rock
+build_rock:
+	rockcraft pack
+	rockcraft.skopeo --insecure-policy copy --dest-tls-verify=false oci-archive:$(ROCK_NAME) docker://localhost:32000/$(IMAGE_NAME)
+
+.PHONY: publish
+publish: ## Publish the package to PYPI
+	$(POETRY) publish
+
+# Cleaning up.
+
+.PHONY: clean-dist
+clean-dist:
+	rm -rf $(BUILD) $(DIST)
+
+.PHONY: clean-pyc
+clean-pyc:
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	rm -rf .pytest_cache .coverage .mypy_cache
+
+.PHONY: clean-venv
+clean-venv:
+	$(POETRY) env remove --all
+
+.PHONY: clean
+clean: clean-dist clean-pyc clean-venv  ## Clean up the virtualenv, Python bytecode and docs
+	rm -rf *.egg-info

tools/workflow-engine/charms/temporal-worker/oci_factory/activities/activity_consume_events.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import subprocess
+from datetime import datetime, timezone
 
 from confluent_kafka import Consumer, KafkaException
 from temporalio import activity
@@ -26,6 +27,23 @@
 # `TWC_LOG_LEVEL` is the mapped value of `log-level` in the charm config
 logging.basicConfig(level=os.environ.get("TWC_LOG_LEVEL", "info").upper())
 
+# Custom log format function
+class CustomFormatter(logging.Formatter):
+    def format(self, record):
+        # timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S:%MSZ")
+        # return f"{record.filename}: {timestamp}: {record.getMessage()}"
+        return f"{record.filename}: {record.getMessage()}"
+
+# Configure the root logger
+logging.basicConfig(
+    level=logging.DEBUG,  # Set logging level
+    handlers=[logging.StreamHandler()],  # Output to console; use FileHandler for files
+)
+
+# Override the default formatter
+for handler in logging.root.handlers:
+    handler.setFormatter(CustomFormatter())
+
 
 @activity.defn
 async def consume(topic: str, consumer_group: str) -> dict:
@@ -38,7 +56,9 @@ async def consume(topic: str, consumer_group: str) -> dict:
         f"Consuming messages from topic {topic} "
         f"with consumer group {consumer_group}"
     )
-
+    activity.heartbeat(f"Consuming messages from topic {topic} "
+        f"with consumer group {consumer_group}"
+    )
     activity_info = activity.info()
     activity_info_formatter = (activity_info.workflow_id, activity_info.workflow_run_id)
 
@@ -55,20 +75,23 @@ async def consume(topic: str, consumer_group: str) -> dict:
         deserialize = registry.topic_deserializer(topic)
         consumer = Consumer(consumer_config)
         consumer.subscribe([topic])
+        activity.heartbeat(f"Subscribed to topic {topic}")
         try:
             while True:
-                msg = consumer.poll(timeout=1.0)
-                print(msg)
-
+                logging.info(f"Waiting for message from event bus on topic {topic}")
+                msg = consumer.poll(timeout=300.0)
+                activity.heartbeat(f"Polling timed out from topic {topic}")
                 if msg is None:
                     continue
+                logging.info(f"Message received: {msg}")
                 if msg.error():
                     raise KafkaException(msg.error())
                 value = deserialize(msg.value())
                 break
         finally:
             consumer.close()
 
+    activity.heartbeat("Sending message to Mattermost")
     logging.info("Release: {}".format(value["release"]))
     message_id = send_message(
         MM_MESSAGE_TITLE,
@@ -79,20 +102,22 @@ async def consume(topic: str, consumer_group: str) -> dict:
     # Details see ROCKS-1197
     curr_file_path = os.path.dirname(os.path.realpath(__file__))
     script_full_path = os.path.join(curr_file_path, "find_images_to_update.py")
-
+    activity.heartbeat(f"Triggering rebuild for release {value['release']}")
     proc = subprocess.Popen(
         ["python3", script_full_path, "{}".format(value.get("release"))],
         stderr=subprocess.PIPE,
     )
+    for line in proc.stderr.readlines():
+        logging.info(line.decode("utf-8").strip())
     success = proc.wait() == 0
-
+    activity.heartbeat("Updating Mattermost message")
     status = "Success" if success else "Failed"
     update_status_and_message(
         message_id,
         success,
         MM_MESSAGE_BODY.format(value["release"], status, *activity_info_formatter),
     )
-
+    activity.heartbeat("Consumer activity completed")
     return {
         "eventbus_message": value,
         "find_images_to_update.py": proc.stderr.read().decode("utf-8"),

tools/workflow-engine/charms/temporal-worker/oci_factory/activities/consumer/ca.crt

@@ -1,7 +1,7 @@
 -----BEGIN CERTIFICATE-----
-MIIFLTCCAxWgAwIBAgIUPw02SFy/2CDfZBZmYVN1S4fIK3IwDQYJKoZIhvcNAQEN
+MIIFLTCCAxWgAwIBAgIUIC6nh9Px2Xr4ZZ4atmBZ5dzCl9swDQYJKoZIhvcNAQEN
 BQAwLTETMBEGA1UECgwKaW8uc3RyaW16aTEWMBQGA1UEAwwNY2x1c3Rlci1jYSB2
-MDAeFw0yNDA3MTUxMjUxMjdaFw0yNTA3MTUxMjUxMjdaMC0xEzARBgNVBAoMCmlv
+MDAeFw0yNTA2MTUxMjU2NThaFw0yNjA2MTUxMjU2NThaMC0xEzARBgNVBAoMCmlv
 LnN0cmltemkxFjAUBgNVBAMMDWNsdXN0ZXItY2EgdjAwggIiMA0GCSqGSIb3DQEB
 AQUAA4ICDwAwggIKAoICAQDqke5PeB5bcZKCgzop2F6NEjavs60Wg/7QGak6IXXA
 zrhw/M4Sxw9u3MydEd1yDpp4q38xH8QRBmzTublWKdXTVcxQiaXPeK6M3nUXOY36
@@ -16,15 +16,16 @@ fnjh/dR34BFfJmcx95w0BUomq38OL5WYAP+M/BQax1ws0U2kTI1e1u8jlTUyA9xi
 m3t0TuIkKTCasIxC90Vn9xbepnChwNpGuI7vZySaisgciAIkrhQDoHKLWFtj04sV
 CwIDAQABo0UwQzAdBgNVHQ4EFgQUJdKnCS58yN0bWek2+dv3FjiFI4cwEgYDVR0T
 AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQENBQADggIB
-AFCD8EcTzpzD/AyH42dl6o0nBRiFNUPNRueuIu6RUhBpyj1xKyqbIseH4aDDMwrI
-InUKiYcXrj9KL72oSdjXIUYYb+GuI7xTNem5I2qRhSWLBiOObPX+Z10+dJyc49Hl
-QPJIxAGPZhoTSYaDuuAe7piFYIEbG5iCEQ5BiQysYYZlPzA5KvofVGsVTcpSFtMz
-+SXsUYi392hk8RMHNvfqlQr++el5wzKi5PRmgX3q0x1mRMQrQsMXDIkNZV1uxUJ0
-zdwwisAubjzRZnvpnrwqLnXtWLsPV5nn7W58VaEqLOI2RVQSd4zlHS2ALb6jcI1v
-VByLu36lG8zwwNDiSlTQtyzBZV8kZ5kyzmWRDmPvhDUfvv1XgCS52tjVDoyBNq2U
-E5azHu7RUD6fqweT7JufOWyRVor2gxccoLvNCDt8X5giAUHBDLYoo4qOKosCrEQU
-sVakBF6vRQPG9ftgPPCH9aEPLnD/xSgomJJXRYzVBjPbH+F8DDy/UtV9MUyEihKI
-piP8g+ctKVefOCMwWkQuMZFlu4jPewz6+F+NMrkxoTqDlT2S4rCh0PphtmzVptJB
-+CmOcoXhFtmiKNxhFF/Zy4SntW+f3lLAzq62gc8LiBAbxBhWRK2TwbR0Upr7vvji
-9k5rAQ+CYQJf7G/Il8UMWwRXaQeCqPAGg8NT3oZAKZnc
+AIs2q1xlcNeaghcyWAc3oCPzqx3azWF1zMCrvgDka1j0PU72qJi7dsfL4RK7of7b
+MjqraebkZcdgKd20uW6oB5G//jtBT8l277235yG2otYklabwrlwrvIqhemUh1ksw
+oLFl366EUg8OVa6fQpARhqFjBugc0TR7KG4LrtGHatvjUZqnBeNAnF/UEbf4UlwC
+sauoaf1YGzU+9/JZKAoCM4mpD11nkVCxdtNVZth2tjpDLToiUT64CQu3i82LBFd5
+H9/biDoWnCFIOMz4bJpmejLcdmuQfI46Q/iPv6q5iJMxx8jBUBTTzyIul5H1hqoU
+IusT8BnlDK/nrADz8RSIYenZdA6Tv5uAu1yrxNZZpZPk/beGN2h1plkx4PJ9sErz
+KI13P3UsgXAp7MAInUsn+bZy5RbF5J9VZ8eYGAhNH2Msc7HsEoZLy1kJFaw2lkc9
+q1J3cvwv8T68xAlbIf9o2/sMKmT312SAszZr2I7qseokmP7JMLjxgHI2LuTezrvw
+rcV5BTPdDeY0UbsWu9IJR1Ow27kCC/cX9fex2MveoFRjGX0PBqBilXiE2IzKAIgQ
+Zn0/se4yPOq211Gu36IRxRMNPCSVvGP8In6Ug9X7KY0P2cheB2G3M0I8v8QtzmKt
+HGolXY7YZA4gR3T3fJRx8Vur0JZaeWmx7Oe5mj06d81X
 -----END CERTIFICATE-----
+