From 5d731ea675f29e0a1467832c10a2d79a97fc7a00 Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Wed, 23 Jul 2025 09:37:08 +0200 Subject: [PATCH 01/12] fix: minio use ssl --- start | 1 + submodules/s3 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/start b/start index d637f12..d5331fc 100755 --- a/start +++ b/start @@ -35,6 +35,7 @@ docker run -d --rm \ -e S3_ENDPOINT_LOCAL=object-storage:9000 \ -e S3_ACCESS_KEY=kern \ -e S3_SECRET_KEY=r6ywtR33!DMlaL*SUUdy \ +-e S3_USE_SSL=0 \ -e NEURAL_SEARCH=http://refinery-neural-search:80 \ -e COGNITION_GATEWAY=http://cognition-gateway:80 \ -e POSTGRES=postgresql://postgres:kern@graphql-postgres:5432 \ diff --git a/submodules/s3 b/submodules/s3 index 3299fb4..f8ac7e0 160000 --- a/submodules/s3 +++ b/submodules/s3 @@ -1 +1 @@ -Subproject commit 3299fb46876e3b4cc29c0a5cef004005a87f0f19 +Subproject commit f8ac7e0d76d5b019e5a9ff8dcbe50747fc9dee15 From a0a795638167ddd8b231fb5ad8362ef6ac7a0e9e Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Thu, 25 Sep 2025 09:16:33 +0200 Subject: [PATCH 02/12] chore: update submodules --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 4cdfbd2..74c6267 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 4cdfbd240114f22ba493d9a552b812499e0c5298 +Subproject commit 74c6267117ea5fc56823c5cd270ce8ea2622fc43 From e7750e5359392dd7b9784e1c47e58b221d4a648a Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Thu, 25 Sep 2025 09:16:43 +0200 Subject: [PATCH 03/12] perf: add telemetry --- app.py | 20 +++++++++++++++++++- start | 7 ++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 0e9f191..c1bf8af 100644 --- a/app.py +++ b/app.py @@ -1,11 +1,29 @@ +import os +import logging from fastapi import FastAPI, responses, status from fastapi.encoders import jsonable_encoder from pydantic import BaseModel from submodules.model.business_objects import general import util from submodules.model import session +from submodules.model import telemetry -app = FastAPI() + +OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317") + +app_name = "refinery-tokenizer" +app = FastAPI(title=app_name) + +if telemetry.ENABLE_TELEMETRY: + print("WARNING: Running telemetry.", flush=True) + telemetry.setting_otlp(app, app_name=app_name, endpoint=OTLP_GRPC_ENDPOINT) + app.add_middleware(telemetry.PrometheusMiddleware, app_name=app_name) + app.add_route("/metrics", telemetry.metrics) + + # Filter out /metrics + logging.getLogger("uvicorn.access").addFilter( + lambda record: "GET /metrics" not in record.getMessage() + ) @app.post("/update_to_newest") diff --git a/start b/start index d5331fc..c10edb8 100755 --- a/start +++ b/start @@ -2,11 +2,13 @@ DEBUG_MODE=false DEBUG_PORT=15674 +ENABLE_TELEMETRY=false -while getopts d flag +while getopts dg flag do case "${flag}" in d) DEBUG_MODE=true;; + g) ENABLE_TELEMETRY=true;; esac done @@ -39,9 +41,12 @@ docker run -d --rm \ -e NEURAL_SEARCH=http://refinery-neural-search:80 \ -e COGNITION_GATEWAY=http://cognition-gateway:80 \ -e POSTGRES=postgresql://postgres:kern@graphql-postgres:5432 \ +-e ENABLE_TELEMETRY=$ENABLE_TELEMETRY \ --mount type=bind,source="$(pwd)"/,target=/app \ -v /var/run/docker.sock:/var/run/docker.sock \ --network dev-setup_default \ +--log-driver=loki \ +--log-opt loki-url="http://$HOST_IP:3100/loki/api/v1/push" \ refinery-updater-dev $CMD > /dev/null 2>&1 echo -ne '\t\t\t [done]\n' From 7cbbee7907e02173f25e686e053d00843477a00e Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Thu, 25 Sep 2025 09:17:09 +0200 Subject: [PATCH 04/12] fix: app name --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index c1bf8af..3bdd748 100644 --- a/app.py +++ b/app.py @@ -11,7 +11,7 @@ OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317") -app_name = "refinery-tokenizer" +app_name = "refinery-updater" app = FastAPI(title=app_name) if telemetry.ENABLE_TELEMETRY: From 34afb04fc1cfcb9848710de1ac221226932ec32e Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Thu, 25 Sep 2025 14:42:17 +0200 Subject: [PATCH 05/12] chore: update submodules --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 74c6267..d6f2988 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 74c6267117ea5fc56823c5cd270ce8ea2622fc43 +Subproject commit d6f29884e9064948651c69b214bb64a030377b97 From 7af5a52cc27d0556a2aff7d528d4d65edb78126f Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Thu, 25 Sep 2025 14:49:15 +0200 Subject: [PATCH 06/12] chore: update telemetry import --- app.py | 3 +-- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 3bdd748..cf372e4 100644 --- a/app.py +++ b/app.py @@ -5,8 +5,7 @@ from pydantic import BaseModel from submodules.model.business_objects import general import util -from submodules.model import session -from submodules.model import telemetry +from submodules.model import session, telemetry OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317") diff --git a/requirements-dev.txt b/requirements-dev.txt index 9534895..e4fd425 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -r requirements.txt -httpx==0.25.0 +httpx==0.28.1 pytest==8.1.1 debugpy==1.8.12 \ No newline at end of file From f0c491bac10ff7dbc9e598285423fe1420c82d84 Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Thu, 25 Sep 2025 14:55:37 +0200 Subject: [PATCH 07/12] ci(fix): drone submodule update --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 650ce3f..ac21e0b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,7 +11,7 @@ steps: commands: - git submodule init - 'git config --global url."https://github.com/".insteadOf git@github.com:' - - "git submodule update --recursive --remote" + - "git submodule update --recursive" - name: build and publish image: plugins/docker settings: From 02c1b8ba3cfd8b4edde4855203429125cd229fe6 Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Thu, 25 Sep 2025 14:56:17 +0200 Subject: [PATCH 08/12] ci(fix): drone submodule update --- .drone.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index ac21e0b..d2ebb27 100644 --- a/.drone.yml +++ b/.drone.yml @@ -73,7 +73,7 @@ steps: commands: - git submodule init - 'git config --global url."https://github.com/".insteadOf git@github.com:' - - "git submodule update --recursive --remote" + - "git submodule update --recursive" - name: build and publish image: plugins/docker settings: @@ -109,7 +109,7 @@ steps: commands: - git submodule init - 'git config --global url."https://github.com/".insteadOf git@github.com:' - - "git submodule update --recursive --remote" + - "git submodule update --recursive" - name: build and publish image: plugins/docker settings: @@ -138,7 +138,7 @@ steps: commands: - git submodule init - 'git config --global url."https://github.com/".insteadOf git@github.com:' - - "git submodule update --recursive --remote" + - "git submodule update --recursive" - name: build and publish image: plugins/docker settings: From 2d1d0e0e8e98f422606381d6072aabc344c6a54e Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Wed, 15 Oct 2025 19:18:11 +0200 Subject: [PATCH 09/12] fix: hanging docker stop --- start | 2 -- 1 file changed, 2 deletions(-) diff --git a/start b/start index c10edb8..26f7b14 100755 --- a/start +++ b/start @@ -45,8 +45,6 @@ docker run -d --rm \ --mount type=bind,source="$(pwd)"/,target=/app \ -v /var/run/docker.sock:/var/run/docker.sock \ --network dev-setup_default \ ---log-driver=loki \ ---log-opt loki-url="http://$HOST_IP:3100/loki/api/v1/push" \ refinery-updater-dev $CMD > /dev/null 2>&1 echo -ne '\t\t\t [done]\n' From 7230d9ec4b15887570bf3c80d5a58d4e2cd7a0da Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Tue, 21 Oct 2025 14:38:00 +0200 Subject: [PATCH 10/12] chore: update submodules --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index d6f2988..687078b 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit d6f29884e9064948651c69b214bb64a030377b97 +Subproject commit 687078b4b48a86324d968b2116044845aad646b6 From 4453b6141ffaf40f141d668243c2d92a62df1306 Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Tue, 21 Oct 2025 14:38:06 +0200 Subject: [PATCH 11/12] perf: enhance monitoring --- app.py | 1 + daemon.py | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 daemon.py diff --git a/app.py b/app.py index cf372e4..c86292c 100644 --- a/app.py +++ b/app.py @@ -15,6 +15,7 @@ if telemetry.ENABLE_TELEMETRY: print("WARNING: Running telemetry.", flush=True) + telemetry.setting_app_name(app_name) telemetry.setting_otlp(app, app_name=app_name, endpoint=OTLP_GRPC_ENDPOINT) app.add_middleware(telemetry.PrometheusMiddleware, app_name=app_name) app.add_route("/metrics", telemetry.metrics) diff --git a/daemon.py b/daemon.py deleted file mode 100644 index d87da0e..0000000 --- a/daemon.py +++ /dev/null @@ -1,11 +0,0 @@ -import threading -from typing import Callable - - -def run(target: Callable, *args, **kwargs) -> None: - threading.Thread( - target=target, - args=args, - kwargs=kwargs, - daemon=True, - ).start() From 063374df57450da952bb30147a2ce5a9b9b9d76e Mon Sep 17 00:00:00 2001 From: andhreljaKern Date: Tue, 21 Oct 2025 16:10:35 +0200 Subject: [PATCH 12/12] chore(opentelemetry): update submodules --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 687078b..fa52e17 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 687078b4b48a86324d968b2116044845aad646b6 +Subproject commit fa52e1725d0691979895644d63c0b61728ea771b