[#221] non-distributed optimization algorithm

.gitignore

@@ -196,3 +196,6 @@ tags
 
 src/bin
 src/bazel**
+
+.nogit/
+.vscode/

Makefile

@@ -49,6 +49,9 @@ run-inference-agent:
 run-inference-agent-client:
 	@bash -x src/scripts/run.sh inference_agent_client $(OPTIONS)
 
+run-evolution:
+	src/bin/evolution $(OPTIONS)
+
 setup-nunet-dms:
 	@bash -x src/scripts/setup-nunet-dms.sh
 

src/evolution/BUILD

@@ -0,0 +1,57 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("@pypi//:requirements.bzl", "requirement")
+
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "py_files",
+    srcs = glob(["*.py"]),
+)
+
+filegroup(
+    name = "py_all_files",
+    srcs = glob(["**/*.py"]),
+)
+
+py_library(
+    name = "evolution",
+    srcs = [":py_all_files"],
+    deps = [
+        ":fitness_functions",
+        ":selection_methods",
+        ":utils",
+        ":optimizer",
+        "//evolution/das_node",
+    ],
+)
+
+py_library(
+    name = "fitness_functions",
+    srcs = ["fitness_functions.py"],
+    deps = [],
+)
+
+py_library(
+    name = "selection_methods",
+    srcs = ["selection_methods.py"],
+    deps = [],
+)
+
+py_library(
+    name = "utils",
+    srcs = ["utils.py"],
+    deps = [],
+)
+
+py_library(
+    name = "optimizer",
+    srcs = ["optimizer.py"],
+    deps = [
+        ":fitness_functions",
+        ":selection_methods",
+        ":utils",
+        "//evolution/das_node:das_node",
+        "//evolution/das_node:query_answer",
+    ],
+)

src/evolution/Dockerfile

@@ -0,0 +1,33 @@
+FROM ubuntu:22.04
+
+ARG DAS_QUERY_ENGINE_BRANCH="masc/change-service-in-grpc"
+
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3.10-distutils \
+    python3.10-venv \
+    python3-pip \
+    binutils \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
+
+WORKDIR /app
+
+ENV PYTHONPATH=/app
+
+RUN apt-get update && apt-get install -y git
+
+RUN git clone https://github.com/singnet/das-query-engine.git
+RUN cd das-query-engine && git checkout ${DAS_QUERY_ENGINE_BRANCH} &&  mv hyperon_das/ ../hyperon_das
+RUN pip3 install --no-cache --upgrade pip hyperon_das_atomdb requests grpcio google protobuf pyinstaller
+
+COPY asset/hyperon_das_node-0.0.1-cp310-abi3-manylinux_2_28_x86_64.whl /app/
+RUN pip3 install /app/hyperon_das_node-0.0.1-cp310-abi3-manylinux_2_28_x86_64.whl
+
+COPY . /app/evolution
+
+COPY build.sh .
+RUN chmod +x build.sh
+
+CMD ["./build.sh"]

src/evolution/README.md

@@ -0,0 +1,56 @@
+# Non Distributed Query Optimizer
+
+## How to build
+
+To build run the following command from the project root:
+
+```bash
+make build-all
+```
+
+This will generate the binary file in the `das/src/bin` directory.
+
+## Usage
+
+Run the optimizer using the main script. The command-line arguments specify the configuration file and the query tokens to optimize.
+
+#### Config file example
+
+```
+mongo_hostname = "localhost"
+mongo_port = 27017
+mongo_username = "root"
+mongo_password = "root"
+redis_hostname = "localhost"
+redis_port = 6379
+redis_cluster = False
+redis_ssl = False
+query_agent_node_id = "localhost:31701"
+query_agent_server_id = "localhost:31700"
+attention_broker_server_id = "localhost:37007"
+max_generations = 2
+max_query_answers = 5
+population_size = 500
+qtd_selected_for_attention_update = 100
+selection_method = "roulette"
+fitness_function = "multiply_strengths"
+```
+
+#### Running client:
+
+```bash
+make run-evolution OPTIONS='--config-file /path/to/config.cfg --query-tokens "LINK_TEMPLATE Evaluation 2 NODE Type Name VARIABLE V1"'
+```
+
+**Parameters:**
+--query-tokens: The query string that will be optimized.
+--config-file: Path to the configuration file (default is config.cfg).
+
+If successful, you should see a message like this:
+
+```
+Starting optimizer
+Processing generation 1/n
+Results:
+['QueryAnswer<1,1> [296bfeeb2ce5148d78f371d0ddf395b2] {(V2: f9a98ccf36f4ba3dcfe2fc99243546fa)} 0.0029751847', 'QueryAnswer<1,1> [2f21f35e3936307c29367adf41aec59e] {(V2: a7d045ace9ea9f9ecbc9094a770cae50)} 0.0025941788', 'QueryAnswer<1,1> [9d6fe9c68e5b29a1b4616ef582e075a3] {(V2: c04cafa6ca7f157321547f4c9ff4bb39)} 0.0025350566', 'QueryAnswer<1,1> [15e8247142c5a46b6079d9df9ea61833] {(V2: f54acd64cd8541c2125588e84da17288)} 0.0024081622', 'QueryAnswer<1,1> [a9335106b2ab5e652749769b72b9e29c] {(V2: 074bd74b5b8c2c87777bf57696ec3edd)} 0.0023660637']
+```

src/evolution/build.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+pyinstaller --onefile ./evolution/main.py
+mkdir -p bin
+mv ./dist/main ./dist/evolution
+mv ./dist/evolution ./bin/evolution
+rm -rf build/ __pycache__

src/evolution/config.example.cfg

@@ -0,0 +1,19 @@
+context = "das-poc"
+mongo_hostname = "localhost"
+mongo_port = 27017
+mongo_username = "root"
+mongo_password = "root"
+redis_hostname = "localhost"
+redis_port = 6379
+redis_cluster = False
+redis_ssl = False
+query_agent_node_id = "localhost:31701"
+query_agent_server_id = "localhost:31700"
+attention_broker_server_id = "localhost:37007"
+number_nodes = 5
+max_generations = 2
+max_query_answers = 30
+population_size = 500
+qtd_selected_for_attention_update = 100
+selection_method = "roulette"
+fitness_function = "multiply_strengths"

src/evolution/das_node/BUILD

@@ -0,0 +1,69 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("@pypi//:requirements.bzl", "requirement")
+
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "py_files",
+    srcs = glob(["*.py"]),
+)
+
+py_library(
+    name = "das_node",
+    srcs = ["das_node.py"],
+    deps = [
+        ":remote_iterator",
+        ":star_node",
+    ],
+)
+
+py_library(
+    name = "query_answer",
+    srcs = ["query_answer.py"],
+    deps = [],
+)
+
+py_library(
+    name = "query_element",
+    srcs = ["query_element.py"],
+    deps = [],
+)
+
+py_library(
+    name = "query_node",
+    srcs = ["query_node.py"],
+    deps = [
+        ":query_answer",
+        ":shared_queue"
+    ],
+)
+
+py_library(
+    name = "remote_iterator",
+    srcs = ["remote_iterator.py"],
+    deps = [
+        ":query_answer",
+        ":query_element",
+        ":query_node",
+    ],
+)
+
+py_library(
+    name = "shared_queue",
+    srcs = ["shared_queue.py"],
+    deps = [],
+)
+
+
+py_library(
+    name = "star_node",
+    srcs = ["star_node.py"],
+    deps = [],
+)
+
+
+
+
+
+

src/evolution/das_node/das_node.py

@@ -0,0 +1,57 @@
+from hyperon_das_node import Message
+
+from evolution.das_node.remote_iterator import RemoteIterator
+from evolution.das_node.star_node import StarNode
+
+
+class DASNode(StarNode):
+    local_host: str
+    next_query_port: int
+    first_query_port: int
+    last_query_port: int
+    PATTERN_MATCHING_QUERY = "pattern_matching_query"
+
+    def __init__(self, node_id: str = None, server_id: str = None):
+        super().__init__(node_id, server_id)
+        self.initialize()
+
+    def pattern_matcher_query(
+        self, tokens: list, context: str = "", update_attention_broker: bool = False
+    ):
+        if self.is_server:
+            raise ValueError("pattern_matcher_query() is not available in DASNode server.")
+        query_id = self.next_query_id()
+        args = [query_id, context, "true" if update_attention_broker else "false"] + tokens
+        self.send(DASNode.PATTERN_MATCHING_QUERY, args, self.server_id)
+        return RemoteIterator(query_id)
+
+    def next_query_id(self) -> str:
+        port = self.next_query_port
+        limit = 0
+        if self.is_server:
+            limit = (self.first_query_port + self.last_query_port) // 2 - 1
+            if self.next_query_port > limit:
+                self.next_query_port = self.first_query_port
+        else:
+            limit = self.last_query_port
+            if self.next_query_port > limit:
+                self.next_query_port = (self.first_query_port + self.last_query_port) // 2
+
+        query_id = f"{self.local_host}:{port}"
+        self.next_query_port += 1
+        return query_id
+
+    def message_factory(self, command: str, args: list) -> Message:
+        message = super().message_factory(command, args)
+        if message:
+            return message
+        return None
+
+    def initialize(self):
+        self.first_query_port = 60000
+        self.last_query_port = 61999
+        self.local_host = self.node_id().split(":")[0]  # Extracting the host part of node_id
+        if self.is_server:
+            self.next_query_port = self.first_query_port
+        else:
+            self.next_query_port = (self.first_query_port + self.last_query_port) // 2