Merge pull request #5 from sensein/dev

StructSense

Author: tekrajchhetri

.gitignore

@@ -1,17 +1,20 @@
 # Ruff
 .ruff_cache
-
+config/agents.yaml
 # Mac
 .DS_Store
-
+crew_memory/
+.venv/
+.*/*
+.ipynb_checkpoints
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
-
+example/.structsense_env
 # C extensions
 *.so
-
+.chainlit/
 # Distribution / packaging
 .Python
 build/
@@ -37,7 +40,7 @@ MANIFEST
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
-
+*.env
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include src/structsense/default_config_ner *.yaml

README.md

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------------
+# DISCLAIMER: This software is provided "as is" without any warranty,
+# express or implied, including but not limited to the warranties of
+# merchantability, fitness for a particular purpose, and non-infringement.
+#
+# In no event shall the authors or copyright holders be liable for any
+# claim, damages, or other liability, whether in an action of contract,
+# tort, or otherwise, arising from, out of, or in connection with the
+# software or the use or other dealings in the software.
+# -----------------------------------------------------------------------------
+
+# @Author  : Tek Raj Chhetri
+# @Email   : tekraj@mit.edu
+# @Web     : https://tekrajchhetri.com/
+# @File    : __init__.py.py
+# @Software: PyCharm

__init__.py

@@ -0,0 +1,130 @@
+#agent config
+# Do not introduce any new variable name
+# the variables inside {} receives data from the previous agent. the {literature} variable is the one that passes the
+# initial input.
+agent_config:
+  extractor_agent:
+    role: >
+      agent role
+    goal: >
+      goal
+    backstory: >
+     agent back story
+    llm:
+      model: openrouter/openai/gpt-4o-mini
+      base_url: https://openrouter.ai/api/v1
+
+  alignment_agent:
+    role: >
+      agent role
+    goal: >
+      agent goal
+    backstory: >
+      agent back story
+    llm:
+      model: openrouter/openai/gpt-4o-mini
+      base_url: https://openrouter.ai/api/v1
+
+  judge_agent:
+    role: >
+      agent role
+    goal: >
+      goal
+    backstory: >
+      agent back story
+    llm:
+      model: openrouter/openai/gpt-4o-mini
+      base_url: https://openrouter.ai/api/v1
+
+  humanfeedback_agent:
+    role: >
+     agent role
+    goal: >
+      agent goal
+    backstory: >
+      agent back story
+    llm:
+      model: openrouter/openai/gpt-4o-mini
+      base_url: https://openrouter.ai/api/v1 
+
+# agent task config
+task_config:
+  extraction_task:
+    description: >
+      #add task description
+      
+      Input: {literature}
+   
+
+    expected_output: >
+      output format: json
+      Example output: add example
+
+    # do not chage this, remove it later
+    agent_id: extractor_agent
+
+  alignment_task:
+    description: > 
+      Inputs: {extracted_structured_information}
+      
+      Instructions:
+       
+
+    expected_output: >
+      output format: json
+      Example output: add example
+
+    # do not chage this, remove it later
+    agent_id: alignment_agent
+
+  judge_task:
+    description: >
+      {aligned_structured_information} 
+         
+      Instuctions: add instructions
+
+    expected_output: > 
+      output format: json 
+      Example output: addd example
+
+    # do not chage this, remove it later
+    agent_id: judge_agent
+
+  humanfeedback_task:
+    description: >
+      Input: {judged_structured_information_with_human_feedback}
+
+      modification_context: 
+        {modification_context}
+      
+      user_feedback_text:
+        {user_feedback_text}
+      
+      Instructions:
+       add you instructions
+      
+  
+
+    expected_output: > 
+      output format: json
+      Example output: addd example
+
+    agent_id: humanfeedback_agent
+
+# embedding config
+# see for more details and parameters for config
+# https://docs.crewai.com/concepts/memory#additional-embedding-providerscl
+embedder_config:
+  provider: ollama
+  config:
+    api_base: http://localhost:11434
+    model: nomic-embed-text:latest
+
+# knowledge search config
+knowledge_config:
+  search_key: #local vector database
+    - entity
+    - label
+# human in loop config
+human_in_loop_config:
+  humanfeedback_agent: true

config/config.yaml

@@ -0,0 +1,8 @@
+services:
+  grobid:
+    image: lfoppiano/grobid:0.8.0
+    ports:
+      - "8070:8070"
+    environment:
+      JAVA_OPTS: "-XX:+UseZGC"
+    init: true

config_template/config.yaml

@@ -0,0 +1,20 @@
+services:
+  ollama:
+    image: ollama/ollama
+    container_name: ollama
+    restart: unless-stopped
+    ports:
+      - "11434:11434"
+    volumes:
+      - ollama_models:/root/.ollama
+    entrypoint: >
+      sh -c "
+      ollama serve & 
+      sleep 5 &&
+      ollama pull nomic-embed-text &&
+      ollama pull deepseek-r1:7b &&
+      tail -f /dev/null
+      "
+
+volumes:
+  ollama_models:

docker/individual/grobid-service/docker-compose.yaml

@@ -0,0 +1,22 @@
+## 🔧 Setting Up
+
+1. **Install Docker**  
+   If Docker is not installed on your system, [download and install it](https://docs.docker.com/get-docker/).
+
+2. **Start the Container**  
+   From the project directory, run:
+   ```bash
+   docker compose up
+3. **Test the Setup**
+    Use the following curl command to verify that the model is responding correctly:
+    ```bash
+    curl -s http://localhost:11434/api/chat \
+      -H "Content-Type: application/json" \
+      -d '{
+        "model": "deepseek-r1:7b",
+        "messages": [
+          { "role": "user", "content": "Why is the sky blue?" }
+        ]
+      }' | jq -r 'select(.message.role=="assistant") | .message.content' | tr -d "\n"
+    
+    ```

docker/individual/ollama/docker-compose.yaml

@@ -0,0 +1,34 @@
+services:
+  weaviate:
+    command:
+      - --host
+      - 0.0.0.0
+      - --port
+      - '8080'
+      - --scheme
+      - http
+    image: cr.weaviate.io/semitechnologies/weaviate:1.29.1
+    ports:
+      - 8080:8080
+      - 50051:50051
+    volumes:
+      - weaviate_data:/var/lib/weaviate
+      - weaviate_backups:/backups  # Add this line
+    restart: on-failure:0
+    environment:
+      QUERY_DEFAULTS_LIMIT: 25
+      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+      BACKUP_FILESYSTEM_PATH: '/backups'  # Add this line
+      ENABLE_MODULES: 'text2vec-ollama,generative-ollama'
+      ENABLE_API_BASED_MODULES: 'true'
+      CLUSTER_HOSTNAME: 'node1'
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false'
+      AUTHENTICATION_APIKEY_ENABLED: 'true'
+      AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'user-a-key,user-b-key'
+      AUTHENTICATION_APIKEY_USERS: 'user-a,user-b'
+      AUTHORIZATION_ENABLE_RBAC: 'true'
+      AUTHORIZATION_RBAC_ROOT_USERS: 'user-a'
+
+volumes:
+  weaviate_data:
+  weaviate_backups: