Restructure project: move yam-indexing module to a dedicated repository

.dockerignore

@@ -0,0 +1,18 @@
+# Python
+__pycache__/
+*.pyc
+*.pyo
+
+# Virtual environments
+.venv/
+
+# Environment / secrets
+.env
+
+# Persistent data (DB)
+yam_indexing_db/
+*.db
+
+# Git
+.git/
+.gitignore

.env.example

@@ -0,0 +1,10 @@
+# SQLite database path in the yam-indexing application
+YAM_INDEXING_DB_PATH=../yam-indexing/yam_indexing_db/yam_events.db
+
+REALTOKENS_API_URL=https://api.realtoken.community/v1/token
+
+API_PORT_INTERNAL=5000
+
+# Telegram alerts [optional]
+TELEGRAM_ALERT_BOT_TOKEN=
+TELEGRAM_ALERT_GROUP_ID=

.github/workflows/branch.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - name: Set up Node.js
         uses: actions/setup-node@v3

.gitignore

@@ -7,21 +7,10 @@ playground*.py
 # Log files and directories
 logs/*
 !logs/.gitkeep
-!logs/api/
-!logs/api/.gitkeep
-!logs/indexing/
-!logs/indexing/.gitkeep
 
 # Ignore log files
 *.log
-*.log.*
-*.log.1
-*.log.2
-*.log.3
-*.log.4
-*.log.5
-
-config.json
+*.log*.*
 
 #venv
 venv/
@@ -37,17 +26,12 @@ UI/.env.local
 # Node modules
 node_modules/
 
-# Generated files
-yam-events-tracker/generated/
-yam-events-tracker/build/
-dist/
-.cache/
-.graphclient/
-
 # Database files
 *.db
 *.sqlite
 *.sqlite3
 
 # OS-specific
-.DS_Store
+.DS_Store
+
+.env

pdf_generator_module/api/app.py → API/core/app.py

@@ -2,44 +2,56 @@
 from flask_cors import CORS
 from .routes import api_bp
 from .services.realtokens_data import start_realtokens_updater
-from pdf_generator_module.logging.logging_config import setup_logging
+from API.logging.logging_config import setup_logging
+from API.logging.send_telegram_alert import send_telegram_alert
+from API.logging.shutdown_handler import install_signal_handlers
 import logging
 import json
 
+import os
+from dotenv import load_dotenv
+load_dotenv()
+RUNNING_IN_DOCKER = os.getenv("RUNNING_IN_DOCKER") == "1"
+
 def create_app():
 
-    # Set up logging at the start of your application
+    # Set up logging at the start of your application and handlers
     setup_logging()
+    install_signal_handlers()
 
     # Get a logger for this module
     logger = logging.getLogger(__name__)
     logger.info("Application started")
+    send_telegram_alert("yam transaction report generator: Application has started")
 
     app = Flask(__name__)
 
-        # Enable CORS for production - Allow all origins for public API
+    # Enable CORS for production - Allow all origins for public API
     CORS(app, 
          origins="*",
          methods=['GET', 'POST', 'OPTIONS'],
          allow_headers=['Content-Type', 'Authorization'],
          supports_credentials=False
     )
+
+    # Required configuration
+    if RUNNING_IN_DOCKER:
+         db_path = "yam_indexing_db/yam_events.db"
+    else:
+         db_path = os.environ["YAM_INDEXING_DB_PATH"]
+    app.config['YAM_INDEXING_DB_PATH'] = db_path
+    app.config['REALTOKENS_API_URL'] = os.environ['REALTOKENS_API_URL']
 
-    # Load configuration from config.json
-    with open('config.json', 'r') as config_file:
-        config = json.load(config_file)
-
-    # Configuration
-    app.config['DB_PATH'] = config['db_path']
-    app.config['API_PORT'] = config['api_port']
-    app.config['REALTOKENS_API_URL'] = config['realtokens_api_url']
+    # Optional configuration (with default)
+    app.config['API_PORT'] = int(os.getenv('API_PORT_INTERNAL', '5000'))
 
     try:
         with open('Ressources/blockchain_contracts.json', 'r') as contracts_file:
             app.config['BLOCKCHAIN_CONTRACTS'] = json.load(contracts_file)['contracts']
         logger.info("Blockchain contracts loaded successfully")
     except Exception as e:
         logger.error(f"Failed to load blockchain contracts: {e}")
+        send_telegram_alert(f"Failed to load blockchain contracts, please check blockchain_contracts.json: {e}")
         raise
 
     # Start RealTokens data service

pdf_generator_module/api/dev_run_api.py → API/core/dev_run_api.py

@@ -2,19 +2,17 @@
 
 ## For DEV mode only! NOT TO BE USED IN PRODUCTION ###
 
-import json
 import sys
+from API.core.app import create_app
+
 import os
-from pdf_generator_module.api.app import create_app
+from dotenv import load_dotenv
+load_dotenv()
 
 if __name__ == '__main__':
     try:
-        # Load configuration to get the port
-        with open('config.json', 'r') as config_file:
-            config = json.load(config_file)
-
         # Get port from config
-        port = config.get('api_port', 5000)  # Default to 5000 if not found
+        port = int(os.getenv('API_PORT_INTERNAL', '5000'))
 
         # Create the Flask app
         app = create_app()
@@ -23,12 +21,6 @@
         print(f"Starting API server on port {port}...")
         app.run(host='0.0.0.0', port=port, debug=True)
 
-    except FileNotFoundError:
-        print("Error: config.json file not found!")
-        sys.exit(1)
-    except json.JSONDecodeError:
-        print("Error: Invalid JSON in config.json!")
-        sys.exit(1)
     except KeyError as e:
         print(f"Error: Missing configuration key: {e}")
         sys.exit(1)

pdf_generator_module/api/routes.py → API/core/routes.py

@@ -4,8 +4,9 @@
 import io
 import logging
 import json
-from pdf_generator_module.query_db import get_accepted_offers_by_buyer_datetime, get_accepted_offers_by_seller_datetime
-from pdf_generator_module.print_pdf import create_report_elements, build_pdf
+from API.query_db import get_accepted_offers_by_buyer_datetime, get_accepted_offers_by_seller_datetime
+from API.print_pdf import create_report_elements, build_pdf
+from API.logging.send_telegram_alert import send_telegram_alert
 
 # Get logger for this module
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ def generate_report():
         blockchain_contracts = current_app.config['BLOCKCHAIN_CONTRACTS']
         realtokens = current_app.config['REALTOKENS']
 
-        events_seller = get_accepted_offers_by_seller_datetime(current_app.config['DB_PATH'], user_addresses, start_date, end_date)
-        events_buyer = get_accepted_offers_by_buyer_datetime(current_app.config['DB_PATH'], user_addresses, start_date, end_date)
+        events_seller = get_accepted_offers_by_seller_datetime(current_app.config['YAM_INDEXING_DB_PATH'], user_addresses, start_date, end_date)
+        events_buyer = get_accepted_offers_by_buyer_datetime(current_app.config['YAM_INDEXING_DB_PATH'], user_addresses, start_date, end_date)
 
         # Format dates for display
         from_datetime_formatted_string = datetime.fromisoformat(start_date.replace('Z', '+00:00')).strftime("%d %B %Y").lstrip('0')
@@ -108,7 +109,7 @@ def generate_report():
 
     except Exception as e:
         logger.error(f"Error generating report: {str(e)}")
-        current_app.logger.error(f"Error generating report: {str(e)}")
+        send_telegram_alert(f"Error generating report: {str(e)}")
         return jsonify({'error': f'Internal server error occurred while generating report: {e}'}), 400
 
 @api_bp.route('/health', methods=['GET'])

pdf_generator_module/api/services/realtokens_data.py → API/core/services/realtokens_data.py

@@ -3,6 +3,7 @@
 import time
 import logging
 from eth_utils import to_checksum_address, is_address
+from API.logging.send_telegram_alert import send_telegram_alert
 
 # Get logger for this module
 logger = logging.getLogger(__name__)
@@ -43,6 +44,7 @@ def start_realtokens_updater(app):
         logger.info("RealTokens service initialized successfully")
     else:
         logger.error("Failed to initialize RealTokens service - cannot start application")
+        send_telegram_alert("Failed to initialize RealTokens service - cannot start application")
         raise Exception("Failed to load initial RealTokens data - cannot start application without this data")
 
     def update_realtokens_periodically():
@@ -56,6 +58,7 @@ def update_realtokens_periodically():
                     logger.info("RealTokens data updated successfully (24h periodic update)")
                 else:
                     logger.error("Failed to update RealTokens data (24h periodic update)")
+                    send_telegram_alert("Failed to update RealTokens data (24h periodic update)")
 
     # Start background thread
     update_thread = threading.Thread(target=update_realtokens_periodically, daemon=True)

pdf_generator_module/logging/logging_config.py → API/logging/logging_config.py

@@ -6,14 +6,14 @@
 
 def setup_logging():
 
-    os.makedirs("logs/api", exist_ok=True)
+    os.makedirs("logs", exist_ok=True)
 
     logging.basicConfig(
         level=logging.INFO,
         format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
         handlers=[
             RotatingFileHandler(
-                'logs/api/app.log', 
+                'logs/app_yam-report.log', 
                 maxBytes=10*1024*1024,  # 10MB per file
                 backupCount=5,          # Keep 5 backup files
                 encoding='utf-8'

API/logging/send_telegram_alert.py

@@ -0,0 +1,42 @@
+import os
+import requests
+from dotenv import load_dotenv
+
+load_dotenv()
+BOT_TOKEN = os.getenv("TELEGRAM_ALERT_BOT_TOKEN")
+GROUP_ID = os.getenv("TELEGRAM_ALERT_GROUP_ID")
+
+TELEGRAM_MAX_LEN = 4096
+
+def escape_markdown_v2(text: str) -> str:
+    # Caractères à échapper en MarkdownV2 (spec Telegram)
+    special = r'_\*\[\]\(\)~`>#+\-=|{}.!'
+    out = []
+    for ch in str(text):
+        if ch in special:
+            out.append("\\" + ch)
+        else:
+            out.append(ch)
+    return "".join(out)
+
+def send_telegram_alert(message, group_id=GROUP_ID, bot_token=BOT_TOKEN):
+    if not bot_token or not group_id:
+        return None
+
+    url = f"https://api.telegram.org/bot{bot_token}/sendMessage"
+
+    msg = str(message)
+    if len(msg) > TELEGRAM_MAX_LEN:
+        msg = msg[:4000] + "\n…(truncated)…"
+
+    msg = escape_markdown_v2(msg)
+
+    payload = {
+        "chat_id": group_id,
+        "text": msg,
+        "disable_web_page_preview": True,
+        "parse_mode": "MarkdownV2",
+    }
+
+    resp = requests.post(url, json=payload, timeout=10)
+    return resp

API/logging/shutdown_handler.py

@@ -0,0 +1,35 @@
+import signal
+import threading
+import os
+import logging
+from API.logging.send_telegram_alert import send_telegram_alert
+
+
+_shutdown_notified = False
+_lock = threading.Lock()
+
+logger = logging.getLogger(__name__)
+
+def notify_shutdown(reason: str):
+    global _shutdown_notified
+    with _lock:
+        if _shutdown_notified:
+            return
+        _shutdown_notified = True
+    logger.info(f"API has been stopped: {reason}")
+    send_telegram_alert(f"The YAM report PDF generator API has been stopped: {reason}")
+
+    # On Windows, if we don't re-raise the interruption, Waitress may keep running.
+    if os.name == "nt":
+        raise KeyboardInterrupt
+
+
+def _handle_sigterm(signum, frame):
+    notify_shutdown("docker stop")
+
+def _handle_sigint(signum, frame):
+    notify_shutdown("ctrl+c")
+
+def install_signal_handlers():
+    signal.signal(signal.SIGTERM, _handle_sigterm)
+    signal.signal(signal.SIGINT, _handle_sigint)

pdf_generator_module/print_pdf/create_report_elements.py → API/print_pdf/create_report_elements.py

@@ -1,5 +1,5 @@
-from pdf_generator_module.print_pdf.internals._utils import _get_report_parameter_section, _format_number, _format_timestamp, _aggregate_total_row
-from pdf_generator_module.print_pdf.internals._style import _get_title_style, _get_user_addresses_style, _get_event_type_subtitle_style, _get_link_style, _get_header_style, _get_common_style, _get_buy_sell_table_style, _get_columns_width_buy_sell_table, _get_columns_width_exchange_table, _get_exchange_table_style
+from API.print_pdf.internals._utils import _get_report_parameter_section, _format_number, _format_timestamp, _aggregate_total_row
+from API.print_pdf.internals._style import _get_title_style, _get_user_addresses_style, _get_event_type_subtitle_style, _get_link_style, _get_header_style, _get_common_style, _get_buy_sell_table_style, _get_columns_width_buy_sell_table, _get_columns_width_exchange_table, _get_exchange_table_style
 from reportlab.platypus import Paragraph, Spacer, Table, TableStyle, PageBreak
 from reportlab.lib.pagesizes import A4
 from reportlab.lib.units import cm

Dockerfile-api

@@ -1,4 +1,4 @@
-FROM python:3.9-slim
+FROM python:3.11-slim
 
 # Créer et définir le répertoire de travail
 WORKDIR /app
@@ -12,15 +12,9 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Copier le reste du code
 COPY . .
 
-# Initialiser la base de données
-# RUN python3 -m yam_indexing_module.initialize_indexing_module
-
 # Exposer les ports nécessaires
 EXPOSE 5000
 
-# Créer un volume pour la base de données
-# VOLUME ["/app/YAM_events.db"]
-
 # Script d'entrée pour démarrer les services
 COPY docker-entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh