Merge pull request #751 from DocArmoryTech/poetic-website

migrate website to Poetry and simplify arch.

Author: DavidCruciani

etc/logrotate.d/misp-modules

@@ -0,0 +1,10 @@
+/var/log/misp-modules*.log {
+	daily
+	missingok
+	rotate 14
+	compress
+	delaycompress
+	notifempty
+	copytruncate
+	create 0640 root root
+}

etc/systemd/system/misp-modules-website.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=MISP modules website
+After=network.target misp-modules.service
+Requires=misp-modules.service
+
+[Service]
+Type=simple
+User=misp
+Group=misp
+WorkingDirectory=/home/misp/misp-modules/website
+Environment="PATH=/home/misp/.local/bin:/usr/bin"
+EnvironmentFile=/home/misp/misp-modules/website/.env
+ExecStart=/home/misp/.local/bin/poetry run gunicorn --workers 4 --bind ${FLASK_URL}:${FLASK_PORT} --access-logfile - --error-logfile - "main:app"
+Restart=always
+RestartSec=10
+StandardOutput=append:/var/log/misp-modules-website_message.log
+StandardError=append:/var/log/misp-modules-website_error.log
+
+[Install]
+WantedBy=multi-user.target

etc/systemd/system/misp-modules.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=MISP modules
+After=network.target
+
+[Service]
+Type=simple
+User=misp
+Group=misp
+WorkingDirectory=/home/misp/misp-modules
+Environment="PATH=/home/misp/.local/bin:/usr/bin"
+ExecStart=/usr/bin/env poetry run misp-modules -l 127.0.0.1
+Restart=always
+RestartSec=10
+StandardOutput=append:/var/log/misp-modules_message.log
+StandardError=append:/var/log/misp-modules_error.log
+
+[Install]
+WantedBy=multi-user.target

website/.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude = 
+    migrations/,
+    app/static/
+
+per-file-ignores = 
+    __init__.py: F401

website/.gitignore

@@ -0,0 +1,8 @@
+# Ignore Python byte‑code cache
+__pycache__/
+
+# Ignore the .env file
+.env
+
+# Ignore everything in instance/
+instance/

website/README.md

@@ -1,54 +1,165 @@
-# MISP-module website
+# MISP Modules Website
 
-Use all modules with a dedicate website without any MISP
+Use all MISP modules through a dedicated website without requiring a MISP instance.
 
-![home](https://github.com/MISP/misp-modules/blob/main/website/doc/home_misp_module.png?raw=true)
+![Home](https://github.com/MISP/misp-modules/blob/main/website/doc/home_misp_module.png?raw=true)
 
-![query](https://github.com/MISP/misp-modules/blob/main/website/doc/query_misp_module.png?raw=true)
+![Query](https://github.com/MISP/misp-modules/blob/main/website/doc/query_misp_module.png?raw=true)
 
 ## Installation
 
-**It is strongly recommended to use a virtual environment**
+The MISP Modules website uses [Poetry](https://python-poetry.org/) for dependency management. It is recommended to install dependencies in a virtual environment managed by Poetry.
 
-If you want to know more about virtual environments, [python has you covered](https://docs.python.org/3/tutorial/venv.html)
+### Prerequisites
+- Python 3.8 or higher
+- Poetry
+- `misp-modules` installed in the parent directory (`../`)
 
-```bash
-sudo apt-get install screen -y
-pip install -r requirements.txt
-git submodule init && git submodule update   ## Initialize misp-objects submodule
-python3 app.py -i                            ## Initialize db
-```
+### Steps
+1. **Clone the Repository**:
+   ```bash
+   git clone https://github.com/MISP/misp-modules.git
+   cd misp-modules/website
+   ```
 
-Don't forget to install **misp-modules**...
+2. **Initialize Submodules**:
+   ```bash
+   git submodule init
+   git submodule update  # Initialize misp-objects submodule
+   ```
 
-## Config
+3. **Install Dependencies**:
+   ```bash
+   poetry install
+   ```
 
-Edit `config.py` 
+4. **Initialize the Database**:
+   ```bash
+   poetry run db-init
+   ```
+   This creates the database (`misp-module.sqlite`), initializes modules, and sets up the admin password (generated in development if not set).
 
-- `SECRET_KEY`: Secret key for the app
+5. **Install `misp-modules`**:
+   Ensure `misp-modules` is installed in the parent directory (`../misp-modules`). Follow the main repository’s instructions for setup.
 
-- `FLASK_URL` : url for the instance
+## Configuration
 
-- `FLASK_PORT`: port for the instance
+Configuration is managed via a `.env` file in `website/`. Copy the example and edit as needed:
 
-- `MISP_MODULE`: url and port where misp-module is running
+```bash
+cp .env.example .env
+nano .env
+```
 
-- `ADMIN_USER`: If True, config page will not be accessible
+### `.env` Settings
+- `DATABASE_URI`: Database URL (default: `sqlite:///misp-module.sqlite`).
+- `SECRET_KEY`: Secure key for the Flask app (generate with `python -c "import secrets; print(secrets.token_hex(16))"` or `openssl rand -hex 16`).
+- `FLASK_URL`: Host for the website (default: `127.0.0.1`).
+- `FLASK_PORT`: Port for the website (default: `7008`).
+- `MISP_MODULE`: URL and port of `misp-modules` (default: `127.0.0.1:6666`).
+- `ADMIN_PASSWORD`: Admin user password (optional in development, required in production).
+- `QUERIES_LIMIT`: Maximum queries allowed (default: `100`).
+- `SESSION_TYPE`: Session storage type (default: `sqlalchemy`).
+- `SESSION_SQLALCHEMY_TABLE`: Session table name (default: `flask_sessions`).
+- `FLASK_APP`: Flask entry point (default: `main`).
+
+Example `.env`:
+```
+DATABASE_URI=sqlite:///misp-module.sqlite
+SECRET_KEY=your-secure-secret-key
+FLASK_URL=127.0.0.1
+FLASK_PORT=7008
+MISP_MODULE=127.0.0.1:6666
+QUERIES_LIMIT=100
+SESSION_TYPE=sqlalchemy
+SESSION_SQLALCHEMY_TABLE=flask_sessions
+FLASK_APP=main
+# ADMIN_PASSWORD=your-admin-password  # Uncomment and set for production
+```
 
-- `ADMIN_PASSWORD`: Password for Admin user if `ADMIN_USER` is True
+## Launch
 
-Rename `config.cfg.sample` to `config.cfg` then edit it:
+### Development
+Run both `misp-modules` and the website in development mode with debug enabled:
 
-- `ADMIN_USER`: If True, config page will not be accessible
+```bash
+poetry run dev-site
+```
 
-- `ADMIN_PASSWORD`: Password for Admin user if `ADMIN_USER` is True
+- If `ADMIN_PASSWORD` is unset in `.env`, a random 20-character password is generated and printed.
+- Access the website at `http://127.0.0.1:7008` (or as configured).
 
-## Launch
+### Production
+Use systemd services for production deployment (see **Systemd Services** below). Ensure `ADMIN_PASSWORD` is set in `.env` to avoid startup errors.
+
+## Admin User
+
+If `ADMIN_PASSWORD` is set in `.env`, the admin user is active. Access the login page at `/login` and use the password from `.env` (or the generated password in development).
+
+- **Development**: If `ADMIN_PASSWORD` is unset, a password is generated and printed to the console.
+- **Production**: `ADMIN_PASSWORD` must be set in `.env`, or the application will fail to start with an error.
+
+## Database Management
+
+Manage the database with the following commands:
 
 ```bash
-./launch.sh -l
+poetry run db-init      # Initialize database and modules
+poetry run db-migrate   # Generate a new migration
+poetry run db-upgrade   # Apply migrations
+poetry run db-downgrade # Revert the latest migration
 ```
 
-## Admin user
+## Systemd Services
+
+Template systemd service files are provided in `etc/` for `misp-modules` and the website.
+
+### Installation
+1. **Copy Service Files**:
+   ```bash
+   sudo cp website/etc/misp-modules.service /etc/systemd/system/
+   sudo cp website/etc/misp-modules-website.service /etc/systemd/system/
+   ```
+
+2. **Reload Systemd**:
+   ```bash
+   sudo systemctl daemon-reload
+   ```
+
+3. **Enable and Start Services**:
+   ```bash
+   sudo systemctl enable misp-modules.service
+   sudo systemctl enable misp-modules-website.service
+   sudo systemctl start misp-modules.service
+   sudo systemctl start misp-modules-website.service
+   ```
+
+4. **Check Status**:
+   ```bash
+   sudo systemctl status misp-modules.service
+   sudo systemctl status misp-modules-website.service
+   ```
+
+Logs are written to `/var/log/misp-modules_*.log` and `/var/log/misp-modules-website_*.log`.
+
+## Log Rotation
+
+Log rotation configurations are provided in `etc/logrotate.d/` to manage service logs.
+
+### Installation
+1. **Copy Logrotate Files**:
+   ```bash
+   sudo cp etc/logrotate.d/misp-modules /etc/logrotate.d/
+   ```
+
+2. **Test Logrotate**:
+   ```bash
+   sudo logrotate -d /etc/logrotate.d/misp-modules
+   ```
+
+Logs are rotated daily, compressed, and retained for 7 days.
 
-If admin user is active, type `/login` in url to access a login page and type the password wrote in `config.py` in `ADMIN_PASSOWRD`.
+## Notes
+- Ensure `misp-modules` is installed and running in `../misp-modules`.
+- Set a secure `ADMIN_PASSWORD` in `.env` for production.
+- Adjust `.service` and `logrotate.d` paths or user settings for your environment.

website/app/__init__.py

@@ -1,28 +1,45 @@
-from flask import Flask
-from flask_sqlalchemy import SQLAlchemy
-from flask_wtf import CSRFProtect
+#!/usr/bin/env python3
+import os
+
+from flask import Flask, render_template
+from flask_login import LoginManager
 from flask_migrate import Migrate
 from flask_session import Session
-from flask_login import LoginManager
-
-from conf.config import config as Config
-import os
+from flask_sqlalchemy import SQLAlchemy
+from flask_wtf import CSRFProtect
 
+from dotenv import load_dotenv
+load_dotenv()
 
 db = SQLAlchemy()
 csrf = CSRFProtect()
 migrate = Migrate()
 session = Session()
 login_manager = LoginManager()
 
+
 def create_app():
     app = Flask(__name__)
-    config_name = os.environ.get("FLASKENV")
 
-    app.config.from_object(Config[config_name])
+    # Configure app from environment variables
+    app.config["SQLALCHEMY_DATABASE_URI"] = os.getenv(
+        "DATABASE_URI", "sqlite:///misp-module.sqlite"
+    )
+    app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+    app.config["SESSION_TYPE"] = os.getenv("SESSION_TYPE", "sqlalchemy")
+    app.config["SESSION_SQLALCHEMY_TABLE"] = os.getenv(
+        "SESSION_SQLALCHEMY_TABLE", "flask_sessions"
+    )
+    app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
+    app.config["FLASK_URL"] = os.getenv("FLASK_URL", "127.0.0.1")
+    app.config["FLASK_PORT"] = int(os.getenv("FLASK_PORT", "7008"))
+    app.config["MISP_MODULE"] = os.getenv("MISP_MODULE", "127.0.0.1:6666")
 
-    Config[config_name].init_app(app)
+    # Validate critical settings
+    if not app.config["SECRET_KEY"]:
+        raise ValueError("SECRET_KEY must be set in .env")
 
+    # Initialize extensions
     db.init_app(app)
     csrf.init_app(app)
     migrate.init_app(app, db, render_as_batch=True)
@@ -31,15 +48,21 @@ def create_app():
     login_manager.login_view = "account.login"
     login_manager.init_app(app)
 
-    from .home import home_blueprint
-    from .history.history import history_blueprint
+    # Register blueprints
     from .account.account import account_blueprint
     from .external_tools.external_tools import external_tools_blueprint
+    from .history.history import history_blueprint
+    from .home import home_blueprint
+
     app.register_blueprint(home_blueprint, url_prefix="/")
     app.register_blueprint(history_blueprint, url_prefix="/")
     app.register_blueprint(account_blueprint, url_prefix="/")
     app.register_blueprint(external_tools_blueprint, url_prefix="/")
     csrf.exempt(home_blueprint)
 
+    # Register 404 error handler
+    @app.errorhandler(404)
+    def error_page_not_found(e):
+        return render_template("404.html"), 404
+
     return app
-