config.py

import os
import yaml
from pathlib import Path

DEFAULT_CONFIG = {
    "project": {
        "name": "stats_scraper",
        "description": "GitHub repository statistics scraper"
    },
    "logging": {
        "level": "DEBUG",  # Default to ERROR level (options: DEBUG, INFO, WARNING, ERROR, CRITICAL)
        "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    },
    "github": {
        "owner": "apache",
        "repo": "superset",
        "api_url": "https://api.github.com"
    },
    "database": {
        "type": "motherduck",  # Default to MotherDuck
        "connection": {
            "motherduck": {
                "database": "superset_stats"
            },
            "postgresql": {
                "host": "localhost",
                "port": 5432,
                "database": "superset_stats",
                "username": "postgres",
                "password": ""
            },
            "sqlite": {
                "path": "superset_stats.db"
            },
            "rds": {
                "host": "",
                "port": 5432,
                "database": "superset_stats",
                "username": "",
                "password": ""
            }
        }
    },
    "tables": {
        "github_stars": "github_stars",
        "github_visitors": "github_visitors",
        "new_contributors": "new_contributors",
        "new_issue_creators": "new_issue_creators",
        "repo_activity": "repo_activity",
        "repo_events": "repo_events",
        "open_issues": "open_issues",
        "open_prs": "open_prs",
        "slack_workspace_stats": "slack_workspace_stats",
        "matomo_analytics": "matomo_analytics"
    },
    "matomo": {
        "base_url": "https://analytics.apache.org/index.php",
        "site_id": 22
    },
    "slack": {
        "workspace_id": ""
    }
}

def load_config():
    """
    Load configuration from default values, config file, and environment variables.
    """
    import logging
    logger = logging.getLogger(__name__)
    
    config = DEFAULT_CONFIG.copy()
    logger.info(f"Default config loaded. Database type: {config['database']['type']}")
    logger.info(f"Default config has debug_writes: {'debug_writes' in config['database']}")
    
    # Load from config file if it exists
    config_file = Path("config.yaml")
    logger.info(f"Looking for config file at: {config_file.absolute()}")
    
    if config_file.exists():
        logger.info(f"Config file found: {config_file}")
        with open(config_file, "r") as f:
            file_content = f.read()
            logger.info(f"Config file content preview: {file_content[:100]}...")
            
            file_config = yaml.safe_load(file_content)
            if file_config:
                logger.info(f"Loaded config from file. Keys: {list(file_config.keys())}")
                if 'database' in file_config:
                    logger.info(f"File config database keys: {list(file_config['database'].keys())}")
                    logger.info(f"File config has debug_writes: {'debug_writes' in file_config['database']}")
                    if 'debug_writes' in file_config['database']:
                        logger.info(f"File config debug_writes value: {file_config['database']['debug_writes']}")
                
                # Deep merge the configurations
                logger.info("Merging configurations...")
                deep_merge(config, file_config)
                
                # Verify merge result
                logger.info(f"After merge, config has debug_writes: {'debug_writes' in config['database']}")
                if 'debug_writes' in config['database']:
                    logger.info(f"After merge, debug_writes value: {config['database']['debug_writes']}")
            else:
                logger.warning("Config file exists but could not be parsed as YAML")
    else:
        logger.warning(f"Config file not found at {config_file.absolute()}")
    
    # Override with environment variables
    # Logging settings
    if os.getenv("LOG_LEVEL"):
        config["logging"]["level"] = os.getenv("LOG_LEVEL")
    
    # GitHub settings
    if os.getenv("GITHUB_OWNER"):
        config["github"]["owner"] = os.getenv("GITHUB_OWNER")
    if os.getenv("GITHUB_REPO"):
        config["github"]["repo"] = os.getenv("GITHUB_REPO")
    
    # Database settings
    if os.getenv("DATABASE_TYPE"):
        config["database"]["type"] = os.getenv("DATABASE_TYPE")
    
    # MotherDuck settings
    if os.getenv("MOTHERDUCK_DATABASE"):
        config["database"]["connection"]["motherduck"]["database"] = os.getenv("MOTHERDUCK_DATABASE")
    
    # Matomo settings
    if os.getenv("MATOMO_BASE_URL"):
        config["matomo"]["base_url"] = os.getenv("MATOMO_BASE_URL")
    if os.getenv("MATOMO_SITE_ID"):
        config["matomo"]["site_id"] = int(os.getenv("MATOMO_SITE_ID"))
    
    # Validate the configuration
    validate_config(config)
    
    return config

def deep_merge(dict1, dict2):
    """
    Deep merge two dictionaries.
    """
    import logging
    logger = logging.getLogger(__name__)
    
    logger.info(f"Deep merging dictionaries. Dict1 keys: {list(dict1.keys())}, Dict2 keys: {list(dict2.keys())}")
    
    for key, value in dict2.items():
        logger.info(f"Processing key: {key}, value type: {type(value)}")
        
        if key in dict1 and isinstance(dict1[key], dict) and isinstance(value, dict):
            logger.info(f"Recursively merging dictionaries for key: {key}")
            logger.info(f"Dict1[{key}] keys before merge: {list(dict1[key].keys())}")
            logger.info(f"Dict2[{key}] keys: {list(value.keys())}")
            
            # Special handling for database config
            if key == "database" and "debug_writes" in value:
                logger.info(f"Found debug_writes in database config with value: {value['debug_writes']}")
            
            deep_merge(dict1[key], value)
            logger.info(f"Dict1[{key}] keys after merge: {list(dict1[key].keys())}")
            
            # Verify debug_writes was merged correctly if it exists
            if key == "database" and "debug_writes" in value:
                logger.info(f"After merge, debug_writes in dict1: {'debug_writes' in dict1[key]}")
                if "debug_writes" in dict1[key]:
                    logger.info(f"After merge, debug_writes value: {dict1[key]['debug_writes']}")
        else:
            logger.info(f"Setting dict1[{key}] = {value}")
            dict1[key] = value

def validate_config(config):
    """
    Validate the configuration.
    """
    import logging
    logger = logging.getLogger(__name__)
    
    logger.info("Validating configuration...")
    logger.info(f"Config database keys before validation: {list(config['database'].keys())}")
    logger.info(f"Config has debug_writes before validation: {'debug_writes' in config['database']}")
    if 'debug_writes' in config['database']:
        logger.info(f"Config debug_writes value before validation: {config['database']['debug_writes']}")
    
    # Validate logging settings
    valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
    if config["logging"]["level"] not in valid_log_levels:
        logger.warning(f"Invalid log level: {config['logging']['level']}. Using ERROR as default.")
        config["logging"]["level"] = "ERROR"
    
    # Validate GitHub settings
    if not config["github"]["owner"]:
        raise ValueError("GitHub owner is required")
    if not config["github"]["repo"]:
        raise ValueError("GitHub repository is required")
    
    # Validate database settings
    db_type = config["database"]["type"]
    logger.info(f"Database type: {db_type}")
    if db_type not in ["motherduck", "postgresql", "sqlite", "rds"]:
        raise ValueError(f"Unsupported database type: {db_type}")
    
    # Validate database connection settings based on type
    if db_type == "motherduck":
        if not config["database"]["connection"]["motherduck"]["database"]:
            raise ValueError("MotherDuck database name is required")
    elif db_type == "postgresql":
        pg_config = config["database"]["connection"]["postgresql"]
        if not pg_config["host"] or not pg_config["database"]:
            raise ValueError("PostgreSQL host and database are required")
    elif db_type == "sqlite":
        if not config["database"]["connection"]["sqlite"]["path"]:
            raise ValueError("SQLite database path is required")
    elif db_type == "rds":
        rds_config = config["database"]["connection"]["rds"]
        if not rds_config["host"] or not rds_config["database"]:
            raise ValueError("RDS host and database are required")
    
    logger.info(f"Config database keys after validation: {list(config['database'].keys())}")
    logger.info(f"Config has debug_writes after validation: {'debug_writes' in config['database']}")
    if 'debug_writes' in config['database']:
        logger.info(f"Config debug_writes value after validation: {config['database']['debug_writes']}")

def configure_logging(config):
    """
    Configure logging based on the configuration.
    """
    import logging
    
    # Map string log levels to logging module constants
    log_level_map = {
        "DEBUG": logging.DEBUG,
        "INFO": logging.INFO,
        "WARNING": logging.WARNING,
        "ERROR": logging.ERROR,
        "CRITICAL": logging.CRITICAL
    }
    
    # Get the log level from config
    log_level = log_level_map.get(config["logging"]["level"], logging.ERROR)
    log_format = config["logging"]["format"]
    
    # Configure the root logger
    logging.basicConfig(
        level=log_level,
        format=log_format
    )
    
    # Log the configured level
    logger = logging.getLogger(__name__)
    logger.debug("Logging configured with level: %s", config["logging"]["level"])

# Get the configuration
config = load_config()

# Configure logging
configure_logging(config)