matomo_client.py

import logging
import os

import requests

from utils import is_valid_date_format

# Set up logging
logger = logging.getLogger(__name__)


class MatomoClient:
    """
    Matomo API client.
    """

    def __init__(self, config):
        self.config = config
        self.token = os.getenv("MATOMO_KEY")
        
        # If token is a placeholder or None, set it to None to allow anonymous access
        if self.token in [None, "your_matomo_token", "your_matomo_key"]:
            self.token = None
            logger.info("No valid Matomo token found, attempting anonymous access")

        self.base_url = config["matomo"]["base_url"]
        self.site_id = config["matomo"]["site_id"]

        logger.info(f"Initialized Matomo client for site ID {self.site_id}")

    def get_analytics_data(self, start_date, end_date):
        """
        Get Matomo analytics data for a date range.

        Args:
            start_date: Start date in YYYY-MM-DD format
            end_date: End date in YYYY-MM-DD format

        Returns:
            List of daily analytics data
        """
        logger.info(f"Fetching Matomo data from {start_date} to {end_date}")
        
        # Log token information for debugging
        if self.token:
            token_length = len(self.token)
            token_prefix = self.token[:4] if token_length >= 4 else self.token
            token_suffix = self.token[-4:] if token_length >= 4 else self.token
            logger.info(f"DEBUG: Using Matomo token (length: {token_length}, format: {token_prefix}...{token_suffix})")
        else:
            logger.warning("DEBUG: No Matomo token available")
            
        # Log environment variable directly
        env_token = os.getenv("MATOMO_KEY")
        if env_token:
            env_token_length = len(env_token)
            env_token_prefix = env_token[:4] if env_token_length >= 4 else env_token
            env_token_suffix = env_token[-4:] if env_token_length >= 4 else env_token
            logger.info(f"DEBUG: Environment MATOMO_KEY (length: {env_token_length}, format: {env_token_prefix}...{env_token_suffix})")
            # Check if tokens match
            if env_token == self.token:
                logger.info("DEBUG: Environment token matches instance token")
            else:
                logger.warning("DEBUG: Environment token DOES NOT match instance token")
        else:
            logger.warning("DEBUG: MATOMO_KEY environment variable not set")

        # Parameters for the API request
        params = {
            "module": "API",
            "method": "API.getBulkRequest",
            "format": "json",
        }

        # Add authentication token if available
        if self.token:
            params["token_auth"] = self.token

        # Create bulk request for visits and unique visitors
        bulk_requests = [
            {
                "module": "VisitsSummary",
                "method": "VisitsSummary.get",
                "idSite": self.site_id,
                "period": "day",
                "date": f"{start_date},{end_date}",
            },
            {
                "module": "VisitsSummary",
                "method": "VisitsSummary.getUniqueVisitors",
                "idSite": self.site_id,
                "period": "day",
                "date": f"{start_date},{end_date}",
            },
        ]

        # Convert bulk requests to URL parameters
        for i, request in enumerate(bulk_requests):
            params[f"urls[{i}]"] = (
                f"module={request['module']}&method={request['method']}&idSite={request['idSite']}&period={request['period']}&date={request['date']}&format=json"
            )
            
        # Log the full request URL and parameters
        logger.info(f"DEBUG: Request URL: {self.base_url}")
        logger.info(f"DEBUG: Request parameters: {params}")

        try:
            # Try with GET request first (current implementation)
            logger.info("DEBUG: Attempting GET request with token in URL parameters")
            response = requests.get(self.base_url, params=params)
            
            # Log response status and preview
            logger.info(f"DEBUG: GET response status: {response.status_code}")
            logger.info(f"DEBUG: GET response preview: {response.text[:200]}...")
            
            # If GET fails with 401, try POST
            if response.status_code == 401:
                logger.info("DEBUG: GET request failed with 401, attempting POST request")
                post_response = requests.post(self.base_url, data=params)
                logger.info(f"DEBUG: POST response status: {post_response.status_code}")
                logger.info(f"DEBUG: POST response preview: {post_response.text[:200]}...")
                
                # If POST succeeds, use that response instead
                if post_response.status_code == 200:
                    logger.info("DEBUG: POST request succeeded, using POST response")
                    response = post_response

            if response.status_code != 200:
                raise Exception(
                    f"Error fetching Matomo data: {response.status_code}, {response.text}"
                )

            data = response.json()

            if not data or len(data) < 2:
                raise Exception(f"Unexpected response format from Matomo API: {data}")

            # Process the response
            visits_data = data[0]
            unique_visitors_data = data[1]

            # Check for API error responses
            if (
                isinstance(visits_data, dict)
                and "result" in visits_data
                and visits_data["result"] == "error"
            ):
                error_msg = visits_data.get("message", "Unknown error")
                raise Exception(f"API error in visits data: {error_msg}")

            if (
                isinstance(unique_visitors_data, dict)
                and "result" in unique_visitors_data
                and unique_visitors_data["result"] == "error"
            ):
                error_msg = unique_visitors_data.get("message", "Unknown error")
                raise Exception(f"API error in unique visitors data: {error_msg}")

            # Combine the data
            result = []
            for date in visits_data:
                # Skip non-date keys like "result"
                if not is_valid_date_format(date):
                    continue

                if date in unique_visitors_data:
                    # Handle different response formats
                    visits = 0
                    pageviews = 0
                    bounce_rate = 0.0
                    avg_time_on_site = 0
                    actions_per_visit = 0.0

                    if isinstance(visits_data[date], dict):
                        visits = visits_data[date].get("nb_visits", 0)
                        pageviews = visits_data[date].get("nb_actions", 0)

                        # Convert bounce_rate from string percentage to float
                        bounce_rate_str = visits_data[date].get("bounce_rate", "0%")
                        try:
                            bounce_rate = float(bounce_rate_str.strip("%")) / 100
                        except (ValueError, TypeError, AttributeError):
                            bounce_rate = 0.0

                        avg_time_on_site = visits_data[date].get("avg_time_on_site", 0)
                        actions_per_visit = visits_data[date].get("nb_actions_per_visit", 0.0)

                        # Ensure numeric types
                        try:
                            avg_time_on_site = int(avg_time_on_site)
                        except (ValueError, TypeError):
                            avg_time_on_site = 0

                        try:
                            actions_per_visit = float(actions_per_visit)
                        except (ValueError, TypeError):
                            actions_per_visit = 0.0

                    elif isinstance(visits_data[date], (int, str)):
                        # If it's a direct value (int or string that can be converted to int)
                        try:
                            visits = int(visits_data[date])
                        except (ValueError, TypeError):
                            visits = 0

                    # Handle unique visitors similarly
                    unique_visitors = 0
                    if isinstance(unique_visitors_data[date], (int, str)):
                        try:
                            unique_visitors = int(unique_visitors_data[date])
                        except (ValueError, TypeError):
                            unique_visitors = 0
                    elif isinstance(unique_visitors_data[date], dict):
                        # If it's a dictionary, try to get a relevant value
                        unique_visitors = unique_visitors_data[date].get("nb_uniq_visitors", 0)

                    result.append(
                        {
                            "date": date,
                            "visits": visits,
                            "unique_visitors": unique_visitors,
                            "pageviews": pageviews,
                            "bounce_rate": bounce_rate,
                            "avg_time_on_site": avg_time_on_site,
                            "actions_per_visit": actions_per_visit,
                        }
                    )

            logger.info(f"Retrieved {len(result)} days of Matomo analytics data")
            return result

        except Exception as e:
            logger.error(f"Error processing Matomo data: {e}")
            raise

    def get_page_urls(self, start_date, end_date, limit=100, max_pages=5, segment=None):
        """
        Get the most visited page URLs for a date range with pagination support.
        Note: This method may not capture all SPA page views. For SPA applications,
        consider using get_page_views() which combines data from both URLs and page titles.

        Args:
            start_date: Start date in YYYY-MM-DD format
            end_date: End date in YYYY-MM-DD format
            limit: Number of URLs to return per page
            max_pages: Maximum number of pages to fetch (default: 5, for up to 500 results with limit=100)
            segment: Optional segment parameter to filter results (e.g., "pageUrl=@app/")

        Returns:
            List of dictionaries with page URL statistics
        """
        logger.info(f"Fetching top page URLs from {start_date} to {end_date}")
        if segment:
            logger.info(f"Using segment filter: {segment}")

        all_results = []
        page = 0

        while page < max_pages:
            offset = page * limit
            logger.info(f"Fetching page {page+1} (offset: {offset}, limit: {limit})")

            # Parameters for the API request
            params = {
                "module": "API",
                "method": "Actions.getPageUrls",
                "idSite": self.site_id,
                "period": "range",
                "date": f"{start_date},{end_date}",
                "format": "json",
                "filter_limit": limit,
                "filter_offset": offset,
            }
            
            # Add segment parameter if provided
            if segment:
                params["segment"] = segment

            # Add authentication token if available
            if self.token:
                params["token_auth"] = self.token

            try:
                # Log the full request URL and parameters for debugging
                logger.info(f"DEBUG: API request parameters: {params}")
                
                response = requests.get(self.base_url, params=params)

                if response.status_code != 200:
                    raise Exception(
                        f"Error fetching page URLs: {response.status_code}, {response.text}"
                    )

                data = response.json()

                # Check for API error response
                if isinstance(data, dict) and "result" in data and data["result"] == "error":
                    error_msg = data.get("message", "Unknown error")
                    raise Exception(f"API error in page URLs data: {error_msg}")

                # If no data or empty list, we've reached the end of results
                if not data or (isinstance(data, list) and len(data) == 0):
                    logger.info(f"No more results after page {page+1}")
                    break

                # Process the response
                page_results = []

                # Log the full structure of the first page result to see what fields are available
                if data and len(data) > 0 and page == 0:
                    logger.info(f"DEBUG: First page result structure: {data[0]}")

                    # Log all keys in the first result
                    logger.info(f"DEBUG: First page result keys: {list(data[0].keys())}")

                    # Check for subtable
                    if "subtable" in data[0]:
                        logger.info(f"DEBUG: First page subtable: {data[0]['subtable']}")

                    # Check for metadata
                    if "metadata" in data[0]:
                        logger.info(f"DEBUG: First page metadata: {data[0]['metadata']}")

                for page_data in data:
                    url = page_data.get("label", "")
                    visits = page_data.get("nb_visits", 0)
                    pageviews = page_data.get("nb_hits", 0)
                    avg_time_on_page = page_data.get("avg_time_on_page", 0)
                    bounce_rate = 0.0

                    # Convert bounce_rate from string percentage to float
                    bounce_rate_str = page_data.get("bounce_rate", "0%")
                    try:
                        bounce_rate = float(bounce_rate_str.strip("%")) / 100
                    except (ValueError, TypeError, AttributeError):
                        bounce_rate = 0.0

                    page_results.append(
                        {
                            "date_range": f"{start_date} to {end_date}",
                            "url": url,
                            "visits": visits,
                            "pageviews": pageviews,
                            "avg_time_on_page": avg_time_on_page,
                            "bounce_rate": bounce_rate,
                        }
                    )

                all_results.extend(page_results)
                logger.info(f"Retrieved {len(page_results)} page URLs for page {page+1}")

                # If we got fewer results than the limit, we've reached the end
                if len(page_results) < limit:
                    logger.info(f"Reached end of results on page {page+1}")
                    break

                # Move to the next page
                page += 1

            except Exception as e:
                logger.error(f"Error processing page URLs data on page {page+1}: {e}")
                raise

        logger.info(f"Retrieved a total of {len(all_results)} page URLs across {page+1} pages")
        return all_results
        
    def get_page_titles(self, start_date, end_date, limit=100, max_pages=5, segment=None):
        """
        Get the most visited page titles for a date range with pagination support.
        This method may better capture SPA navigation events that set document titles.

        Args:
            start_date: Start date in YYYY-MM-DD format
            end_date: End date in YYYY-MM-DD format
            limit: Number of titles to return per page
            max_pages: Maximum number of pages to fetch (default: 5, for up to 500 results with limit=100)
            segment: Optional segment parameter to filter results (e.g., "actionType==pageview")

        Returns:
            List of dictionaries with page title statistics
        """
        logger.info(f"Fetching top page titles from {start_date} to {end_date}")
        if segment:
            logger.info(f"Using segment filter: {segment}")

        all_results = []
        page = 0

        while page < max_pages:
            offset = page * limit
            logger.info(f"Fetching page titles {page+1} (offset: {offset}, limit: {limit})")

            # Parameters for the API request
            params = {
                "module": "API",
                "method": "Actions.getPageTitles",
                "idSite": self.site_id,
                "period": "range",
                "date": f"{start_date},{end_date}",
                "format": "json",
                "filter_limit": limit,
                "filter_offset": offset,
            }
            
            # Add segment parameter if provided
            if segment:
                params["segment"] = segment

            # Add authentication token if available
            if self.token:
                params["token_auth"] = self.token

            try:
                # Log the full request URL and parameters for debugging
                logger.info(f"DEBUG: API request parameters: {params}")
                
                response = requests.get(self.base_url, params=params)

                if response.status_code != 200:
                    raise Exception(
                        f"Error fetching page titles: {response.status_code}, {response.text}"
                    )

                data = response.json()

                # Check for API error response
                if isinstance(data, dict) and "result" in data and data["result"] == "error":
                    error_msg = data.get("message", "Unknown error")
                    raise Exception(f"API error in page titles data: {error_msg}")

                # If no data or empty list, we've reached the end of results
                if not data or (isinstance(data, list) and len(data) == 0):
                    logger.info(f"No more results after page {page+1}")
                    break

                # Process the response
                page_results = []

                # Log the full structure of the first page result to see what fields are available
                if data and len(data) > 0 and page == 0:
                    logger.info(f"DEBUG: First page title result structure: {data[0]}")
                    logger.info(f"DEBUG: First page title result keys: {list(data[0].keys())}")

                for page_data in data:
                    title = page_data.get("label", "")
                    url = page_data.get("url", "")  # Some APIs include the URL
                    visits = page_data.get("nb_visits", 0)
                    pageviews = page_data.get("nb_hits", 0)
                    avg_time_on_page = page_data.get("avg_time_on_page", 0)
                    bounce_rate = 0.0

                    # Convert bounce_rate from string percentage to float
                    bounce_rate_str = page_data.get("bounce_rate", "0%")
                    try:
                        bounce_rate = float(bounce_rate_str.strip("%")) / 100
                    except (ValueError, TypeError, AttributeError):
                        bounce_rate = 0.0

                    page_results.append(
                        {
                            "date_range": f"{start_date} to {end_date}",
                            "title": title,
                            "url": url,
                            "visits": visits,
                            "pageviews": pageviews,
                            "avg_time_on_page": avg_time_on_page,
                            "bounce_rate": bounce_rate,
                        }
                    )

                all_results.extend(page_results)
                logger.info(f"Retrieved {len(page_results)} page titles for page {page+1}")

                # If we got fewer results than the limit, we've reached the end
                if len(page_results) < limit:
                    logger.info(f"Reached end of results on page {page+1}")
                    break

                # Move to the next page
                page += 1

            except Exception as e:
                logger.error(f"Error processing page titles data on page {page+1}: {e}")
                raise

        logger.info(f"Retrieved a total of {len(all_results)} page titles across {page+1} pages")
        return all_results

    def get_referrers(self, start_date, end_date, limit=100):
        """
        Get the top referrers for a date range.

        Args:
            start_date: Start date in YYYY-MM-DD format
            end_date: End date in YYYY-MM-DD format
            limit: Maximum number of referrers to return

        Returns:
            List of dictionaries with referrer statistics
        """
        logger.info(f"Fetching top referrers from {start_date} to {end_date}")

        # Parameters for the API request
        params = {
            "module": "API",
            "method": "Referrers.getAll",
            "idSite": self.site_id,
            "period": "range",
            "date": f"{start_date},{end_date}",
            "format": "json",
            "filter_limit": limit,
        }

        # Add authentication token if available
        if self.token:
            params["token_auth"] = self.token

        try:
            response = requests.get(self.base_url, params=params)

            if response.status_code != 200:
                raise Exception(
                    f"Error fetching referrers: {response.status_code}, {response.text}"
                )

            data = response.json()

            # Check for API error response
            if isinstance(data, dict) and "result" in data and data["result"] == "error":
                error_msg = data.get("message", "Unknown error")
                raise Exception(f"API error in referrers data: {error_msg}")

            # Process the response
            result = []
            for referrer_type, referrers in data.items():
                if not isinstance(referrers, list):
                    continue

                for referrer in referrers:
                    label = referrer.get("label", "")
                    visits = referrer.get("nb_visits", 0)
                    unique_visitors = referrer.get("nb_uniq_visitors", 0)

                    result.append(
                        {
                            "date_range": f"{start_date} to {end_date}",
                            "referrer_type": referrer_type,
                            "referrer": label,
                            "visits": visits,
                            "unique_visitors": unique_visitors,
                        }
                    )

            logger.info(f"Retrieved {len(result)} referrers")
            return result

        except Exception as e:
            logger.error(f"Error processing referrers data: {e}")
            raise

    def get_page_views(self, start_date, end_date, limit=100, max_pages=5, include_path_patterns=None):
        """
        Get comprehensive page view data for a date range, combining both page URLs and page titles.
        This method is optimized for SPA (Single Page Application) tracking, as it captures both
        traditional page URLs and page title changes that are common in SPAs.

        Args:
            start_date: Start date in YYYY-MM-DD format
            end_date: End date in YYYY-MM-DD format
            limit: Number of results to return per page
            max_pages: Maximum number of pages to fetch
            include_path_patterns: Optional list of path patterns to specifically include
                                  (e.g., ["dashboard/", "app/"])

        Returns:
            Dictionary with combined page view data:
            {
                "urls": List of dictionaries with page URL statistics,
                "titles": List of dictionaries with page title statistics,
                "combined": List of dictionaries with combined statistics,
                "spa_routes": List of dictionaries with SPA-specific routes
            }
        """
        logger.info(f"Fetching comprehensive page view data from {start_date} to {end_date}")
        
        # Get page URLs
        urls_data = self.get_page_urls(start_date, end_date, limit, max_pages)
        logger.info(f"Retrieved {len(urls_data)} page URLs")
        
        # Get page titles
        titles_data = self.get_page_titles(start_date, end_date, limit, max_pages)
        logger.info(f"Retrieved {len(titles_data)} page titles")
        
        # Check for SPA-specific path patterns
        spa_routes = []
        if include_path_patterns:
            logger.info(f"Checking for SPA-specific path patterns: {include_path_patterns}")
            for pattern in include_path_patterns:
                try:
                    pattern_data = self.get_page_urls(
                        start_date, end_date, limit, 1,
                        segment=f"pageUrl=@{pattern}"
                    )
                    if pattern_data:
                        logger.info(f"Found {len(pattern_data)} URLs with pattern '{pattern}'")
                        for item in pattern_data:
                            item["pattern"] = pattern
                            spa_routes.append(item)
                except Exception as e:
                    logger.warning(f"Error fetching data for pattern '{pattern}': {e}")
        
        # Enhanced URL-Title matching
        # Step 1: Extract path components from URLs for better matching
        url_paths = {}
        for item in urls_data:
            url = item["url"]
            # Extract the path component (remove query params, fragments, etc.)
            path = url.split('?')[0].split('#')[0].strip('/')
            if path:
                if path not in url_paths:
                    url_paths[path] = []
                url_paths[path].append(item)
        
        # Step 2: Create a mapping of titles to potential URLs
        title_keywords = {}
        for title_item in titles_data:
            title = title_item.get("title", "")
            if title:
                # Extract keywords from title (remove common words, punctuation)
                keywords = [k.lower() for k in title.split() if len(k) > 3 and k.lower() not in
                           ["the", "and", "for", "with", "your", "from", "that", "this", "superset"]]
                if keywords:
                    for keyword in keywords:
                        if keyword not in title_keywords:
                            title_keywords[keyword] = []
                        title_keywords[keyword].append(title_item)
        
        # Step 3: Match titles with URLs based on path components and keywords
        title_to_url_map = {}
        url_to_title_map = {}
        
        # First pass: direct path matching
        for title_item in titles_data:
            title = title_item.get("title", "")
            # If the title already has a URL, use it
            if title_item.get("url", ""):
                title_to_url_map[title] = title_item["url"]
                url_to_title_map[title_item["url"]] = title
                continue
                
            # Try to match based on keywords in the title
            matched = False
            if title:
                keywords = [k.lower() for k in title.split() if len(k) > 3]
                for keyword in keywords:
                    for path, url_items in url_paths.items():
                        if keyword.lower() in path.lower():
                            # Found a potential match
                            title_to_url_map[title] = url_items[0]["url"]
                            url_to_title_map[url_items[0]["url"]] = title
                            matched = True
                            break
                    if matched:
                        break
        
        # Create combined dataset
        combined_data = []
        processed_urls = set()
        processed_titles = set()
        
        # First, process items that have both URL and title
        for title_item in titles_data:
            title = title_item.get("title", "")
            if not title:
                continue
                
            url = title_item.get("url", "") or title_to_url_map.get(title, "")
            if url:
                # Find matching URL item
                url_item = next((item for item in urls_data if item["url"] == url), None)
                
                if url_item:
                    # We have both URL and title data
                    combined_item = {
                        "date_range": title_item["date_range"],
                        "url": url,
                        "title": title,
                        "visits": max(title_item["visits"], url_item["visits"]),
                        "pageviews": max(title_item["pageviews"], url_item["pageviews"]),
                        "avg_time_on_page": url_item.get("avg_time_on_page", 0),
                        "bounce_rate": url_item.get("bounce_rate", 0),
                        "source": "both"
                    }
                    combined_data.append(combined_item)
                    processed_urls.add(url)
                    processed_titles.add(title)
                else:
                    # We only have title data
                    combined_item = {
                        "date_range": title_item["date_range"],
                        "url": url,  # Use the mapped URL if available
                        "title": title,
                        "visits": title_item["visits"],
                        "pageviews": title_item["pageviews"],
                        "avg_time_on_page": title_item.get("avg_time_on_page", 0),
                        "bounce_rate": title_item.get("bounce_rate", 0),
                        "source": "title"
                    }
                    combined_data.append(combined_item)
                    processed_titles.add(title)
        
        # Add remaining title items
        for title_item in titles_data:
            title = title_item.get("title", "")
            if title and title not in processed_titles:
                # Try to infer a URL from the title
                inferred_url = ""
                keywords = [k.lower() for k in title.split() if len(k) > 3 and k.lower() not in
                           ["the", "and", "for", "with", "your", "from", "that", "this", "superset"]]
                
                if "intro" in title.lower() or "introduction" in title.lower():
                    inferred_url = "docs/intro"
                elif "dashboard" in title.lower():
                    inferred_url = "dashboard"
                elif "api" in title.lower():
                    inferred_url = "docs/api"
                elif "faq" in title.lower():
                    inferred_url = "docs/faq"
                elif keywords:
                    # Use the first significant keyword as a path component
                    inferred_url = f"docs/{keywords[0]}"
                
                combined_item = {
                    "date_range": title_item["date_range"],
                    "url": inferred_url,
                    "title": title,
                    "visits": title_item["visits"],
                    "pageviews": title_item["pageviews"],
                    "avg_time_on_page": title_item.get("avg_time_on_page", 0),
                    "bounce_rate": title_item.get("bounce_rate", 0),
                    "source": "title"
                }
                combined_data.append(combined_item)
                processed_titles.add(title)
        
        # Add remaining URL items
        for url_item in urls_data:
            url = url_item["url"]
            if url not in processed_urls:
                # Try to infer a title from the URL
                inferred_title = ""
                if url:
                    # Remove query params and fragments
                    clean_url = url.split('?')[0].split('#')[0]
                    # Extract the last path component
                    path_components = clean_url.strip('/').split('/')
                    if path_components:
                        last_component = path_components[-1]
                        if last_component:
                            # Convert to title case and replace hyphens/underscores with spaces
                            inferred_title = last_component.replace('-', ' ').replace('_', ' ').title()
                            if path_components[0].lower() == "docs" and len(path_components) > 1:
                                inferred_title += " | Documentation"
                
                combined_item = {
                    "date_range": url_item["date_range"],
                    "url": url,
                    "title": inferred_title or url_to_title_map.get(url, ""),
                    "visits": url_item["visits"],
                    "pageviews": url_item["pageviews"],
                    "avg_time_on_page": url_item.get("avg_time_on_page", 0),
                    "bounce_rate": url_item.get("bounce_rate", 0),
                    "source": "url"
                }
                combined_data.append(combined_item)
        
        # Sort by visits (descending)
        combined_data.sort(key=lambda x: x["visits"], reverse=True)
        
        logger.info(f"Created combined dataset with {len(combined_data)} entries")
        logger.info(f"Successfully matched titles with URLs for {len([i for i in combined_data if i['url'] and i['title']])} entries")
        
        return {
            "urls": urls_data,
            "titles": titles_data,
            "combined": combined_data,
            "spa_routes": spa_routes
        }
    
    def get_visitor_countries(self, start_date, end_date):
        """
        Get visitor data by country for a date range.

        Args:
            start_date: Start date in YYYY-MM-DD format
            end_date: End date in YYYY-MM-DD format

        Returns:
            List of dictionaries with country visitor statistics
        """
        logger.info(f"Fetching visitor countries from {start_date} to {end_date}")
        
        # Log token information for debugging
        if self.token:
            token_length = len(self.token)
            token_prefix = self.token[:4] if token_length >= 4 else self.token
            token_suffix = self.token[-4:] if token_length >= 4 else self.token
            logger.info(f"DEBUG: Using Matomo token (length: {token_length}, format: {token_prefix}...{token_suffix})")
        else:
            logger.warning("DEBUG: No Matomo token available")
            
        # Log environment variable directly
        env_token = os.getenv("MATOMO_KEY")
        if env_token:
            env_token_length = len(env_token)
            env_token_prefix = env_token[:4] if env_token_length >= 4 else env_token
            env_token_suffix = env_token[-4:] if env_token_length >= 4 else env_token
            logger.info(f"DEBUG: Environment MATOMO_KEY (length: {env_token_length}, format: {env_token_prefix}...{env_token_suffix})")
        else:
            logger.warning("DEBUG: MATOMO_KEY environment variable not set")

        # Parameters for the API request
        params = {
            "module": "API",
            "method": "UserCountry.getCountry",
            "idSite": self.site_id,
            "period": "range",
            "date": f"{start_date},{end_date}",
            "format": "json",
        }

        # Add authentication token if available
        if self.token:
            params["token_auth"] = self.token
            
        # Log the full request URL and parameters
        logger.info(f"DEBUG: Request URL: {self.base_url}")
        logger.info(f"DEBUG: Request parameters: {params}")

        try:
            # Try with GET request first (current implementation)
            logger.info("DEBUG: Attempting GET request with token in URL parameters")
            response = requests.get(self.base_url, params=params)
            
            # Log response status and preview
            logger.info(f"DEBUG: GET response status: {response.status_code}")
            logger.info(f"DEBUG: GET response preview: {response.text[:200]}...")
            
            # If GET fails with 401, try POST
            if response.status_code == 401:
                logger.info("DEBUG: GET request failed with 401, attempting POST request")
                post_response = requests.post(self.base_url, data=params)
                logger.info(f"DEBUG: POST response status: {post_response.status_code}")
                logger.info(f"DEBUG: POST response preview: {post_response.text[:200]}...")
                
                # If POST succeeds, use that response instead
                if post_response.status_code == 200:
                    logger.info("DEBUG: POST request succeeded, using POST response")
                    response = post_response

            if response.status_code != 200:
                raise Exception(
                    f"Error fetching visitor countries: {response.status_code}, {response.text}"
                )

            data = response.json()

            # Check for API error response
            if isinstance(data, dict) and "result" in data and data["result"] == "error":
                error_msg = data.get("message", "Unknown error")
                raise Exception(f"API error in visitor countries data: {error_msg}")

            # Process the response
            result = []
            for country in data:
                country_code = country.get("code", "")
                country_name = country.get("label", "")
                visits = country.get("nb_visits", 0)
                unique_visitors = country.get("nb_uniq_visitors", 0)
                actions = country.get("nb_actions", 0)

                # Skip entries without a country code (like "Unknown")
                if not country_code:
                    country_code = "xx"  # Use "xx" for unknown

                # Get the continent if available
                continent_code = country.get("continent", "")

                # Get the country flag URL if available
                flag_url = country.get("flag", "")

                # Get the latitude and longitude if available
                latitude = country.get("lat", 0)
                longitude = country.get("long", 0)

                # For pageviews, we'll use actions as an approximation if not directly available
                pageviews = country.get("nb_pageviews", actions)  # Use actions as fallback
                
                result.append(
                    {
                        "date_range": f"{start_date} to {end_date}",
                        "country_code": country_code,
                        "country_name": country_name,
                        "continent_code": continent_code,
                        "visits": visits,
                        "unique_visitors": unique_visitors,
                        "actions": actions,
                        "pageviews": pageviews,  # Add pageviews to the result
                        "latitude": latitude,
                        "longitude": longitude,
                        "flag_url": flag_url,
                    }
                )

            logger.info(f"Retrieved visitor data for {len(result)} countries")
            return result

        except Exception as e:
            logger.error(f"Error processing visitor countries data: {e}")
            raise