Monitor analytics for errors

Author: MrEricSir

api/generate_reports_job.py

@@ -4,6 +4,8 @@
 This script is executed by Cloud Scheduler via Cloud Run Jobs (daily at midnight).
 """
 
+import logging
+import os
 import sys
 from pathlib import Path
 
@@ -12,7 +14,15 @@
 PROJECT_ROOT = API_DIR.parent
 sys.path.insert(0, str(PROJECT_ROOT))
 
-from lib.analytics import generate_all_reports, init_db
+from lib.analytics import generate_all_reports, init_db, check_database_health
+
+# Configure logging for Cloud Logging
+# On Cloud Run, logs at ERROR level will trigger alerts
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(levelname)s: %(message)s'
+)
+logger = logging.getLogger(__name__)
 
 
 def main():
@@ -21,7 +31,37 @@ def main():
         print("Starting MuniMetro analytics report generation...")
         print("-" * 60)
 
-        # Ensure database exists
+        # Check database health first
+        health = check_database_health()
+        is_cloud_run = os.getenv('CLOUD_RUN') is not None
+
+        print(f"Database health check:")
+        print(f"  exists: {health['exists']}")
+        if is_cloud_run:
+            print(f"  restored_from_gcs: {health['restored_from_gcs']}")
+        print(f"  has_data: {health['has_data']}")
+        print(f"  check_count: {health['check_count']}")
+        if health['error']:
+            print(f"  error: {health['error']}")
+
+        # Check for critical issues that should trigger alerts
+        if is_cloud_run and not health['restored_from_gcs'] and not health['exists']:
+            # On Cloud Run with no database - this is a problem
+            logger.error(
+                "ANALYTICS_NO_DATABASE: No analytics database found. "
+                "Database was not restored from GCS and does not exist locally. "
+                "This may indicate a GCS backup issue or first-time deployment."
+            )
+            # Continue to generate empty reports, but log the error
+
+        if not health['has_data']:
+            if health['check_count'] == 0:
+                logger.warning(
+                    "ANALYTICS_NO_DATA: Analytics database has no status check records. "
+                    "Reports will be empty until status checks are logged."
+                )
+
+        # Ensure database schema exists
         init_db()
 
         # Generate reports for all time periods
@@ -40,10 +80,12 @@ def main():
             print("\n✓ All reports generated successfully")
             sys.exit(0)
         else:
+            logger.error("ANALYTICS_REPORT_FAILED: Some reports failed to generate")
             print("\n⚠️ Some reports failed to generate", file=sys.stderr)
             sys.exit(1)
 
     except Exception as e:
+        logger.error(f"ANALYTICS_ERROR: Exception during report generation: {e}")
         print(f"\n❌ Error during report generation: {e}", file=sys.stderr)
         import traceback
         traceback.print_exc()

deploy/cloud/setup-monitoring.sh

@@ -9,6 +9,7 @@ PROJECT_ID="${GCP_PROJECT_ID:-munimetro}"
 REGION="${GCP_REGION:-us-west1}"
 API_SERVICE="munimetro-api"
 CHECKER_JOB="munimetro-checker"
+REPORTS_JOB="munimetro-reports"
 ALERT_EMAIL="${ALERT_EMAIL:-}"
 
 echo "=========================================="
@@ -256,7 +257,64 @@ curl -s -X POST \
     -d "$JOB_ALERT_JSON" \
     "https://monitoring.googleapis.com/v3/projects/$PROJECT_ID/alertPolicies" > /dev/null
 rm /tmp/alert-job-failures.json
-echo "✓ Job failure alert policy created"
+echo "✓ Checker job failure alert policy created"
+
+# Create alert policy for Reports job failures (includes missing database errors)
+cat > /tmp/alert-reports-failures.json <<EOF
+{
+  "displayName": "MuniMetro Reports Job Failures",
+  "conditions": [{
+    "displayName": "Reports job failing",
+    "conditionThreshold": {
+      "filter": "resource.type=\"cloud_run_job\" AND resource.labels.job_name=\"$REPORTS_JOB\" AND metric.type=\"run.googleapis.com/job/completed_execution_count\" AND metric.labels.result=\"failed\"",
+      "comparison": "COMPARISON_GT",
+      "thresholdValue": 0,
+      "duration": "0s",
+      "aggregations": [{
+        "alignmentPeriod": "86400s",
+        "perSeriesAligner": "ALIGN_DELTA"
+      }]
+    }
+  }],
+  "alertStrategy": {
+    "autoClose": "86400s"
+  },
+  "combiner": "OR",
+  "enabled": true,
+  "notificationChannels": ["$CHANNEL_ID"],
+  "documentation": {
+    "content": "The MuniMetro analytics reports job has failed. This may indicate a missing or corrupted analytics database. Check Cloud Run logs for ANALYTICS_NO_DATABASE or ANALYTICS_ERROR messages.",
+    "mimeType": "text/markdown"
+  }
+}
+EOF
+
+EXISTING_REPORTS_ALERT=$(echo "$EXISTING_POLICIES" | python3 -c "
+import sys, json
+try:
+    data = json.load(sys.stdin)
+    for policy in data.get('alertPolicies', []):
+        if policy.get('displayName') == 'MuniMetro Reports Job Failures':
+            print(policy['name'])
+            sys.exit(0)
+except:
+    pass
+" 2>/dev/null)
+
+if [ -n "$EXISTING_REPORTS_ALERT" ]; then
+    echo "⚠️  Reports job failure alert policy already exists, updating..."
+    curl -s -X DELETE -H "Authorization: Bearer $ACCESS_TOKEN" \
+        "https://monitoring.googleapis.com/v3/$EXISTING_REPORTS_ALERT" > /dev/null
+fi
+
+REPORTS_ALERT_JSON=$(cat < /tmp/alert-reports-failures.json)
+curl -s -X POST \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "$REPORTS_ALERT_JSON" \
+    "https://monitoring.googleapis.com/v3/projects/$PROJECT_ID/alertPolicies" > /dev/null
+rm /tmp/alert-reports-failures.json
+echo "✓ Reports job failure alert policy created"
 
 # Create alert policy for API error rate
 cat > /tmp/alert-error-rate.json <<EOF
@@ -323,8 +381,9 @@ echo "=========================================="
 echo ""
 echo "Configured alerts:"
 echo "  1. API Down - Health check failing for 5+ minutes"
-echo "  2. Job Failures - Checker job failing repeatedly"
-echo "  3. High Error Rate - API returning >10% errors"
+echo "  2. Checker Job Failures - Status checker job failing repeatedly"
+echo "  3. Reports Job Failures - Analytics reports job failing (includes missing database)"
+echo "  4. High Error Rate - API returning >10% errors"
 echo ""
 echo "Notifications will be sent to: $ALERT_EMAIL"
 echo ""

lib/analytics.py

@@ -688,3 +688,63 @@ def generate_all_reports():
             results[days] = {'success': False}
 
     return results
+
+
+def check_database_health():
+    """
+    Check if the analytics database exists and has data.
+
+    Returns:
+        dict: {
+            'exists': bool,           # Database file exists locally
+            'restored_from_gcs': bool,  # Was restored from GCS (Cloud Run only)
+            'has_data': bool,         # Has any status checks recorded
+            'check_count': int,       # Number of status checks
+            'error': str or None      # Error message if any
+        }
+    """
+    result = {
+        'exists': False,
+        'restored_from_gcs': False,
+        'has_data': False,
+        'check_count': 0,
+        'error': None
+    }
+
+    try:
+        # On Cloud Run, try to restore from GCS first
+        if _is_cloud_run():
+            result['restored_from_gcs'] = restore_db_from_gcs()
+
+        # Check if database file exists
+        result['exists'] = LOCAL_DB_PATH.exists()
+
+        if not result['exists']:
+            return result
+
+        # Check if database has any data
+        conn = get_db_connection()
+        cursor = conn.cursor()
+
+        # Check if tables exist
+        cursor.execute('''
+            SELECT name FROM sqlite_master
+            WHERE type='table' AND name='status_checks'
+        ''')
+        if not cursor.fetchone():
+            conn.close()
+            result['error'] = 'status_checks table does not exist'
+            return result
+
+        # Count status checks
+        cursor.execute('SELECT COUNT(*) as count FROM status_checks')
+        count = cursor.fetchone()['count']
+        conn.close()
+
+        result['check_count'] = count
+        result['has_data'] = count > 0
+
+    except Exception as e:
+        result['error'] = str(e)
+
+    return result