phase7_gmail_v2.py

#!/usr/bin/env python3
"""
Phase 7 – Gmail Message Extraction (Refactored)
Decompress and extract Gmail messages from bigTopDataDB
"""

import sqlite3
import zlib
import re
import json
from pathlib import Path
from datetime import datetime
from typing import Dict, List, Optional
import sys

sys.path.insert(0, str(Path(__file__).parent))
from config import EXTRACTED, find_databases, get_output_dir, PhaseOutput, calculate_sha256


def decompress_message(blob: bytes) -> Optional[bytes]:
    if not blob:
        return None
    
    try:
        data = blob[1:] if blob[0] == 0x00 else blob
        decompressed = zlib.decompress(data)
        return decompressed
    except:
        return None


def extract_strings(data: bytes, min_length: int = 5) -> List[str]:
    strings = []
    current = b""
    
    for byte in data:
        if 32 <= byte < 127:
            current += bytes([byte])
        else:
            if len(current) >= min_length:
                try:
                    strings.append(current.decode('ascii'))
                except:
                    pass
            current = b""
    
    if len(current) >= min_length:
        try:
            strings.append(current.decode('ascii'))
        except:
            pass
    
    return strings


def extract_email_metadata(strings: List[str]) -> Dict:
    metadata = {
        'sender': None,
        'sender_email': None,
        'recipient': None,
        'subject': None,
        'date': None,
        'content_preview': None,
    }
    
    full_text = ' | '.join(strings)
    
    email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
    emails = re.findall(email_pattern, full_text)
    
    if emails:
        for email in emails:
            if 'gmail.com' not in email.lower() and 'google.com' not in email.lower():
                if email.startswith('no-reply') or email.startswith('notification') or 'noreply' in email.lower():
                    metadata['sender_email'] = email
                    break
        
        for email in emails:
            if 'heisenbergcarro@gmail.com' in email:
                metadata['recipient'] = email
                break
    
    for platform in ['Twitter', 'Instagram', 'TikTok', 'Facebook', 'Reddit', 'YouTube', 'LinkedIn']:
        if platform.lower() in full_text.lower():
            metadata['sender'] = platform
            break
    
    content_parts = []
    for s in strings:
        if len(s) > 20 and not any(skip in s.lower() for skip in ['margin', 'padding', 'border', 'width=', 'height=', 'style=']):
            content_parts.append(s)
            if len(content_parts) >= 5:
                break
    
    if content_parts:
        metadata['content_preview'] = ' '.join(content_parts[:5])[:500]
    
    return metadata


def main():
    output = PhaseOutput(7, "gmail")
    out_dir = get_output_dir("gmail")
    
    gmail_dbs = find_databases("com.google.android.gm")
    
    md = ["# Phase 7 – Gmail Message Extraction\n\n"]
    md.append(f"**Erstellt:** {datetime.now().strftime('%Y-%m-%d %H:%M')}  \n\n---\n\n")
    
    md.append("## 1. Datenbanken\n\n")
    
    if not gmail_dbs:
        md.append("**Keine Gmail Datenbank gefunden**\n\n")
        output.set_summary("status", "no_database_found")
    else:
        md.append(f"**Gefundene Datenbanken:** {len(gmail_dbs)}\n\n")
        output.set_summary("databases_found", len(gmail_dbs))
        
        for db_path in gmail_dbs:
            md.append(f"- {db_path.name}\n")
        
        bigtop_dbs = [d for d in gmail_dbs if "bigTopDataDB" in d.name]
        
        if not bigtop_dbs:
            for search_path in [
                EXTRACTED / "Dump" / "data_mirror" / "data_ce" / "null" / "0" / "com.google.android.gm" / "databases",
                EXTRACTED / "Dump" / "data" / "user" / "0" / "com.google.android.gm" / "databases",
                EXTRACTED / "Dump" / "data" / "data" / "com.google.android.gm" / "databases",
            ]:
                if search_path.exists():
                    for f in search_path.glob("bigTopDataDB*"):
                        if '-wal' not in f.name and '-shm' not in f.name:
                            bigtop_dbs.append(f)
        
        if bigtop_dbs:
            bigtop_dbs = sorted(bigtop_dbs, key=lambda x: x.stat().st_size, reverse=True)
            main_db = bigtop_dbs[0]
            output.add_source(main_db, "Gmail bigTopDataDB database")
            md.append(f"\n**Analysiere:** {main_db.name}\n\n")
            
            md.append("## 2. Nachrichten-Extraktion\n\n")
            
            try:
                con = sqlite3.connect(f"file:{main_db}?mode=ro", uri=True)
                
                con.execute("SELECT COUNT(*) FROM item_messages WHERE zipped_message_proto IS NOT NULL")
                total_count = con.execute("SELECT COUNT(*) FROM item_messages WHERE zipped_message_proto IS NOT NULL").fetchone()[0]
                
                md.append(f"**Gesamt-Nachrichten:** {total_count}\n\n")
                output.set_summary("total_messages", total_count)
                
                messages = []
                decompressed_count = 0
                
                cur = con.execute("""
                    SELECT row_id, server_perm_id, zipped_message_proto 
                    FROM item_messages 
                    WHERE zipped_message_proto IS NOT NULL
                    ORDER BY row_id
                    LIMIT 200
                """)
                
                for row_id, server_perm_id, blob in cur.fetchall():
                    decompressed = decompress_message(blob)
                    
                    if decompressed:
                        decompressed_count += 1
                        strings = extract_strings(decompressed)
                        metadata = extract_email_metadata(strings)
                        
                        msg = {
                            'row_id': row_id,
                            'server_perm_id': server_perm_id,
                            'size_compressed': len(blob),
                            'size_decompressed': len(decompressed),
                        }
                        msg.update(metadata)
                        messages.append(msg)
                        
                        if metadata.get('sender'):
                            output.add_finding("email_sender", f"msg_{row_id}", metadata['sender'], metadata.get('sender_email', ''))
                
                output.set_summary("decompressed", decompressed_count)
                output.set_summary("success_rate", f"{100*decompressed_count//total_count if total_count else 0}%")
                
                md.append(f"**Erfolgreich dekomprimiert:** {decompressed_count}/{total_count}\n\n")
                
                md.append("## 3. Absender\n\n")
                
                senders = {}
                for msg in messages:
                    sender = msg.get('sender') or msg.get('sender_email')
                    if sender:
                        senders[sender] = senders.get(sender, 0) + 1
                
                md.append("| Absender | Anzahl |\n|---|---|\n")
                
                for sender, count in sorted(senders.items(), key=lambda x: x[1], reverse=True)[:20]:
                    md.append(f"| {sender} | {count} |\n")
                
                json_path = out_dir / "gmail_messages.json"
                json_path.write_text(json.dumps(messages, indent=2, default=str))
                md.append(f"\n**JSON gespeichert:** {json_path.name}\n\n")
                
                con.close()
                
            except Exception as e:
                md.append(f"**Fehler:** {e}\n\n")
                output.set_summary("error", str(e))
    
    md.append("\n---\n\n")
    md.append(f"**Report erstellt:** {datetime.now().isoformat()}\n")
    
    report_path = out_dir / "gmail_report.md"
    report_path.write_text("".join(md))
    
    outputs = output.write()
    
    print(f"[OK] Phase 7 Complete")
    print(f"     JSON: {list(outputs.keys())[0]}")
    print(f"     Markdown: {report_path}")


if __name__ == "__main__":
    main()