apps/summarization: refactor export (#88)

Author: shn-liqd

apps/projects/export_utils.py

@@ -3,8 +3,8 @@
 from django.core.management.base import BaseCommand
 from django.db.models import Q
 
-from apps.projects.export_utils import generate_full_export
 from apps.projects.models import Project
+from apps.summarization.export_utils.core import generate_full_export
 
 
 class Command(BaseCommand):

apps/projects/management/commands/export_project_data.py

@@ -6,13 +6,16 @@
 from sentry_sdk import capture_exception
 
 from apps.contrib.models import Settings
+from apps.summarization.export_utils.attachments.handlers import (
+    collect_document_attachments,
+)
+from apps.summarization.export_utils.attachments.handlers import (
+    integrate_document_summaries,
+)
+from apps.summarization.export_utils.core import generate_full_export
 from apps.summarization.pydantic_models import ProjectSummaryResponse
 from apps.summarization.services import AIService
 
-from .export_utils import collect_document_attachments
-from .export_utils import generate_full_export
-from .export_utils import integrate_document_summaries
-
 logger = logging.getLogger(__name__)
 
 

apps/projects/utils.py

@@ -0,0 +1,105 @@
+def _make_absolute_url(attachment_url, request=None, base_url=None):
+    """Build absolute URL from attachment_url using request or base_url."""
+    if request is not None:
+        return request.build_absolute_uri(attachment_url)
+    if base_url:
+        base = base_url.rstrip("/")
+        path = (
+            attachment_url
+            if attachment_url.startswith("/")
+            else f"/{attachment_url.lstrip('/')}"
+        )
+        return f"{base}{path}"
+    return None
+
+
+def collect_document_attachments(export_data, request=None, base_url=None):
+    """
+    Collect all document attachments from project fields (information, result).
+
+    Args:
+        export_data: The full export dictionary (as returned by generate_full_export())
+        request: Optional Django Request object for build_absolute_uri(). If None, base_url is used.
+        base_url: Optional base URL (e.g. settings.WAGTAILADMIN_BASE_URL) when request is None.
+
+    Returns:
+        tuple: (documents_dict, handle_to_source)
+            - documents_dict: {handle: absolute_url, ...}
+            - handle_to_source: {handle: "project_information" | "project_result", ...}
+    """
+    documents_dict = {}
+    handle_to_source = {}
+
+    if request is None and not base_url:
+        return documents_dict, handle_to_source
+
+    project_data = export_data.get("project", {})
+
+    # Collect attachments from information field
+    information_attachments = project_data.get("information_attachments", [])
+    for attachment_index, attachment_url in enumerate(information_attachments):
+        handle = f"project_information_attachment_{attachment_index}"
+        absolute_url = _make_absolute_url(
+            attachment_url, request=request, base_url=base_url
+        )
+        if absolute_url:
+            documents_dict[handle] = absolute_url
+            handle_to_source[handle] = "project_information"
+
+    # Collect attachments from result field
+    result_attachments = project_data.get("result_attachments", [])
+    for attachment_index, attachment_url in enumerate(result_attachments):
+        handle = f"project_result_attachment_{attachment_index}"
+        absolute_url = _make_absolute_url(
+            attachment_url, request=request, base_url=base_url
+        )
+        if absolute_url:
+            documents_dict[handle] = absolute_url
+            handle_to_source[handle] = "project_result"
+
+    return documents_dict, handle_to_source
+
+
+def integrate_document_summaries(
+    export_data: dict,
+    document_summaries: list,
+    handle_to_source: dict[str, str],
+):
+    """
+    Integrate document summaries into export_data by project field source.
+
+    Args:
+        export_data: Export dictionary (modified in-place)
+        document_summaries: List of DocumentSummaryItem objects
+        handle_to_source: Mapping from handle to source field ("project_information", "project_result")
+    """
+    # Initialize document_summaries structure
+    project_summaries = {
+        "information": [],
+        "result": [],
+    }
+
+    # Group summaries by source field
+    for summary_item in document_summaries:
+        handle = summary_item.handle
+        source = handle_to_source.get(handle)
+
+        if source == "project_information":
+            project_summaries["information"].append(
+                {
+                    "handle": summary_item.handle,
+                    "summary": summary_item.summary,
+                }
+            )
+        elif source == "project_result":
+            project_summaries["result"].append(
+                {
+                    "handle": summary_item.handle,
+                    "summary": summary_item.summary,
+                }
+            )
+
+    # Integrate summaries into export_data
+    if "project" not in export_data:
+        export_data["project"] = {}
+    export_data["project"]["document_summaries"] = project_summaries

apps/summarization/export_utils/attachments/handlers.py

@@ -0,0 +1,124 @@
+from adhocracy4.polls.models import Poll
+from apps.budgeting.models import Proposal
+from apps.debate.models import Subject
+from apps.documents.models import Chapter
+from apps.ideas.models import Idea
+from apps.mapideas.models import MapIdea
+from apps.offlineevents.models import OfflineEvent
+from apps.topicprio.models import Topic
+
+from .models.debates import export_debate
+from .models.documents import export_document_chapters
+from .models.ideas import export_idea
+from .models.mapideas import export_mapidea
+from .models.offline_events import export_offline_event
+from .models.polls import export_poll
+from .models.proposals import export_proposal
+from .models.topics import export_topic
+from .processing.cleaning import clean_export
+from .processing.extractors import extract_attachments
+from .processing.grouping import restructure_by_phase
+from .processing.module_utils import get_module_status
+from .processing.module_utils import get_module_type_from_name
+
+
+def generate_full_export(project):
+    """Generate complete project export data - module first approach"""
+    from adhocracy4.modules.models import Module
+
+    # Project metadata
+    project_data = {
+        "name": project.name,
+        "description": project.description,
+        "description_attachments": extract_attachments(project.description),
+        "information": getattr(project, "information", None),
+        "information_attachments": extract_attachments(
+            getattr(project, "information", "")
+        ),
+        "slug": project.slug,
+        "organisation": project.organisation.name,
+        "result": project.result,
+        "result_attachments": extract_attachments(project.result),
+        "url": project.get_absolute_url(),
+    }
+
+    modules_data = []
+    for module in Module.objects.filter(project=project, is_draft=False):
+        module_data = {
+            "module_id": module.id,
+            "module_name": module.name,
+            "module_type": get_module_type_from_name(module.name),
+            "active_status": get_module_status(module),
+            "module_start": str(module.module_start),
+            "module_end": str(module.module_end),
+            "description": module.description,
+            "url": module.get_absolute_url(),
+            "content": {},
+        }
+
+        # Ideas
+        ideas = Idea.objects.filter(module=module)
+        if ideas.exists():
+            module_data["content"]["ideas"] = [export_idea(i) for i in ideas]
+
+        # MapIdeas
+        mapideas = (
+            MapIdea.objects.filter(module__project=project)
+            .select_related("category")
+            .prefetch_related("labels")
+        )
+        if mapideas.exists():
+            module_data["content"]["mapideas"] = [export_mapidea(m) for m in mapideas]
+
+        # Polls
+        polls = Poll.objects.filter(module=module).prefetch_related(
+            "questions__choices__votes__other_vote",
+        )
+        if polls.exists():
+            module_data["content"]["polls"] = [export_poll(p) for p in polls]
+
+        # Topics
+        topics = (
+            Topic.objects.filter(module=module)
+            .select_related("category")
+            .prefetch_related("labels")
+        )
+        if topics.exists():
+            module_data["content"]["topics"] = [export_topic(t) for t in topics]
+
+        # Proposals
+        proposals = (
+            Proposal.objects.filter(module=module)
+            .select_related("category")
+            .prefetch_related("labels")
+        )
+        if proposals.exists():
+            module_data["content"]["proposals"] = [
+                export_proposal(p) for p in proposals
+            ]
+
+        # Debates
+        debates = Subject.objects.filter(module=module)
+        if debates.exists():
+            module_data["content"]["debates"] = [export_debate(d) for d in debates]
+
+        # Documents
+        if Chapter.objects.filter(module=module).exists():
+            module_data["content"]["documents"] = export_document_chapters(module)
+
+        modules_data.append(module_data)
+
+    # Offline events
+    offline_events = []
+    for event in OfflineEvent.objects.filter(project=project):
+        offline_events.append(export_offline_event(event))
+
+    export_data = {
+        "project": project_data,
+        "modules": modules_data,
+        "offline_events": offline_events,
+    }
+
+    structured_result = restructure_by_phase(export_data)
+    cleaned_result = clean_export(structured_result)
+    return cleaned_result

apps/summarization/export_utils/core.py

@@ -0,0 +1,16 @@
+from ..processing.extractors import extract_comments
+
+
+def export_debate(debate):
+    """Export a single debate subject with all its data."""
+    return {
+        "id": debate.id,
+        "name": debate.name,
+        "description": debate.description,
+        # "created": debate.created.isoformat(),
+        "reference_number": debate.reference_number,
+        "slug": debate.slug,
+        "comment_count": debate.comments.count(),
+        "comments": extract_comments(debate.comments.all()),
+        "comment_creator_count": debate.comment_creator_count,
+    }

apps/summarization/export_utils/models/debates.py

@@ -0,0 +1,46 @@
+from apps.documents.models import Chapter
+
+from ..processing.extractors import extract_attachments
+from ..processing.extractors import extract_comments
+
+
+def export_paragraph(paragraph):
+    """Export a single paragraph."""
+    return {
+        "id": paragraph.id,
+        "name": paragraph.name,
+        "text": str(paragraph.text),
+        "attachments": extract_attachments(str(paragraph.text)),
+        "weight": paragraph.weight,
+        # "created": paragraph.created.isoformat(),
+        "comment_count": paragraph.comments.count(),
+        "comments": extract_comments(paragraph.comments.all()),
+    }
+
+
+def export_document_chapters(module):
+    """Export all chapters and paragraphs for a module."""
+    chapters_data = []
+    chapters = Chapter.objects.filter(module=module).order_by("weight")
+
+    for chapter in chapters:
+        chapters_data.append(
+            {
+                "id": chapter.id,
+                "name": chapter.name,
+                "url": chapter.get_absolute_url(),
+                "weight": chapter.weight,
+                # "created": chapter.created.isoformat(),
+                "prev_chapter_id": chapter.prev.id if chapter.prev else None,
+                "next_chapter_id": chapter.next.id if chapter.next else None,
+                "paragraph_count": chapter.paragraphs.count(),
+                "paragraphs": [
+                    export_paragraph(p)
+                    for p in chapter.paragraphs.all().order_by("weight")
+                ],
+                "chapter_comment_count": chapter.comments.count(),
+                "chapter_comments": extract_comments(chapter.comments.all()),
+            }
+        )
+
+    return chapters_data

apps/summarization/export_utils/models/documents.py

@@ -0,0 +1,22 @@
+from ..processing.extractors import extract_attachments
+from ..processing.extractors import extract_comments
+from ..processing.extractors import extract_ratings
+
+
+def export_idea(idea):
+    """Export a single idea with all its data."""
+    return {
+        "id": idea.id,
+        "name": idea.name,
+        "description": str(idea.description),
+        "attachments": extract_attachments(str(idea.description)),
+        # "created": idea.created.isoformat(),
+        "reference_number": idea.reference_number,
+        "category": idea.category.name if idea.category else None,
+        "labels": [label.name for label in idea.labels.all()],
+        "comment_count": idea.comments.count(),
+        "comments": extract_comments(idea.comments.all()),
+        "rating_count": idea.ratings.count(),
+        "ratings": extract_ratings(idea.ratings.all()),
+        "images": [i.name for i in idea._a4images_current_images],
+    }

apps/summarization/export_utils/models/ideas.py

@@ -0,0 +1,29 @@
+from .ideas import export_idea
+
+
+def export_mapidea(mapidea):
+    """Export a single map idea with all its data."""
+    data = export_idea(mapidea)  # Reuse base idea export
+
+    # Handle point - could be Point object or dict
+    point = None
+    if mapidea.point:
+        if hasattr(mapidea.point, "y"):  # It's a Point object
+            point = {
+                "lat": mapidea.point.y,
+                "lng": mapidea.point.x,
+            }
+        elif isinstance(mapidea.point, dict):  # It's already a dict
+            point = {
+                "lat": mapidea.point.get("y") or mapidea.point.get("lat"),
+                "lng": mapidea.point.get("x") or mapidea.point.get("lng"),
+            }
+
+    data.update(
+        {
+            "point": point,
+            "point_label": mapidea.point_label,
+        }
+    )
+
+    return data

apps/summarization/export_utils/models/mapideas.py

@@ -0,0 +1,18 @@
+from ..processing.extractors import extract_attachments
+
+
+def export_offline_event(event):
+    """Export a single offline event."""
+    return {
+        "id": event.id,
+        "name": event.name,
+        "event_type": event.event_type,
+        "date": event.date.isoformat(),
+        "description": str(event.description),
+        "attachments": extract_attachments(str(event.description)),
+        "slug": event.slug,
+        "url": event.get_absolute_url(),
+        "timeline_index": event.get_timeline_index,
+        # "created": event.created.isoformat(),
+        "modified": event.modified.isoformat() if event.modified else None,
+    }