feat: add support for flashcards, translations, and document search (Gemini)

Author: Manishram-ai

backend/agent.py

@@ -247,6 +247,40 @@
                         },
                         "description": "Relevant educational YouTube video URLs",
                     },
+                    "learning_objectives": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "3-5 learning objectives the block covers",
+                    },
+                    "flashcards": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "front": {"type": "string"},
+                                "back": {"type": "string"},
+                                "variant": {
+                                    "type": "string",
+                                    "enum": ["vocabulary", "terminology", "concept", "formula", "definition"],
+                                },
+                            },
+                            "required": ["front", "back"],
+                        },
+                        "description": "Flashcards for spaced repetition study. Each card has a front (question/term) and back (answer/definition).",
+                    },
+                    "translations": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "original": {"type": "string"},
+                                "translated": {"type": "string"},
+                                "language": {"type": "string"},
+                            },
+                            "required": ["original", "translated"],
+                        },
+                        "description": "Side-by-side translations of key terms, phrases, or passages. Include the target language name.",
+                    },
                 },
                 "required": ["content"],
             },
@@ -1142,7 +1176,7 @@ def execute_tool(name, arguments, session=None):
             "content": content,
         }
         # Pass through structured fields if provided.
-        for field_name in ("learning_objectives", "key_terms", "references", "quiz_questions", "youtube_urls"):
+        for field_name in ("learning_objectives", "key_terms", "references", "quiz_questions", "youtube_urls", "flashcards", "translations"):
             val = arguments.get(field_name)
             if isinstance(val, list) and val:
                 result[field_name] = val
@@ -1334,6 +1368,31 @@ def _build_content_nodes(created_main_block, tool_results=None):
         if questions:
             nodes.append({"type": "quiz", "questions": questions})
 
+    fc = created_main_block.get("flashcards")
+    if isinstance(fc, list) and fc:
+        cards = []
+        for i, card in enumerate(fc):
+            if isinstance(card, dict) and card.get("front") and card.get("back"):
+                cards.append({
+                    "id": f"fc-{i + 1}",
+                    "front": str(card["front"]),
+                    "back": str(card["back"]),
+                    "variant": str(card.get("variant") or "concept"),
+                })
+        if cards:
+            nodes.append({"type": "flashcards", "cards": cards})
+
+    translations = created_main_block.get("translations")
+    if isinstance(translations, list) and translations:
+        for i, t in enumerate(translations):
+            if isinstance(t, dict) and t.get("original") and t.get("translated"):
+                nodes.append({
+                    "type": "translation",
+                    "original": str(t["original"]),
+                    "translated": str(t["translated"]),
+                    "language": str(t.get("language") or ""),
+                })
+
     yt = created_main_block.get("youtube_urls")
     if isinstance(yt, list) and yt:
         for y in yt:

backend/lambda_handler.py

@@ -6055,14 +6055,40 @@ def process_async_job(event):
     requested_content_types = event.get("requested_content_types") or []
     owner_user_id = str(event.get("owner_user_id") or "").strip()
 
-    # Prepend content-type requests to user message so the agent knows what to include.
+    # Prepend content-type requests with explicit format guidance per type.
+    _CONTENT_TYPE_FORMAT_HINTS = {
+        "quiz": "quiz -> use `create_main_block` with `quiz_questions` array (multiple_choice or true_false)",
+        "flashcards": "flashcards -> use `create_main_block` with `flashcards` array ({front, back, variant})",
+        "images": "images -> call `search_brave_images` or `search_wiki_images` inline",
+        "animation": "animation -> call `generate_math_animation` for math/physics/CS concepts",
+        "youtube": "youtube embed -> use `create_main_block` with `youtube_urls` array",
+        "tables": "tables -> include markdown tables (| col1 | col2 |) in your content",
+        "graphs": "graphs/charts -> include a markdown table with quantitative data AND append a chart tag on the next line (<<bar>>, <<pie>>, <<line>>, <<scatter>>, <<area>>, etc.)",
+        "code": "code -> include fenced code blocks with language tags (```python, ```javascript, etc.)",
+        "equations": "equations -> include LaTeX math: $inline$ or $$display$$ blocks",
+        "quote": "quote -> include blockquotes using > prefix",
+        "definition": "definition -> use blockquote format: > **Definition: Term** — explanation",
+        "translation": "translation -> use `create_main_block` with `translations` array ({original, translated, language})",
+    }
     if requested_content_types and isinstance(requested_content_types, list):
-        types_str = ", ".join(str(t) for t in requested_content_types)
-        user_message = (
-            f"[IMPORTANT — The user has specifically requested the following content types be included "
-            f"in your response: {types_str}. You MUST include these content types in your answer. "
-            f"Use the appropriate tools and formatting for each requested type.]\n\n{user_message}"
+        hints = []
+        plain_names = []
+        for t in requested_content_types:
+            t_str = str(t).strip()
+            if not t_str:
+                continue
+            plain_names.append(t_str)
+            hint = _CONTENT_TYPE_FORMAT_HINTS.get(t_str)
+            if hint:
+                hints.append(f"  - {hint}")
+        types_str = ", ".join(plain_names)
+        format_block = "\n".join(hints) if hints else ""
+        instruction = (
+            f"[IMPORTANT — The user has specifically requested the following content types: {types_str}.\n"
+            f"You MUST include these in your answer. Format guidance:\n"
+            f"{format_block}]\n\n"
         )
+        user_message = instruction + user_message
 
     started_at = time.time()
     print(f"Processing job {job_id} for session {session_id}")

backend/tutor_prompts.py

@@ -202,6 +202,15 @@ def get_threaded_response_system_prompt(
         "- Use a search query specific to the user's follow-up question or the concept being explained.",
         "- Do NOT include images in a concluding section.",
         "",
+        "Structured-field mapping (when user requests specific content types, use `create_main_block`):",
+        "- quiz -> `quiz_questions`",
+        "- flashcards -> `flashcards` (array of {front, back, variant})",
+        "- translation -> `translations` (array of {original, translated, language})",
+        "- key terms -> `key_terms`",
+        "- learning objectives -> `learning_objectives`",
+        "- references -> `references`",
+        "- youtube/video resources -> `youtube_urls`",
+        "",
         f"Parent block topic: {safe_topic}",
         "",
         "Full parent block content (use this context):",
@@ -242,6 +251,8 @@ def get_component_edit_system_prompt(
         "",
         "Structured-field mapping guidance:",
         "- quiz -> `quiz_questions`",
+        "- flashcards -> `flashcards` (array of {front, back, variant})",
+        "- translation -> `translations` (array of {original, translated, language})",
         "- key terms -> `key_terms`",
         "- learning objectives -> `learning_objectives`",
         "- references -> `references`",
@@ -319,7 +330,7 @@ def get_component_edit_system_prompt(
 - The `create_main_block` tool is the primary path for delivering final teaching content.
 - For simple queries (weather, basic math), DO NOT call this tool. Just answer them directly.
 - For educational queries, use your best judgment given the user's profile to call `create_main_block` with rich textbook-style markdown content.
-- **CRITICAL**: If the user explicitly asks for quizzes, flashcards, key terms, or learning objectives (even as a follow-up), you MUST use the `create_main_block` tool and populate the corresponding structured fields. Do NOT output raw quiz text in your direct response.
+- **CRITICAL**: If the user explicitly asks for quizzes, flashcards, translations, key terms, or learning objectives (even as a follow-up), you MUST use the `create_main_block` tool and populate the corresponding structured fields (`quiz_questions`, `flashcards`, `translations`, `key_terms`, `learning_objectives`). Do NOT output raw quiz or flashcard text in your direct response.
 
 **ADAPTIVE PROFILE - IMPORTANT:**
 You have access to the `update_learner_profile` tool. Use it to dynamically adjust the user's profile when you observe:

frontend/src/components/app/ChatInput.tsx

@@ -17,7 +17,7 @@ import { usePreparedAttachments } from "@/hooks/usePreparedAttachments"
 import { useAuth } from "@/context/AuthContext"
 import ContentTypePicker, { ContentTypeChips } from "@/components/ui/ContentTypePicker"
 import { readStoredModel, writeStoredModel } from "@/lib/models"
-import { readStoredContentTypes, writeStoredContentTypes } from "@/lib/contentTypes"
+import { readStoredContentTypes, writeStoredContentTypes, buildContentTypeAutoPrompt } from "@/lib/contentTypes"
 
 /* ═══════════════════════════════════════════════════════════
  *  ChatInput — memoized input component with typewriter,
@@ -103,7 +103,8 @@ const ChatInput = memo(function ChatInput() {
 
   const handleAttachClick = useCallback(() => fileInputRef.current?.click(), [])
 
-  const hasInput = inputValue.length > 0 || attachments.length > 0
+  const hasContentTypes = (inputFeatures.requestedContentTypes?.length ?? 0) > 0
+  const hasInput = inputValue.length > 0 || attachments.length > 0 || hasContentTypes
   const lit = hasInput && sendHover
 
   const handleFileChange = useCallback(async (e: React.ChangeEvent<HTMLInputElement>) => {
@@ -144,10 +145,12 @@ const ChatInput = memo(function ChatInput() {
     if (localSendInFlightRef.current || isSending) return
 
     const content = inputValue.trim()
-    if (!content && attachments.length === 0) return
+    if (!content && attachments.length === 0 && !hasContentTypes) return
     if (!canSend) return
 
-    const messageText = content || "See attached files."
+    // Auto-generate prompt when user selected content types but typed nothing
+    const messageText = content
+      || (hasContentTypes ? buildContentTypeAutoPrompt(inputFeatures.requestedContentTypes!) : "See attached files.")
     const msgAttachments = attachments.length > 0 ? attachedToMessageFormat(attachments) : undefined
     const previousInputValue = inputValue
     const previousAttachments = attachments

frontend/src/components/blocks/SessionView.tsx

@@ -38,7 +38,7 @@ import {
 } from "@/lib/models"
 import { EditIcon, TrashIcon } from "@/components/app/Icons"
 import ContentTypePicker, { ContentTypeChips } from "@/components/ui/ContentTypePicker"
-import { readStoredContentTypes, writeStoredContentTypes } from "@/lib/contentTypes"
+import { readStoredContentTypes, writeStoredContentTypes, buildContentTypeAutoPrompt } from "@/lib/contentTypes"
 import type { BlockActionRequest } from "@/types/block-actions"
 
 /* ═══════════════════════════════════════════════════════════
@@ -2200,6 +2200,7 @@ export function SessionView({ initialSession, backendSessionId, readOnly = false
             onDismissHighlightQuote={() => setHighlightQuote(null)}
             contentTypes={inputContentTypes}
             onContentTypesChange={setInputContentTypes}
+            topicHint={activeBlock?.title || activeBlock?.originQuestion}
           />
         </div>
       </div>
@@ -2852,6 +2853,8 @@ interface BlockChatInputBarProps {
   /** Content type picker */
   contentTypes?: string[]
   onContentTypesChange?: (ids: string[]) => void
+  /** Topic hint for auto-prompt when user sends with content types but no text */
+  topicHint?: string
 }
 
 export function BlockChatInputBar({
@@ -2871,6 +2874,7 @@ export function BlockChatInputBar({
   onDismissHighlightQuote,
   contentTypes: contentTypesProp,
   onContentTypesChange,
+  topicHint,
 }: BlockChatInputBarProps) {
   const COLORS = useThemeColors()
   const isDark = useIsDark()
@@ -2919,7 +2923,8 @@ export function BlockChatInputBar({
   )
   const mic = useSpeechRecognition(onTranscript)
 
-  const hasInput = value.length > 0 || attachments.length > 0 || !!highlightQuote
+  const hasContentTypes = contentTypes.length > 0
+  const hasInput = value.length > 0 || attachments.length > 0 || !!highlightQuote || hasContentTypes
   const lit = hasInput && sendHover
 
   const handleFileChange = useCallback(async (e: React.ChangeEvent<HTMLInputElement>) => {
@@ -2946,16 +2951,19 @@ export function BlockChatInputBar({
 
   const doSend = useCallback(async () => {
     const text = value.trim()
-    if (!text && attachments.length === 0 && !highlightQuote) return
+    if (!text && attachments.length === 0 && !highlightQuote && !hasContentTypes) return
     if (!canSend) return
-    const didSend = await onSend(text || " ", attachments, webSearch, {
+    // Auto-generate prompt when user selected content types but typed nothing
+    const sendText = text
+      || (hasContentTypes ? buildContentTypeAutoPrompt(contentTypes, topicHint) : " ")
+    const didSend = await onSend(sendText, attachments, webSearch, {
       preparedUploadIds: preparedUploadIds.length ? preparedUploadIds : undefined,
       preparedBundleId,
     })
     if (!didSend) return
     onChange("")
     clearAttachments()
-  }, [value, attachments, onSend, onChange, webSearch, highlightQuote, canSend, preparedUploadIds, preparedBundleId, clearAttachments])
+  }, [value, attachments, onSend, onChange, webSearch, highlightQuote, hasContentTypes, contentTypes, topicHint, canSend, preparedUploadIds, preparedBundleId, clearAttachments])
 
   const bg = isDark ? "#0A0A0A" : "#F8F8F8"
   const border = isDark ? "rgba(255,255,255,0.2)" : "rgba(0,0,0,0.18)"

frontend/src/lib/contentTypes.ts

@@ -68,6 +68,22 @@ export const CONTENT_TYPE_ICONS: Record<string, string> = {
   simulation: "M21 16V8a2 2 0 00-1-1.73l-7-4a2 2 0 00-2 0l-7 4A2 2 0 003 8v8a2 2 0 001 1.73l7 4a2 2 0 002 0l7-4A2 2 0 0021 16z",
 }
 
+/** Build an auto-prompt when user sends with content types selected but no custom text.
+ *  Uses the active block's topic as context so the LLM generates relevant content. */
+export function buildContentTypeAutoPrompt(ids: string[], topicHint?: string): string {
+  const labels = ids
+    .map((id) => CONTENT_TYPE_MAP[id]?.label)
+    .filter(Boolean)
+  if (labels.length === 0) return ""
+  const joined = labels.length === 1
+    ? labels[0].toLowerCase()
+    : labels.slice(0, -1).map((l) => l.toLowerCase()).join(", ") + " and " + labels[labels.length - 1].toLowerCase()
+  if (topicHint) {
+    return `Generate ${joined} about "${topicHint}".`
+  }
+  return `Generate ${joined} based on what we've covered so far.`
+}
+
 const STORAGE_KEY = "fp-requested-content-types"
 
 export function readStoredContentTypes(): string[] {