feat: Add grant source management with CRUD operations (Phase 2.2)

README.md

@@ -27,35 +27,35 @@ graph LR
     %% PHASE 1: DISCOVERY
     subgraph PHASE_1 [PHASE 1: DISCOVERY & MATCHING]
         direction TB
-        Sources[📡 Sources: EU/NSF/Foundations] --> Firecrawl[🔥 Firecrawl Engine]
-        Firecrawl --> Analyzer[💎 AI Feature/Relevance Extractor]
+        Sources[Sources: EU/NSF/Foundations] --> Firecrawl[Firecrawl Engine]
+        Firecrawl --> Analyzer[AI Feature/Relevance Extractor]
         Analyzer -->|Score > 50| DB[(Supabase DB)]
-        Analyzer -->|Score < 50| Discard[🗑️ Discard]
+        Analyzer -->|Score < 50| Discard[Discard]
     end
 
     %% PHASE 2: INTERFACE
     subgraph PHASE_2 [PHASE 2: PLATFORM]
-        DB <==>|Real-time| Dashboard[🖥️ SvelteKit Dashboard]
-        Dashboard -->|User Clicks Apply| API[⚡ API Gateway]
+        DB <==>|Real-time| Dashboard[SvelteKit Dashboard]
+        Dashboard -->|User Clicks Apply| API[API Gateway]
     end
 
     %% PHASE 3: ORCHESTRATION
     subgraph PHASE_3 [PHASE 3: ORCHESTRATION]
         API --> Strategy{Strategy Router}
-        Strategy -->|Fast Track| Gemini[✨ Gemini 2.5 Pro]
-        Strategy -->|Research Track| LabRunner[🔬 Lab Orchestrator]
+        Strategy -->|Fast Track| Gemini[Gemini 2.5 Pro]
+        Strategy -->|Research Track| LabRunner[Lab Orchestrator]
     end
 
     %% PHASE 4: AGENT LAB
     subgraph PHASE_4 [PHASE 4: AGENT LABORATORY]
         direction TB
-        LabRunner --> PhD[📚 PhD Student\n(Claude Opus 4.5)]
-        PhD --> Postdoc1[📋 Postdoc Plan\n(Gemini 3 Pro)]
-        Postdoc1 --> Eng[🛠️ ML & SW Engineers\n(GPT-5 Codex + Claude)]
-        Eng --> Postdoc2[📈 Results Analysis\n(Gemini 3 Pro)]
-        Postdoc2 --> Prof[📝 Professor Writing\n(Claude Sonnet 4.5)]
-        Prof --> Review[✨ Review Board\n(Gemini 3 Pro)]
-        Review --> Final[📄 Research-Grade Proposal]
+        LabRunner --> PhD[PhD Student - Claude Opus 4.5]
+        PhD --> Postdoc1[Postdoc Plan - Gemini 3 Pro]
+        Postdoc1 --> Eng[ML and SW Engineers - GPT-5 Codex]
+        Eng --> Postdoc2[Results Analysis - Gemini 3 Pro]
+        Postdoc2 --> Prof[Professor Writing - Claude Sonnet 4.5]
+        Prof --> Review[Review Board - Gemini 3 Pro]
+        Review --> Final[Research-Grade Proposal]
     end
 ```
 

backend/.dockerignore

@@ -0,0 +1,12 @@
+node_modules
+npm-debug.log
+.env
+.env.local
+.git
+.gitignore
+README.md
+*.md
+test_services.js
+output.txt
+output_2.txt
+ai-researcher

backend/Dockerfile

@@ -0,0 +1,24 @@
+# Use Node.js LTS
+FROM node:20-slim
+
+# Set working directory
+WORKDIR /app
+
+# Copy package files
+COPY package*.json ./
+
+# Install dependencies
+RUN npm ci --only=production
+
+# Copy source code
+COPY . .
+
+# Expose port
+EXPOSE 3000
+
+# Set environment variables
+ENV NODE_ENV=production
+ENV PORT=3000
+
+# Start the server
+CMD ["npm", "start"]

backend/fly.toml

@@ -0,0 +1,33 @@
+# fly.toml - Fly.io configuration for AI Grant Crawler Backend
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+
+app = "ai-grant-crawler-backend"
+primary_region = "iad"
+
+[build]
+  dockerfile = "Dockerfile"
+
+[env]
+  NODE_ENV = "production"
+  PORT = "3000"
+
+[http_service]
+  internal_port = 3000
+  force_https = true
+  auto_stop_machines = "stop"
+  auto_start_machines = true
+  min_machines_running = 0
+  processes = ["app"]
+
+[[vm]]
+  memory = "512mb"
+  cpu_kind = "shared"
+  cpus = 1
+
+# Health check
+[[services.http_checks]]
+  interval = "30s"
+  timeout = "5s"
+  grace_period = "10s"
+  method = "GET"
+  path = "/health"

backend/schema.sql

@@ -111,11 +111,17 @@ CREATE TABLE IF NOT EXISTS grant_sources (
     id SERIAL PRIMARY KEY,
     url TEXT NOT NULL UNIQUE,
     name TEXT,
-    type TEXT, -- 'portal', 'aggregator', 'direct'
+    type TEXT, -- 'portal', 'aggregator', 'direct', 'manual'
     last_crawled_at TIMESTAMP,
-    status TEXT DEFAULT 'active'
+    status TEXT DEFAULT 'active',
+    scrape_frequency TEXT DEFAULT 'daily', -- 'hourly', 'daily', 'weekly', 'monthly'
+    created_at TIMESTAMP DEFAULT NOW()
 );
 
+-- Add scrape_frequency column if it doesn't exist (for existing databases)
+ALTER TABLE grant_sources ADD COLUMN IF NOT EXISTS scrape_frequency TEXT DEFAULT 'daily';
+ALTER TABLE grant_sources ADD COLUMN IF NOT EXISTS created_at TIMESTAMP DEFAULT NOW();
+
 -- Grant Discovery columns
 ALTER TABLE grants ADD COLUMN IF NOT EXISTS relevance_score INTEGER DEFAULT 0;
 ALTER TABLE grants ADD COLUMN IF NOT EXISTS keywords TEXT[];

backend/src/routes/admin/sources.js

@@ -11,21 +11,85 @@ const supabase = createClient(
 
 // GET all sources
 router.get("/", async (req, res) => {
-  const { data, error } = await supabase.from("grant_sources").select("*");
+  const { data, error } = await supabase
+    .from("grant_sources")
+    .select("*")
+    .order("created_at", { ascending: false });
   if (error) return res.status(500).json({ error: error.message });
   res.json(data);
 });
 
 // ADD new source
 router.post("/", async (req, res) => {
-  const { url, name, type } = req.body;
+  const { url, name, type, scrape_frequency } = req.body;
+  const { data, error } = await supabase
+    .from("grant_sources")
+    .insert([{ 
+      url, 
+      name, 
+      type: type || 'manual',
+      status: 'active',
+      scrape_frequency: scrape_frequency || 'daily'
+    }])
+    .select();
+
+  if (error) return res.status(500).json({ error: error.message });
+  res.json(data[0]);
+});
+
+// UPDATE source (toggle status, update frequency)
+router.patch("/:id", async (req, res) => {
+  const { id } = req.params;
+  const { status, scrape_frequency, name, url } = req.body;
+
+  const updates = {};
+  if (status !== undefined) updates.status = status;
+  if (scrape_frequency !== undefined) updates.scrape_frequency = scrape_frequency;
+  if (name !== undefined) updates.name = name;
+  if (url !== undefined) updates.url = url;
+
   const { data, error } = await supabase
     .from("grant_sources")
-    .insert([{ url, name, type }])
+    .update(updates)
+    .eq("id", id)
     .select();
 
   if (error) return res.status(500).json({ error: error.message });
+  if (!data || data.length === 0) return res.status(404).json({ error: "Source not found" });
   res.json(data[0]);
 });
 
+// DELETE source
+router.delete("/:id", async (req, res) => {
+  const { id } = req.params;
+
+  const { error } = await supabase
+    .from("grant_sources")
+    .delete()
+    .eq("id", id);
+
+  if (error) return res.status(500).json({ error: error.message });
+  res.json({ success: true });
+});
+
+// GET source health/stats
+router.get("/stats", async (req, res) => {
+  const { data: sources, error } = await supabase
+    .from("grant_sources")
+    .select("*");
+
+  if (error) return res.status(500).json({ error: error.message });
+
+  const stats = {
+    total: sources.length,
+    active: sources.filter(s => s.status === 'active').length,
+    inactive: sources.filter(s => s.status === 'inactive').length,
+    lastCrawled: sources
+      .filter(s => s.last_crawled_at)
+      .sort((a, b) => new Date(b.last_crawled_at) - new Date(a.last_crawled_at))[0]?.last_crawled_at || null
+  };
+
+  res.json(stats);
+});
+
 export default router;