Merge pull request #34 from TBosak/main

Implementing webhook configuration

Author: TBosak

.gitignore

@@ -166,6 +166,7 @@ dist
 .yarn/unplugged
 .yarn/build-state.yml
 .yarn/install-state.gz
-.pnp.\*
+.pnp.*
 public/feeds/
-configs/
+configs/
+feed-history/

README.md

@@ -98,20 +98,20 @@ Some websites structure content across multiple layers:
 
 ## 🔧 To Do
 
-- [ ] Add ALL possible RSS fields to models
+- [x] Add ALL possible RSS fields to models
 - [x] Add option for parallel iterators
 - [ ] Scraping how-to video
 - [x] Add feed preview pane
 - [ ] Store/compare feed data to enable timestamping feed items
 - [x] Create dockerfile
 - [ ] Create Helm chart files
 - [x] Create GUI
-- [ ] Utilities
+- [x] Utilities
   - [x] HTML stripper
   - [x] Source URL wrapper for relative links
-  - [ ] Nested link follower/drilldown functionality for each feed item property
+  - [x] Nested link follower/drilldown functionality for each feed item property
   - [x] Adjust date parser logic with overrides from an optional date format input
-  - [ ] Add selector suggestion engine
+  - [x] Add selector suggestion engine
 - [ ] Amass contributors
 
 <br>

index.ts

@@ -364,11 +364,22 @@ app.post("/", async (ctx) => {
       user: extract("emailUsername"),
       encryptedPassword: encrypt(extract("emailPassword"), encryptionKey),
       folder: extract("emailFolder"),
+      emailCount: parseInt(extract("emailCount", "10")) || 10,
     };
     feedOptions.feedLanguage = "en"; 
     feedOptions.feedDescription = `Emails from folder: ${emailConfigData.folder}`;
   }
 
+  // Webhook configuration
+  const webhookConfig = {
+    enabled: extractBool("webhookEnabled"),
+    url: extract("webhookUrl", ""),
+    format: extract("webhookFormat", "xml") as "xml" | "json",
+    newItemsOnly: extractBool("webhookNewItemsOnly", true),
+    headers: extractJson("webhookHeaders"),
+    customPayload: extract("webhookCustomPayload", "").trim() || undefined,
+  };
+
   const finalFeedConfig = {
     feedId,
     feedName,
@@ -379,6 +390,7 @@ app.post("/", async (ctx) => {
     advanced: extractBool("advanced"), 
     headers: extractJson("headers"),   
     cookies: cookies.length > 0 ? cookies : undefined, 
+    webhook: webhookConfig.enabled && webhookConfig.url ? webhookConfig : undefined,
 
     config: feedType === "email" ? emailConfigData : configData, 
     ...(feedType === "webScraping" && { article: articleData }), 
@@ -647,6 +659,28 @@ app.get("/feeds", async (ctx) => {
           function confirmDelete(feedId) {
             return confirm("Are you sure you want to delete this feed?");
           }
+          
+          async function triggerWebhook(feedId) {
+            try {
+              const response = await fetch('/trigger-webhook', {
+                method: 'POST',
+                headers: {
+                  'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({ feedId })
+              });
+              
+              const result = await response.json();
+              
+              if (response.ok) {
+                alert('Webhook triggered successfully!\\n\\nFeed: ' + feedId + '\\nWebhook URL: ' + result.webhookUrl + '\\nItems sent: ' + result.itemCount);
+              } else {
+                alert('Webhook failed: ' + result.error);
+              }
+            } catch (error) {
+              alert('Error triggering webhook: ' + error.message);
+            }
+          }
         </script>
         <header style="text-align:center;"><h1>Active RSS Feeds</h1></header>
         <div>
@@ -677,6 +711,7 @@ app.get("/feeds", async (ctx) => {
     }
 
     // Build the card for this feed
+    const hasWebhook = config.webhook?.enabled && config.webhook?.url;
     response += `
       <article>
         <header>
@@ -685,14 +720,20 @@ app.get("/feeds", async (ctx) => {
         <p><strong>Feed ID:</strong> ${feedId}</p>
         <p><strong>Build Time:</strong> ${lastBuildDate}</p>
         <p><strong>Feed Type:</strong> ${feedType}</p>
+        ${hasWebhook ? `<p><strong>Webhook:</strong> ✅ Enabled</p>` : '<p><strong>Webhook:</strong> ❌ Disabled</p>'}
         <footer>
         <div class="grid">
             <a href="public/feeds/${feedId}.xml" style="margin-right: auto;line-height:3em;">View Feed</a>
+            ${hasWebhook ? `
+            <button onclick="triggerWebhook('${feedId}')" class="outline">
+              🪝 Trigger Webhook
+            </button>
+            ` : ''}
             <form action="/delete-feed" method="POST" style="display:inline;" onsubmit="return confirmDelete('${feedId}')">
               <input type="hidden" name="feedId" value="${feedId}">
               <button type="submit" style="width:25%;margin-left:auto;float:right;" class="outline contrast">Delete</button>
+            </form>
           </div>
-          </form>
         </footer>
       </article>
     `;
@@ -827,6 +868,66 @@ app.post("/delete-feed", async (c) => {
   }
 });
 
+app.post("/trigger-webhook", async (c) => {
+  const { feedId } = await c.req.json();
+
+  if (!feedId) {
+    return c.json({ error: "Feed ID is required" }, 400);
+  }
+
+  try {
+    const sanitizedFeedId = basename(feedId as string);
+    const configPath = join(configsDir, `${sanitizedFeedId}.yaml`);
+    
+    // Check if feed exists
+    if (!existsSync(configPath)) {
+      return c.json({ error: "Feed not found" }, 404);
+    }
+
+    // Load feed configuration
+    const yamlContent = await readFile(configPath, "utf8");
+    const feedConfig = yaml.load(yamlContent) as any;
+
+    if (!feedConfig.webhook?.enabled || !feedConfig.webhook?.url) {
+      return c.json({ error: "Webhook not configured for this feed" }, 400);
+    }
+
+    // Read current RSS feed
+    const rssPath = join(feedPath, `${sanitizedFeedId}.xml`);
+    if (!existsSync(rssPath)) {
+      return c.json({ error: "RSS feed not generated yet" }, 404);
+    }
+
+    const rssXml = await readFile(rssPath, "utf8");
+    
+    // Import webhook utilities
+    const { sendWebhook, createWebhookPayload, createJsonWebhookPayload } = await import("./utilities/webhook.utility");
+    
+    // Create webhook payload
+    const payload = feedConfig.webhook.format === "json"
+      ? createJsonWebhookPayload(feedConfig, rssXml, "manual")
+      : createWebhookPayload(feedConfig, rssXml, "manual");
+
+    // Send webhook
+    const success = await sendWebhook(feedConfig.webhook, payload);
+
+    if (success) {
+      return c.json({ 
+        message: "Webhook triggered successfully", 
+        feedId: sanitizedFeedId,
+        webhookUrl: feedConfig.webhook.url,
+        itemCount: payload.itemCount
+      });
+    } else {
+      return c.json({ error: "Failed to send webhook" }, 500);
+    }
+
+  } catch (error) {
+    console.error("Error triggering webhook:", error);
+    return c.json({ error: "Internal server error" }, 500);
+  }
+});
+
 app.post("/imap/folders", async (c) => {
   const config = await c.req.json<Config>();
   console.log("IMAP config:", config);

node/imap-watch.utility.ts

@@ -5,7 +5,7 @@ import Imap from "node-imap";
 import libmime from "libmime";
 import minimist from "minimist";
 import RSS from "rss";
-import { existsSync, readFileSync, writeFileSync } from "fs";
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
 import { simpleParser } from "mailparser";
 import { decrypt } from "../utilities/security.utility.ts";
 import { fileURLToPath } from "url";
@@ -62,6 +62,7 @@ interface RSSItemOptions {
 }
 
 interface RSSFeedOptions {
+    webhook: any;
     title: string;                      // Title of the feed
     description: string;                // Description of the feed
     feed_url: string;                   // URL of the RSS feed itself (where it will be published)
@@ -207,7 +208,7 @@ class ImapWatcher {
       this.fetchRecentStartupEmails();
 
       this.imap.on("mail", (n) => {
-        console.log(`[IMAP] New mail event: ${n}`);
+        console.log(`[IMAP] New mail event received for feed ${this.config.feedId}: ${n} new email(s)`);
         this.fetchNewEmails();
       });
 
@@ -438,15 +439,23 @@ class ImapWatcher {
             .map((result) => (result as PromiseFulfilledResult<Email>).value);
 
           if (emails.length > 0) {
-            console.log("[IMAP] Recent emails fetched, updating RSS...");
+            console.log(`[IMAP] Recent emails fetched for feed ${this.config.feedId}, updating RSS with ${emails.length} emails...`);
             const rss = buildRSSFromEmailFolder(emails, this.config);
             writeFileSync(
               path.join(__dirname, "../public/feeds", `${this.config.feedId}.xml`),
               rss,
             );
-            console.log("[IMAP] RSS Feed regenerated");
+            console.log(`[IMAP] RSS Feed regenerated for feed ${this.config.feedId}`);
+            
+            // Handle webhook if configured
+            if (this.config.webhook?.enabled && this.config.webhook?.url) {
+              console.log(`[IMAP] Calling webhook handler for feed ${this.config.feedId}`);
+              this.handleWebhook(rss);
+            } else {
+              console.log(`[IMAP] Webhook not configured for feed ${this.config.feedId} - enabled: ${this.config.webhook?.enabled}, url: ${!!this.config.webhook?.url}`);
+            }
           } else {
-            console.log("[IMAP] No valid emails found.");
+            console.log(`[IMAP] No valid emails found for feed ${this.config.feedId}`);
           }
         });
         console.log("[IMAP] Completed processing new emails.");
@@ -469,6 +478,97 @@ class ImapWatcher {
       this.imap.end();
     }
   }
+
+  private async handleWebhook(rssXml: string): Promise<void> {
+    try {
+      console.log(`[IMAP] Webhook handler called for feed ${this.config.feedId}`);
+      
+      if (!this.config.webhook?.enabled || !this.config.webhook?.url) {
+        console.log(`[IMAP] Webhook not configured for feed ${this.config.feedId} - enabled: ${this.config.webhook?.enabled}, url: ${!!this.config.webhook?.url}`);
+        return;
+      }
+      
+      console.log(`[IMAP] Webhook configured for feed ${this.config.feedId} - URL: ${this.config.webhook.url}, format: ${this.config.webhook.format}, newItemsOnly: ${this.config.webhook.newItemsOnly}`);
+
+      let shouldSendWebhook = true;
+      let webhookRssXml = rssXml;
+      
+      // Check if only new items should be sent
+      if (this.config.webhook.newItemsOnly) {
+        const historyPath = path.join(__dirname, "../feed-history", `${this.config.feedId}.xml`);
+        let previousRss = null;
+        
+        try {
+          if (existsSync(historyPath)) {
+            previousRss = readFileSync(historyPath, "utf8");
+          }
+        } catch (err) {
+          console.warn("[IMAP] Could not read feed history:", err.message);
+        }
+        
+        // Use centralized new item detection
+        try {
+          const { getNewItemsFromRSS } = await import("../utilities/webhook.utility.ts");
+          const newItemsRss = getNewItemsFromRSS(rssXml, previousRss);
+          
+          if (!newItemsRss) {
+            shouldSendWebhook = false;
+            console.log(`[IMAP] No new items detected for feed ${this.config.feedId}, skipping webhook`);
+          } else {
+            webhookRssXml = newItemsRss;
+            console.log(`[IMAP] New items detected for feed ${this.config.feedId}, will send webhook`);
+          }
+        } catch (importErr) {
+          console.warn("[IMAP] Could not import webhook utilities, falling back to simple comparison:", importErr.message);
+          // Fallback to simple comparison
+          if (previousRss && previousRss.trim() === rssXml.trim()) {
+            shouldSendWebhook = false;
+            console.log("[IMAP] No changes detected, skipping webhook");
+          }
+        }
+      }
+      
+      if (shouldSendWebhook) {
+        try {
+          // Use centralized webhook system
+          const { sendWebhook, createWebhookPayload, createJsonWebhookPayload } = await import("../utilities/webhook.utility.ts");
+          
+          const payload = this.config.webhook.format === "json"
+            ? createJsonWebhookPayload(this.config, webhookRssXml, "automatic")
+            : createWebhookPayload(this.config, webhookRssXml, "automatic");
+
+          const success = await sendWebhook(this.config.webhook, payload);
+          
+          if (success) {
+            console.log(`[IMAP] Webhook sent successfully for feed ${this.config.feedId} to ${this.config.webhook.url}`);
+            
+            // Store current RSS for future comparison using centralized history utility
+            try {
+              const { storeFeedHistory } = await import("../utilities/feed-history.utility.ts");
+              await storeFeedHistory(this.config.feedId, rssXml);
+              console.log(`[IMAP] Feed history stored for feed ${this.config.feedId}`);
+            } catch (historyErr) {
+              console.warn(`[IMAP] Could not store feed history for feed ${this.config.feedId}:`, historyErr.message);
+              // Fallback to manual file storage
+              const historyDir = path.join(__dirname, "../feed-history");
+              if (!existsSync(historyDir)) {
+                mkdirSync(historyDir, { recursive: true });
+              }
+              writeFileSync(path.join(historyDir, `${this.config.feedId}.xml`), rssXml, "utf8");
+              console.log(`[IMAP] Feed history stored manually for feed ${this.config.feedId}`);
+            }
+          } else {
+            console.warn(`[IMAP] Webhook failed for feed ${this.config.feedId} to ${this.config.webhook.url}`);
+          }
+        } catch (webhookErr) {
+          console.error(`[IMAP] Error using centralized webhook system:`, webhookErr.message);
+        }
+      }
+    } catch (error) {
+      console.error(`[IMAP] Webhook error:`, error.message);
+    }
+  }
+
 }
 
 export function buildRSSFromEmailFolder(emails: Email[], feedSetup: RSSFeedOptions): string {
@@ -591,9 +691,16 @@ const completeFeedConfig: RSSFeedOptions = {
     feedImage: rawConfig.feedImage,
     generator: rawConfig.generator,
     config: imapOriginalConfig,
+    webhook: rawConfig.webhook, // Pass through webhook configuration
 };
 
 console.log("[IMAP Node Watcher] ImapWatcher will attempt to connect to host:", completeFeedConfig.config?.host, "port:", completeFeedConfig.config?.port);
+console.log("[IMAP Node Watcher] Webhook configuration:", JSON.stringify({
+    enabled: completeFeedConfig.webhook?.enabled,
+    url: completeFeedConfig.webhook?.url ? '[REDACTED]' : undefined,
+    format: completeFeedConfig.webhook?.format,
+    newItemsOnly: completeFeedConfig.webhook?.newItemsOnly
+}, null, 2));
 
 const watcher = new ImapWatcher(completeFeedConfig); 
 

package.json

@@ -2,7 +2,7 @@
   "name": "mkfd",
   "module": "index.ts",
   "type": "module",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "scripts": {
     "start": "bun run index.ts",
     "dev": "bun run index.ts --watch --passkey=admin123 --cookieSecret=a18c1fd2211edd76a18c1fd2211edd76 --encryptionKey=a18c1fd2211edd76"