Skip to content

Commit 932ab6d

Browse files
committed
added flaresolverr integration
1 parent 8438f90 commit 932ab6d

18 files changed

+995
-114
lines changed

.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,4 +172,4 @@ configs/
172172
feed-history/
173173
truenas/
174174
extensions/
175-
.claude/settings.local.json
175+
.claude

README.md

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -200,20 +200,24 @@ Some websites structure content across multiple layers:
200200

201201
- [x] Add ALL possible RSS fields to models
202202
- [x] Add option for parallel iterators
203-
- [ ] Scraping how-to video
204-
- [ ] Email feed how-to video
203+
- [ ] Scraping how-to video/gif
204+
- [ ] Email feed how-to video/gif
205205
- [x] Add feed preview pane
206-
- [ ] Store/compare feed data to enable timestamping feed items
206+
- [ ] Store/compare all feed data to enable timestamping feed items
207207
- [x] Create dockerfile
208-
- [ ] Create Helm chart files
209208
- [x] Create GUI
210209
- [x] Utilities
211210
- [x] HTML stripper
212211
- [x] Source URL wrapper for relative links
213212
- [x] Nested link follower/drilldown functionality for each feed item property
214213
- [x] Adjust date parser logic with overrides from an optional date format input
215214
- [x] Add selector suggestion engine
216-
- [ ] Front-end redesign (react + shadcn)
215+
- [x] Front-end redesign (react + shadcn)
216+
- [ ] Redesign active feeds page
217+
- [ ] New dark mode or theme system
218+
- [x] Flaresolverr integration
219+
- [ ] Change detection feeds
220+
- [ ] With diff descriptions
217221
- [ ] Amass contributors
218222

219223
<br>

bun.lock

Lines changed: 26 additions & 27 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

frontend/src/components/forms/AdditionalOptions.tsx

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ export const AdditionalOptions = ({
4343
return (
4444
<Accordion type="single" collapsible className="w-full">
4545
<AccordionItem value="additional">
46-
<AccordionTrigger>
46+
<AccordionTrigger data-accordion-trigger="additional">
4747
<span className="flex items-center gap-2">
4848
<Settings2 className="h-4 w-4" />
4949
Additional Options
@@ -98,11 +98,17 @@ export const AdditionalOptions = ({
9898
<Checkbox
9999
id="advanced"
100100
checked={watch("advanced")}
101-
onCheckedChange={(checked) =>
102-
setValue("advanced", checked as boolean)
103-
}
101+
disabled={watch("flaresolverr.enabled")}
102+
onCheckedChange={(checked) => {
103+
setValue("advanced", checked as boolean);
104+
if (checked) {
105+
setValue("flaresolverr.enabled", false);
106+
}
107+
}}
104108
/>
105-
<Label htmlFor="advanced">Use Advanced Scraping</Label>
109+
<Label htmlFor="advanced" className={watch("flaresolverr.enabled") ? "text-muted-foreground" : ""}>
110+
Use Advanced Scraping
111+
</Label>
106112
<Tooltip>
107113
<TooltipTrigger asChild>
108114
<Info className="h-4 w-4 text-muted-foreground" />
@@ -115,6 +121,66 @@ export const AdditionalOptions = ({
115121
</div>
116122
)}
117123

124+
{/* FlareSolverr - Hide for email feeds */}
125+
{!isEmailFeed && (
126+
<div id="flaresolverr-section" className="space-y-3 transition-all">
127+
<div className="flex items-center space-x-2">
128+
<Checkbox
129+
id="flaresolverr.enabled"
130+
checked={watch("flaresolverr.enabled")}
131+
disabled={watch("advanced")}
132+
onCheckedChange={(checked) => {
133+
setValue("flaresolverr.enabled", checked as boolean);
134+
if (checked) {
135+
setValue("advanced", false);
136+
}
137+
}}
138+
/>
139+
<Label htmlFor="flaresolverr.enabled" className={watch("advanced") ? "text-muted-foreground" : ""}>
140+
Use FlareSolverr
141+
</Label>
142+
<Tooltip>
143+
<TooltipTrigger asChild>
144+
<Info className="h-4 w-4 text-muted-foreground" />
145+
</TooltipTrigger>
146+
<TooltipContent>
147+
Use FlareSolverr to bypass Cloudflare and other bot
148+
protection systems.
149+
</TooltipContent>
150+
</Tooltip>
151+
</div>
152+
153+
{watch("flaresolverr.enabled") && (
154+
<div className="ml-6 space-y-3">
155+
<div className="space-y-2">
156+
<Label htmlFor="flaresolverr.serverUrl">
157+
FlareSolverr Server URL
158+
</Label>
159+
<Input
160+
id="flaresolverr.serverUrl"
161+
{...register("flaresolverr.serverUrl")}
162+
placeholder="http://localhost:8191"
163+
type="url"
164+
/>
165+
</div>
166+
<div className="space-y-2">
167+
<Label htmlFor="flaresolverr.timeout">
168+
Timeout (milliseconds)
169+
</Label>
170+
<Input
171+
id="flaresolverr.timeout"
172+
{...register("flaresolverr.timeout")}
173+
type="number"
174+
min="1000"
175+
defaultValue="60000"
176+
placeholder="60000"
177+
/>
178+
</div>
179+
</div>
180+
)}
181+
</div>
182+
)}
183+
118184
{/* Strict Mode */}
119185
<div className="flex items-center space-x-2">
120186
<Checkbox

frontend/src/components/forms/FeedBuilderForm.tsx

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { APIForm } from "./APIForm";
1010
import { EmailForm } from "./EmailForm";
1111
import { AdditionalOptions } from "./AdditionalOptions";
1212
import { FeedPreview } from "./FeedPreview";
13+
import { FlareSolverrIndicator } from "./FlareSolverrIndicator";
1314
import { LoadingSpinner } from "@/components/ui/loading-spinner";
1415
import { Eye, Rocket, Globe, Code, Mail, Tag, Settings } from "lucide-react";
1516

@@ -261,6 +262,9 @@ export const FeedBuilderForm = () => {
261262
previewXml={previewXml}
262263
/>
263264
</form>
265+
266+
{/* FlareSolverr Status Indicator */}
267+
<FlareSolverrIndicator watch={watch} feedType={feedType} />
264268
</>
265269
);
266270
};
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
import { createPortal } from "react-dom";
2+
import { useEffect, useState } from "react";
3+
import { UseFormWatch } from "react-hook-form";
4+
import { FeedFormData } from "@/types/feed";
5+
6+
interface FlareSolverrIndicatorProps {
7+
watch: UseFormWatch<FeedFormData>;
8+
feedType?: "webScraping" | "api" | "email";
9+
}
10+
11+
export const FlareSolverrIndicator = ({
12+
watch,
13+
feedType,
14+
}: FlareSolverrIndicatorProps) => {
15+
const isEnabled = watch("flaresolverr.enabled");
16+
const serverUrl = watch("flaresolverr.serverUrl");
17+
const isWebScraping = feedType === "webScraping";
18+
const [connectionStatus, setConnectionStatus] = useState<"connected" | "error" | "checking">("checking");
19+
20+
// Verify FlareSolverr server is active
21+
useEffect(() => {
22+
if (!isEnabled || !serverUrl || !isWebScraping) {
23+
setConnectionStatus("checking");
24+
return;
25+
}
26+
27+
setConnectionStatus("checking");
28+
29+
// Debounce the server check
30+
const timeoutId = setTimeout(async () => {
31+
try {
32+
// Check via our backend proxy to avoid CORS issues
33+
const response = await fetch("/api/flaresolverr/health", {
34+
method: "POST",
35+
headers: { "Content-Type": "application/json" },
36+
body: JSON.stringify({ serverUrl }),
37+
signal: AbortSignal.timeout(5000),
38+
});
39+
40+
const data = await response.json();
41+
setConnectionStatus(data.active === true ? "connected" : "error");
42+
} catch (error) {
43+
setConnectionStatus("error");
44+
}
45+
}, 500); // 500ms debounce
46+
47+
return () => clearTimeout(timeoutId);
48+
}, [isEnabled, serverUrl, isWebScraping]);
49+
50+
// Only show for web scraping feeds when FlareSolverr is enabled
51+
if (!isWebScraping || !isEnabled) {
52+
return null;
53+
}
54+
55+
const handleClick = () => {
56+
// Find the FlareSolverr section
57+
const flaresolverrSection = document.getElementById("flaresolverr-section");
58+
59+
if (flaresolverrSection) {
60+
// First, ensure the Additional Options accordion is open
61+
const accordionTrigger = document.querySelector(
62+
'[data-accordion-trigger="additional"]'
63+
) as HTMLButtonElement;
64+
65+
if (accordionTrigger) {
66+
const accordionContent = accordionTrigger.getAttribute("data-state");
67+
68+
// If accordion is closed, open it
69+
if (accordionContent === "closed") {
70+
accordionTrigger.click();
71+
}
72+
}
73+
74+
// Wait a brief moment for accordion animation, then scroll
75+
setTimeout(() => {
76+
flaresolverrSection.scrollIntoView({
77+
behavior: "smooth",
78+
block: "center",
79+
});
80+
81+
// Add a brief highlight effect
82+
flaresolverrSection.classList.add("ring-2", "ring-blue-500", "ring-offset-2", "rounded-lg");
83+
setTimeout(() => {
84+
flaresolverrSection.classList.remove("ring-2", "ring-blue-500", "ring-offset-2", "rounded-lg");
85+
}, 2000);
86+
}, 300);
87+
}
88+
};
89+
90+
const badge = (
91+
<button
92+
type="button"
93+
onClick={handleClick}
94+
className="fixed bottom-6 right-6 z-[9999] flex items-center gap-2 px-3 py-2 bg-slate-700 dark:bg-slate-800 text-white rounded-full shadow-lg hover:shadow-xl hover:bg-slate-600 dark:hover:bg-slate-700 transition-all hover:scale-105 cursor-pointer"
95+
title={`FlareSolverr: ${connectionStatus === "connected" ? "Connected" : connectionStatus === "error" ? "Connection Error" : "Checking..."}`}
96+
>
97+
<img
98+
src="/public/flaresolverr.svg"
99+
alt="FlareSolverr"
100+
className="h-5 w-5"
101+
/>
102+
<span className="text-sm font-medium">FlareSolverr</span>
103+
{/* Status indicator circle */}
104+
<div className="relative flex items-center justify-center">
105+
<div
106+
className={`h-2 w-2 rounded-full ${
107+
connectionStatus === "connected"
108+
? "bg-green-500"
109+
: connectionStatus === "error"
110+
? "bg-red-500"
111+
: "bg-yellow-500"
112+
}`}
113+
/>
114+
{connectionStatus === "connected" && (
115+
<div className="absolute h-2 w-2 rounded-full bg-green-500 animate-ping opacity-75" />
116+
)}
117+
</div>
118+
</button>
119+
);
120+
121+
// Use portal to render at document body level to ensure fixed positioning works
122+
return typeof document !== "undefined" ? createPortal(badge, document.body) : null;
123+
};

frontend/src/components/forms/SelectorPlayground.tsx

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,32 @@
11
import { useState, useEffect } from 'react'
22
import { UseFormSetValue } from 'react-hook-form'
3-
import { FeedFormData } from '@/types/feed'
3+
import { FeedFormData, FlareSolverrConfig } from '@/types/feed'
44
import { Button } from '@/components/ui/button'
55
import { Wand2 } from 'lucide-react'
66

77
interface SelectorPlaygroundProps {
88
feedUrl?: string
99
setValue: UseFormSetValue<FeedFormData>
10+
flaresolverr?: FlareSolverrConfig
1011
}
1112

12-
export const SelectorPlayground = ({ feedUrl, setValue }: SelectorPlaygroundProps) => {
13+
export const SelectorPlayground = ({ feedUrl, setValue, flaresolverr }: SelectorPlaygroundProps) => {
1314
const [isPlaygroundOpen, setIsPlaygroundOpen] = useState(false)
1415
const [showSelectorActions, setShowSelectorActions] = useState(false)
1516
const [currentSelector, setCurrentSelector] = useState<string | null>(null)
1617

18+
const buildProxyUrl = () => {
19+
const params = new URLSearchParams({ url: feedUrl || '' })
20+
if (flaresolverr?.enabled && flaresolverr?.serverUrl) {
21+
params.set('flaresolverrEnabled', 'true')
22+
params.set('flaresolverrUrl', flaresolverr.serverUrl)
23+
if (flaresolverr.timeout) {
24+
params.set('flaresolverrTimeout', flaresolverr.timeout.toString())
25+
}
26+
}
27+
return `/proxy?${params.toString()}`
28+
}
29+
1730
useEffect(() => {
1831
// Listen for selector updates from the iframe
1932
const handleMessage = (event: MessageEvent) => {
@@ -127,7 +140,7 @@ export const SelectorPlayground = ({ feedUrl, setValue }: SelectorPlaygroundProp
127140
onClick={(e) => e.stopPropagation()}
128141
>
129142
<iframe
130-
src={`/proxy?url=${encodeURIComponent(feedUrl || '')}`}
143+
src={buildProxyUrl()}
131144
className="w-full h-full border-0"
132145
sandbox="allow-same-origin allow-scripts allow-popups allow-forms allow-modals"
133146
title="Selector Playground"

frontend/src/components/forms/WebScrapingForm.tsx

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,10 @@ export const WebScrapingForm = ({
5858
const response = await fetch("/utils/suggest-selectors", {
5959
method: "POST",
6060
headers: { "Content-Type": "application/json" },
61-
body: JSON.stringify({ url: feedUrl }),
61+
body: JSON.stringify({
62+
url: feedUrl,
63+
flaresolverr: watch("flaresolverr")
64+
}),
6265
});
6366

6467
if (!response.ok) throw new Error("Failed to fetch selectors.");
@@ -91,7 +94,11 @@ export const WebScrapingForm = ({
9194

9295
return (
9396
<>
94-
<SelectorPlayground feedUrl={feedUrl} setValue={setValue} />
97+
<SelectorPlayground
98+
feedUrl={feedUrl}
99+
setValue={setValue}
100+
flaresolverr={watch("flaresolverr")}
101+
/>
95102
<div className="space-y-6 mt-4">
96103
{/* Target URL */}
97104
<div className="space-y-2">

frontend/src/types/feed.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ export interface WebhookConfig {
3838
customPayload?: string;
3939
}
4040

41+
// FlareSolverr configuration
42+
export interface FlareSolverrConfig {
43+
enabled?: boolean;
44+
serverUrl?: string;
45+
timeout?: number;
46+
}
47+
4148
// Web Scraping Feed Configuration
4249
export interface WebScrapingConfig {
4350
feedUrl: string;
@@ -229,6 +236,7 @@ export interface FeedConfig {
229236
advanced?: boolean;
230237
strict?: boolean;
231238
webhook?: WebhookConfig;
239+
flaresolverr?: FlareSolverrConfig;
232240

233241
// Type-specific configurations
234242
webScraping?: WebScrapingConfig;

0 commit comments

Comments
 (0)