From ec337334198f9bd3910e808509b5fe37d4abd65c Mon Sep 17 00:00:00 2001 From: Abu Bakkar Siddique Date: Sun, 13 Apr 2025 15:20:17 +0530 Subject: [PATCH 1/8] Resolve the functionality to duplicate a page that has been already duplicated --- .../src/routes/editor/LayersPanel/Tree/PageTreeNode.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/studio/src/routes/editor/LayersPanel/Tree/PageTreeNode.tsx b/apps/studio/src/routes/editor/LayersPanel/Tree/PageTreeNode.tsx index 5cfa04e61..27968ad0e 100644 --- a/apps/studio/src/routes/editor/LayersPanel/Tree/PageTreeNode.tsx +++ b/apps/studio/src/routes/editor/LayersPanel/Tree/PageTreeNode.tsx @@ -75,7 +75,13 @@ const PageTreeNode: React.FC = ({ node, style }) => { const handleDuplicate = async () => { try { - await editorEngine.pages.duplicatePage(node.data.path, node.data.path); + const basePath = node.data.path; + const newPath = basePath.replace(/(\/[^/]+)$/, (match) => { + const baseName = getBaseName(match); + const newName = `${baseName}1`; + return `/${newName}`; + }); + await editorEngine.pages.duplicatePage(basePath, newPath); toast({ title: 'Page duplicated', From e0edcb6bce0209b603ebe18fdba4fa86767d6683 Mon Sep 17 00:00:00 2001 From: Abu Bakkar Siddique Date: Sun, 13 Apr 2025 23:42:39 +0530 Subject: [PATCH 2/8] Added the warning in the input chat that links wouldn't work --- apps/studio/src/locales/en/translation.json | 2 +- apps/web/client/messages/en.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/studio/src/locales/en/translation.json b/apps/studio/src/locales/en/translation.json index 0f153e79d..9c915e853 100644 --- a/apps/studio/src/locales/en/translation.json +++ b/apps/studio/src/locales/en/translation.json @@ -68,7 +68,7 @@ "title": "What kind of website do you want to make?", "description": "Tell us a bit about your project. Be as detailed as possible.", "input": { - "placeholder": "Paste a reference screenshot, write a novel, get creative...", + "placeholder": "Paste a reference screenshot, write a novel, get creative...\nlink (www.*****.com or anything with an https:://) won't work.", "imageUpload": "Upload Image Reference", "fileReference": "File Reference", "submit": "Start building your site" diff --git a/apps/web/client/messages/en.json b/apps/web/client/messages/en.json index bf0fe1fdb..60df6f499 100644 --- a/apps/web/client/messages/en.json +++ b/apps/web/client/messages/en.json @@ -68,7 +68,7 @@ "title": "What kind of website do you want to make?", "description": "Tell us a bit about your project. Be as detailed as possible.", "input": { - "placeholder": "Paste a reference screenshot, write a novel, get creative...", + "placeholder": "Paste a reference screenshot, write a novel, get creative...\nlink (www.*****.com or anything with an https:://) won't work.", "imageUpload": "Upload Image Reference", "fileReference": "File Reference", "submit": "Start building your site" From 8d405d570963dcb204c5a2607ef9d88f8d8fbd91 Mon Sep 17 00:00:00 2001 From: Abu Bakkar Siddique Date: Mon, 14 Apr 2025 20:29:42 +0530 Subject: [PATCH 3/8] Added firecrawl to crawl any website and provide the response data with the llm prompt --- apps/studio/.env.example | 4 +- apps/studio/package.json | 1 + apps/studio/src/lib/projects/create.ts | 9 +- apps/studio/src/lib/services/crawler.ts | 60 ++++++++++++ apps/studio/src/locales/en/translation.json | 4 + .../projects/PromptCreation/PromptingCard.tsx | 96 ++++++++++++++++++- apps/web/client/messages/en.json | 4 + 7 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 apps/studio/src/lib/services/crawler.ts diff --git a/apps/studio/.env.example b/apps/studio/.env.example index 11d4b66d5..8730642a9 100644 --- a/apps/studio/.env.example +++ b/apps/studio/.env.example @@ -4,4 +4,6 @@ VITE_SUPABASE_ANON_KEY= VITE_MIXPANEL_TOKEN= # Add your keys here to use Anthropic directly -VITE_ANTHROPIC_API_KEY= \ No newline at end of file +VITE_ANTHROPIC_API_KEY= +# Add your Firecrawl API key here to use Firecrawl directly +VITE_FIRECRAWL_API_KEY= \ No newline at end of file diff --git a/apps/studio/package.json b/apps/studio/package.json index 8442f61a5..27058afc0 100644 --- a/apps/studio/package.json +++ b/apps/studio/package.json @@ -40,6 +40,7 @@ "@emotion/react": "^11.13.3", "@emotion/styled": "^11.13.0", "@fontsource-variable/inter": "^5.1.0", + "@mendable/firecrawl-js": "^1.23.9", "@onlook/foundation": "*", "@onlook/supabase": "*", "@onlook/ui": "*", diff --git a/apps/studio/src/lib/projects/create.ts b/apps/studio/src/lib/projects/create.ts index bfa99208b..9dcf42acb 100644 --- a/apps/studio/src/lib/projects/create.ts +++ b/apps/studio/src/lib/projects/create.ts @@ -89,9 +89,15 @@ export class CreateManager { } } - async sendPrompt(prompt: string, images: ImageMessageContext[], blank: boolean = false) { + async sendPrompt( + prompt: string, + images: ImageMessageContext[], + crawledContent?: string, + blank: boolean = false, + ) { sendAnalytics('prompt create project', { prompt, + crawledContent, blank, }); @@ -105,6 +111,7 @@ export class CreateManager { result = await invokeMainChannel(MainChannels.CREATE_NEW_PROJECT_PROMPT, { prompt, images, + crawledContent, }); } diff --git a/apps/studio/src/lib/services/crawler.ts b/apps/studio/src/lib/services/crawler.ts new file mode 100644 index 000000000..d07bec49a --- /dev/null +++ b/apps/studio/src/lib/services/crawler.ts @@ -0,0 +1,60 @@ +import FirecrawlApp from '@mendable/firecrawl-js'; + +export interface CrawlOptions { + limit?: number; + scrapeOptions?: { + formats?: ( + | 'markdown' + | 'html' + | 'rawHtml' + | 'content' + | 'links' + | 'screenshot' + | 'screenshot@fullPage' + | 'extract' + | 'json' + | 'changeTracking' + )[]; + }; +} + +export class CrawlerService { + private static instance: CrawlerService; + private app: FirecrawlApp; + + private constructor() { + this.app = new FirecrawlApp({ + apiKey: process.env.VITE_FIRECRAWL_API_KEY, + }); + } + + static getInstance(): CrawlerService { + if (!CrawlerService.instance) { + CrawlerService.instance = new CrawlerService(); + } + return CrawlerService.instance; + } + + async crawlUrl( + url: string, + options: CrawlOptions = { + limit: 100, + scrapeOptions: { + formats: ['markdown', 'html'], + }, + }, + ) { + try { + const response = await this.app.crawlUrl(url, options); + + if (!response.success) { + throw new Error(`Failed to crawl: ${response.error}`); + } + + return response; + } catch (error) { + console.error('Crawl error:', error); + throw error; + } + } +} diff --git a/apps/studio/src/locales/en/translation.json b/apps/studio/src/locales/en/translation.json index 9c915e853..7b916c493 100644 --- a/apps/studio/src/locales/en/translation.json +++ b/apps/studio/src/locales/en/translation.json @@ -73,6 +73,10 @@ "fileReference": "File Reference", "submit": "Start building your site" }, + "crawl": { + "title": "Crawl a website", + "description": "Paste a link to a website you want to duplicate" + }, "blankStart": "Start from a blank page" } }, diff --git a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx index 402dc6533..eb213b31e 100644 --- a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx +++ b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx @@ -14,6 +14,8 @@ import { useEffect, useRef, useState } from 'react'; import useResizeObserver from 'use-resize-observer'; import { DraftImagePill } from '../../editor/EditPanel/ChatTab/ContextPills/DraftingImagePill'; import { useTranslation } from 'react-i18next'; +import { CrawlerService } from '@/lib/services/crawler'; +import { toast } from '@onlook/ui/use-toast'; export const PromptingCard = () => { const projectsManager = useProjectsManager(); @@ -28,6 +30,9 @@ export const PromptingCard = () => { const [isComposing, setIsComposing] = useState(false); const imageRef = useRef(null); const { t } = useTranslation(); + const [urlInput, setUrlInput] = useState(''); + const [isCrawling, setIsCrawling] = useState(false); + const [crawlValue, setCrawlValue] = useState(''); useEffect(() => { const handleEscapeKey = (e: KeyboardEvent) => { @@ -45,11 +50,11 @@ export const PromptingCard = () => { console.warn('Input is too short'); return; } - projectsManager.create.sendPrompt(inputValue, selectedImages, false); + projectsManager.create.sendPrompt(inputValue, selectedImages, crawlValue, false); }; const handleBlankSubmit = async () => { - projectsManager.create.sendPrompt('', [], true); + projectsManager.create.sendPrompt('', [], '', true); }; const handleDragOver = (e: React.DragEvent) => { @@ -179,6 +184,35 @@ export const PromptingCard = () => { } }; + const handleCrawlSubmit = async () => { + if (!urlInput.trim()) { + console.warn('URL input is empty'); + return; + } + setIsCrawling(true); + try { + const crawler = CrawlerService.getInstance(); + const response = await crawler.crawlUrl(urlInput); + const responseData = response.data; + setCrawlValue(JSON.stringify(responseData)); + + toast({ + title: 'URl crawled', + description: response.success, + }); + } catch (error) { + console.error('Failed to crawl URL:', error); + toast({ + variant: 'destructive', + title: 'Failed to Crawl URL', + description: error instanceof Error ? error.message : 'An unknown error occurred', + }); + } finally { + setIsCrawling(false); + setUrlInput(''); + } + }; + return (
@@ -382,6 +416,64 @@ export const PromptingCard = () => { + + + + + {t('projects.prompt.crawl.title')} + + + {t('projects.prompt.crawl.description')} + + + +
+
+ setUrlInput(e.target.value)} + placeholder="Enter URL to crawl..." + className={cn( + 'flex-1 h-9 px-3 rounded-md', + 'bg-background-secondary/80 backdrop-blur-sm', + 'border border-border', + 'text-sm text-foreground-primary', + 'placeholder:text-foreground-secondary', + 'focus:outline-none focus:ring-2 focus:ring-ring', + )} + /> + +
+
+ +