diff --git a/examples/go/child-workflows/main.go b/examples/go/child-workflows/main.go index 7fc17927f2..4c6f900827 100644 --- a/examples/go/child-workflows/main.go +++ b/examples/go/child-workflows/main.go @@ -75,6 +75,7 @@ func Child(client *hatchet.Client) *hatchet.StandaloneTask { ) } + func main() { client, err := hatchet.NewClient() if err != nil { @@ -126,6 +127,7 @@ func main() { return err } + _ = childResult n := 5 diff --git a/examples/go/migration-guides/mergent.go b/examples/go/migration-guides/mergent.go index 54bef30777..777d988c25 100644 --- a/examples/go/migration-guides/mergent.go +++ b/examples/go/migration-guides/mergent.go @@ -45,6 +45,7 @@ func ProcessImageMergent(req MergentRequest) (*MergentResponse, error) { }, nil } + // > After (Hatchet) type ImageProcessInput struct { ImageURL string `json:"image_url"` diff --git a/examples/go/on-event/main.go b/examples/go/on-event/main.go index 73ade3c216..e4605b1d52 100644 --- a/examples/go/on-event/main.go +++ b/examples/go/on-event/main.go @@ -40,6 +40,7 @@ func Lower(client *hatchet.Client) *hatchet.StandaloneTask { ) } + // > Accessing the filter payload func accessFilterPayload(ctx hatchet.Context, input EventInput) (*LowerTaskOutput, error) { fmt.Println(ctx.FilterPayload()) @@ -48,6 +49,7 @@ func accessFilterPayload(ctx hatchet.Context, input EventInput) (*LowerTaskOutpu }, nil } + // > Declare with filter func LowerWithFilter(client *hatchet.Client) *hatchet.StandaloneTask { return client.NewStandaloneTask( @@ -64,6 +66,7 @@ func LowerWithFilter(client *hatchet.Client) *hatchet.StandaloneTask { ) } + func Upper(client *hatchet.Client) *hatchet.StandaloneTask { return client.NewStandaloneTask( "upper", func(ctx hatchet.Context, input EventInput) (*UpperTaskOutput, error) { diff --git a/examples/go/sticky-workers/main.go b/examples/go/sticky-workers/main.go index f34306d704..def4bfe80a 100644 --- a/examples/go/sticky-workers/main.go +++ b/examples/go/sticky-workers/main.go @@ -48,6 +48,7 @@ func StickyDag(client *hatchet.Client) *hatchet.Workflow { return stickyDag } + type ChildInput struct { N int `json:"n"` } @@ -90,3 +91,4 @@ func Sticky(client *hatchet.Client) *hatchet.StandaloneTask { return sticky } + diff --git a/examples/go/streaming/consumer/main.go b/examples/go/streaming/consumer/main.go index 358a10c1d3..59d120e44e 100644 --- a/examples/go/streaming/consumer/main.go +++ b/examples/go/streaming/consumer/main.go @@ -34,3 +34,4 @@ func main() { fmt.Println("\nStreaming completed!") } + diff --git a/examples/go/streaming/server/main.go b/examples/go/streaming/server/main.go index c527918e4b..4438a5f19f 100644 --- a/examples/go/streaming/server/main.go +++ b/examples/go/streaming/server/main.go @@ -54,3 +54,4 @@ func main() { log.Println("Failed to start server:", err) } } + diff --git a/examples/go/streaming/shared/task.go b/examples/go/streaming/shared/task.go index adce03d0e4..8b6070a473 100644 --- a/examples/go/streaming/shared/task.go +++ b/examples/go/streaming/shared/task.go @@ -46,6 +46,7 @@ func StreamTask(ctx hatchet.Context, input StreamTaskInput) (*StreamTaskOutput, }, nil } + func StreamingWorkflow(client *hatchet.Client) *hatchet.StandaloneTask { return client.NewStandaloneTask("stream-example", StreamTask) } diff --git a/frontend/docs/components/AgentLoopDiagram.tsx b/frontend/docs/components/AgentLoopDiagram.tsx new file mode 100644 index 0000000000..4b2d03597e --- /dev/null +++ b/frontend/docs/components/AgentLoopDiagram.tsx @@ -0,0 +1,285 @@ +import React, { useState, useEffect } from "react"; + +const PHASES = ["thought", "action", "observation"] as const; +type Phase = (typeof PHASES)[number]; + +const PHASE_CONFIG: Record = { + thought: { label: "Thought", color: "#818cf8" }, + action: { label: "Action", color: "#38bdf8" }, + observation: { label: "Observation", color: "#fbbf24" }, +}; + +/** Small SVG icons rendered inline, no emojis */ +const PhaseIcon: React.FC<{ phase: Phase; active: boolean }> = ({ + phase, + active, +}) => { + const color = active ? PHASE_CONFIG[phase].color : "#6b7280"; + const size = 18; + + switch (phase) { + case "thought": + // Lightbulb icon + return ( + + + + + + ); + case "action": + // Zap/bolt icon + return ( + + + + ); + case "observation": + // Eye icon + return ( + + + + + ); + } +}; + +const AgentLoopDiagram: React.FC = () => { + const [phaseIdx, setPhaseIdx] = useState(0); + const [iteration, setIteration] = useState(1); + + useEffect(() => { + const interval = setInterval(() => { + setPhaseIdx((prev) => { + if (prev === PHASES.length - 1) { + setIteration((i) => (i >= 3 ? 1 : i + 1)); + return 0; + } + return prev + 1; + }); + }, 1400); + return () => clearInterval(interval); + }, []); + + const phase = PHASES[phaseIdx]; + + // Horizontal layout: 3 nodes evenly spaced + const svgW = 520; + const svgH = 160; + const nodeY = 70; + const nodeSpacing = 160; + const startX = 100; + + const nodes = PHASES.map((_, i) => ({ + x: startX + i * nodeSpacing, + y: nodeY, + })); + + return ( +
+ + + + + + + + + + + {/* Forward arrows between nodes */} + {nodes.slice(0, -1).map((from, i) => { + const to = nodes[i + 1]; + const isActive = phaseIdx === i; + return ( + + ); + })} + + {/* Return arrow: curved path from Observation back to Thought */} + {(() => { + const from = nodes[nodes.length - 1]; + const to = nodes[0]; + const isActive = phaseIdx === PHASES.length - 1; + const curveY = nodeY + 58; + return ( + + ); + })()} + + {/* Loop label on return arrow */} + + iteration {iteration}/3 + + + {/* Phase nodes */} + {PHASES.map((p, i) => { + const pos = nodes[i]; + const config = PHASE_CONFIG[p]; + const isActive = phase === p; + + return ( + + {/* Glow ring */} + {isActive && ( + + + + )} + {/* Node box */} + + {/* Icon */} + + + + {/* Label */} + + {config.label} + + + ); + })} + + + {/* Status indicators */} +
+ {PHASES.map((p) => ( +
+ + {PHASE_CONFIG[p].label} +
+ ))} +
+ + ); +}; + +export default AgentLoopDiagram; diff --git a/frontend/docs/components/AgentLoopDiagramWrapper.tsx b/frontend/docs/components/AgentLoopDiagramWrapper.tsx new file mode 100644 index 0000000000..b015e3f552 --- /dev/null +++ b/frontend/docs/components/AgentLoopDiagramWrapper.tsx @@ -0,0 +1,7 @@ +import dynamic from "next/dynamic"; + +const AgentLoopDiagram = dynamic(() => import("./AgentLoopDiagram"), { + ssr: false, +}); + +export default AgentLoopDiagram; diff --git a/frontend/docs/components/BatchProcessingDiagram.tsx b/frontend/docs/components/BatchProcessingDiagram.tsx new file mode 100644 index 0000000000..f4f0082406 --- /dev/null +++ b/frontend/docs/components/BatchProcessingDiagram.tsx @@ -0,0 +1,298 @@ +import React, { useState, useEffect } from "react"; + +const ITEMS = Array.from({ length: 8 }, (_, i) => i); + +/** SVG checkmark for completed items */ +const CheckIcon: React.FC<{ x: number; y: number; color: string }> = ({ + x, + y, + color, +}) => ( + + + + + +); + +const BatchProcessingDiagram: React.FC = () => { + const [processedCount, setProcessedCount] = useState(0); + + useEffect(() => { + const interval = setInterval(() => { + setProcessedCount((prev) => (prev >= ITEMS.length ? 0 : prev + 1)); + }, 600); + return () => clearInterval(interval); + }, []); + + const colors = { + pending: "#374151", + processing: "#fbbf24", + done: "#34d399", + }; + + return ( +
+ + {/* Trigger box */} + + {/* List icon */} + + + + + + + + + + + + Batch Input + + + {/* Arrow from trigger to items */} + + + + {/* Fan-out items grid (2 rows x 4 cols) */} + {ITEMS.map((item) => { + const col = item % 4; + const row = Math.floor(item / 4); + const x = 120 + col * 52; + const y = 35 + row * 55; + + let status: "pending" | "processing" | "done"; + if (item < processedCount) { + status = "done"; + } else if (item === processedCount) { + status = "processing"; + } else { + status = "pending"; + } + + const color = colors[status]; + + return ( + + {status === "processing" && ( + + + + )} + + {/* Status indicator */} + {status === "done" ? ( + + ) : status === "processing" ? ( + // Spinning indicator + + + + + + + + ) : ( + // File icon for pending + + + + + + + )} + + Item {item + 1} + + + ); + })} + + {/* Arrow from items to results */} + + + + {/* Results box */} + + {/* Bar chart icon */} + + + + + + + + + Results + + + + {/* Progress bar */} +
+
+
+
+ + {processedCount}/{ITEMS.length} + +
+
+ ); +}; + +export default BatchProcessingDiagram; diff --git a/frontend/docs/components/BatchProcessingDiagramWrapper.tsx b/frontend/docs/components/BatchProcessingDiagramWrapper.tsx new file mode 100644 index 0000000000..6500214f8f --- /dev/null +++ b/frontend/docs/components/BatchProcessingDiagramWrapper.tsx @@ -0,0 +1,8 @@ +import dynamic from "next/dynamic"; + +const BatchProcessingDiagram = dynamic( + () => import("./BatchProcessingDiagram"), + { ssr: false }, +); + +export default BatchProcessingDiagram; diff --git a/frontend/docs/components/BranchingDiagram.tsx b/frontend/docs/components/BranchingDiagram.tsx new file mode 100644 index 0000000000..10d7262b65 --- /dev/null +++ b/frontend/docs/components/BranchingDiagram.tsx @@ -0,0 +1,329 @@ +import React, { useState } from "react"; + +const BranchingDiagram: React.FC = () => { + const [isLeft, setIsLeft] = useState(true); + + const nodeWidth = 140; + const nodeHeight = 50; + const nodeRx = 10; + + // Active vs dimmed styles + const activeOpacity = 1; + const dimmedOpacity = 0.2; + + const leftActive = isLeft; + const rightActive = !isLeft; + + return ( +
+ {/* Toggle */} +
+ + value = 72 + + + + value = 23 + +
+ + {/* Diagram */} +
+ + + + + + + + + + + + + + + + + + + {/* Task A — always active */} + + + + Task A + + + + {/* Condition diamond — always active */} + + + + {isLeft ? "> 50 ✓" : "≤ 50 ✓"} + + + + {/* Left Branch */} + + + + Left Branch + + + {leftActive ? "runs ✓" : "skipped"} + + + + {/* Right Branch */} + + + + Right Branch + + + {rightActive ? "runs ✓" : "skipped"} + + + + {/* Task B — always active */} + + + + Task B + + + + {/* Edge: Task A -> diamond — always active */} + + + {/* Edge: diamond -> Left Branch */} + + + {/* Edge: diamond -> Right Branch */} + + + {/* Edge: Left Branch -> Task B */} + + + {/* Edge: Right Branch -> Task B */} + + +
+
+ ); +}; + +export default BranchingDiagram; diff --git a/frontend/docs/components/BranchingDiagramWrapper.tsx b/frontend/docs/components/BranchingDiagramWrapper.tsx new file mode 100644 index 0000000000..3be3775056 --- /dev/null +++ b/frontend/docs/components/BranchingDiagramWrapper.tsx @@ -0,0 +1,7 @@ +import dynamic from "next/dynamic"; + +const BranchingDiagram = dynamic(() => import("./BranchingDiagram"), { + ssr: false, +}); + +export default BranchingDiagram; diff --git a/frontend/docs/components/CycleDiagram.tsx b/frontend/docs/components/CycleDiagram.tsx new file mode 100644 index 0000000000..891f092747 --- /dev/null +++ b/frontend/docs/components/CycleDiagram.tsx @@ -0,0 +1,284 @@ +import React, { useState, useEffect } from "react"; + +const CycleDiagram: React.FC = () => { + const [iteration, setIteration] = useState(0); + const maxIterations = 3; + + useEffect(() => { + const timer = setInterval(() => { + setIteration((prev) => (prev + 1) % (maxIterations + 1)); + }, 2000); + return () => clearInterval(timer); + }, []); + + const nodeWidth = 140; + const nodeHeight = 50; + const nodeRx = 10; + + // Positions + const taskX = 60; + const taskY = 100; + const checkX = 300; + const checkY = 125; + const doneX = 520; + const doneY = 100; + + const isDone = iteration === maxIterations; + + return ( +
+ {/* Iteration counter */} +
+ Iteration: + {[0, 1, 2].map((i) => ( + + {i + 1} + + ))} + + {isDone ? "done!" : "running..."} + +
+ + {/* Diagram */} +
+ + + + + + + + + + + + + + + + + + + {/* Task box */} + + + Task + + + do work + + + {/* Condition diamond */} + + + + {isDone ? "done ✓" : "done?"} + + + {isDone ? "" : "not yet"} + + + + {/* Done box */} + + + Complete + + + return result + + + {/* Edge: Task -> Check */} + + + {/* Edge: Check -> Done (right) */} + + {/* "yes" label on done edge */} + + yes + + + {/* Loop-back edge: Check -> Task (curved below) */} + + {/* "no, loop" label */} + + no → re-run + + +
+
+ ); +}; + +export default CycleDiagram; diff --git a/frontend/docs/components/CycleDiagramWrapper.tsx b/frontend/docs/components/CycleDiagramWrapper.tsx new file mode 100644 index 0000000000..f0cf3c91c8 --- /dev/null +++ b/frontend/docs/components/CycleDiagramWrapper.tsx @@ -0,0 +1,7 @@ +import dynamic from "next/dynamic"; + +const CycleDiagram = dynamic(() => import("./CycleDiagram"), { + ssr: false, +}); + +export default CycleDiagram; diff --git a/frontend/docs/components/DurableWorkflowDiagram.tsx b/frontend/docs/components/DurableWorkflowDiagram.tsx new file mode 100644 index 0000000000..d09c6772f7 --- /dev/null +++ b/frontend/docs/components/DurableWorkflowDiagram.tsx @@ -0,0 +1,369 @@ +import React, { useState, useEffect } from "react"; + +const DurableWorkflowDiagram: React.FC = () => { + // Phases: 0=running, 1=checkpoint, 2=interrupted, 3=resumed, 4=complete + const [phase, setPhase] = useState(0); + + useEffect(() => { + const durations = [1500, 1200, 1800, 1200, 1500]; + const timer = setTimeout(() => { + setPhase((prev) => (prev + 1) % 5); + }, durations[phase]); + return () => clearTimeout(timer); + }, [phase]); + + const nodeW = 120; + const nodeH = 44; + const rx = 8; + + // Timeline positions + const steps = [ + { x: 30, label: "Do Work", sub: "step 1" }, + { x: 175, label: "Checkpoint", sub: "save state" }, + { x: 320, label: "Interrupted", sub: "worker crash" }, + { x: 465, label: "Restore", sub: "new worker" }, + { x: 610, label: "Complete", sub: "step 2" }, + ]; + const y = 90; + + const phaseColors = [ + { + fill: "rgba(49,46,129,0.3)", + stroke: "rgb(99,102,241)", + text: "#c7d2fe", + sub: "#818cf8", + }, + { + fill: "rgba(120,53,15,0.25)", + stroke: "rgb(245,158,11)", + text: "#fcd34d", + sub: "#d97706", + }, + { + fill: "rgba(127,29,29,0.25)", + stroke: "rgb(239,68,68)", + text: "#fca5a5", + sub: "#ef4444", + }, + { + fill: "rgba(6,78,59,0.3)", + stroke: "rgb(16,185,129)", + text: "#a7f3d0", + sub: "#6ee7b7", + }, + { + fill: "rgba(6,78,59,0.3)", + stroke: "rgb(16,185,129)", + text: "#a7f3d0", + sub: "#6ee7b7", + }, + ]; + + const statusLabels = [ + "running...", + "checkpointing...", + "interrupted!", + "restoring...", + "complete!", + ]; + const statusColors = ["#818cf8", "#fcd34d", "#fca5a5", "#6ee7b7", "#6ee7b7"]; + + return ( +
+ {/* Status bar */} +
+ Durable task: + {steps.map((s, i) => ( + + ))} + + {statusLabels[phase]} + +
+ + {/* Diagram */} +
+ + + + + + + + + + + + + + + + + + + + + + + {/* Timeline base line */} + + + {/* Progress line */} + = 3 + ? "rgb(16,185,129)" + : "rgb(99,102,241)" + } + strokeWidth="2" + style={{ transition: "all 0.6s ease" }} + /> + + {/* Edges between nodes */} + {steps.slice(0, -1).map((s, i) => { + const nextX = steps[i + 1].x; + const isCurrent = i === phase; + + let edgeColor = "#333"; + if (i < phase) { + edgeColor = + i === 1 + ? "rgb(245,158,11)" + : i === 2 + ? "rgb(16,185,129)" + : "rgb(99,102,241)"; + } + if (isCurrent) { + edgeColor = phaseColors[phase].stroke; + } + + return ( + + ); + })} + + {/* Nodes */} + {steps.map((s, i) => { + const isActive = i === phase; + const isPast = i < phase; + + let fill = "rgba(30,30,30,0.15)"; + let stroke = "#444"; + let textColor = "#666"; + let subColor = "#555"; + + if (isActive) { + fill = phaseColors[i].fill; + stroke = phaseColors[i].stroke; + textColor = phaseColors[i].text; + subColor = phaseColors[i].sub; + } else if (isPast) { + fill = phaseColors[i].fill; + stroke = phaseColors[i].stroke; + textColor = phaseColors[i].text; + subColor = phaseColors[i].sub; + } + + return ( + + phase ? 0.3 : 1, + }} + /> + + {s.label} + + + {s.sub} + + + {/* Crash indicator (SVG bolt) */} + {i === 2 && isActive && ( + + + + + + )} + + {/* Checkpoint indicator (SVG save/disk) */} + {i === 1 && (isActive || isPast) && ( + + + + + + + + )} + + {/* Timeline dot */} + + + ); + })} + + {/* Arrow showing "skip replay" from checkpoint to restore */} + {phase >= 3 && ( + = 3 ? 1 : 0, + transition: "opacity 0.5s ease", + }} + > + + + replay from checkpoint + + + )} + +
+
+ ); +}; + +export default DurableWorkflowDiagram; diff --git a/frontend/docs/components/DurableWorkflowDiagramWrapper.tsx b/frontend/docs/components/DurableWorkflowDiagramWrapper.tsx new file mode 100644 index 0000000000..82f43e5398 --- /dev/null +++ b/frontend/docs/components/DurableWorkflowDiagramWrapper.tsx @@ -0,0 +1,10 @@ +import dynamic from "next/dynamic"; + +const DurableWorkflowDiagram = dynamic( + () => import("./DurableWorkflowDiagram"), + { + ssr: false, + }, +); + +export default DurableWorkflowDiagram; diff --git a/frontend/docs/components/EventDrivenDiagram.tsx b/frontend/docs/components/EventDrivenDiagram.tsx new file mode 100644 index 0000000000..86f4a003de --- /dev/null +++ b/frontend/docs/components/EventDrivenDiagram.tsx @@ -0,0 +1,274 @@ +import React, { useState, useEffect } from "react"; + +type SourceType = "Webhook" | "Cron" | "Event"; + +const SOURCES: { label: SourceType; color: string }[] = [ + { label: "Webhook", color: "#818cf8" }, + { label: "Cron", color: "#38bdf8" }, + { label: "Event", color: "#fbbf24" }, +]; + +const SourceIcon: React.FC<{ + type: SourceType; + color: string; + size?: number; +}> = ({ type, color, size = 14 }) => { + const props = { + width: size, + height: size, + viewBox: "0 0 24 24", + fill: "none", + stroke: color, + strokeWidth: "2", + strokeLinecap: "round" as const, + strokeLinejoin: "round" as const, + }; + + switch (type) { + case "Webhook": + // Link icon + return ( + + + + + ); + case "Cron": + // Clock icon + return ( + + + + + ); + case "Event": + // Signal/broadcast icon + return ( + + + + + + + + + ); + } +}; + +const GearIcon: React.FC<{ color: string; size?: number }> = ({ + color, + size = 14, +}) => ( + + + + +); + +const EventDrivenDiagram: React.FC = () => { + const [activeSource, setActiveSource] = useState(0); + const [pulseVisible, setPulseVisible] = useState(false); + + useEffect(() => { + const interval = setInterval(() => { + setPulseVisible(true); + setTimeout(() => setPulseVisible(false), 800); + setTimeout(() => { + setActiveSource((prev) => (prev + 1) % SOURCES.length); + }, 1000); + }, 2000); + return () => clearInterval(interval); + }, []); + + return ( +
+ + {/* Event sources */} + {SOURCES.map((src, i) => { + const y = 30 + i * 55; + const isActive = i === activeSource; + + return ( + + {/* Source box */} + + {/* Icon via foreignObject */} + + + + + {src.label} + + + {/* Arrow to Hatchet */} + + + {/* Pulse dot */} + {isActive && pulseVisible && ( + + + + + )} + + ); + })} + + {/* Hatchet engine center */} + + + Hatchet + + + Engine + + + {/* Workers */} + {[0, 1, 2].map((i) => { + const y = 30 + i * 55; + const isActive = pulseVisible && i === activeSource; + + return ( + + {/* Arrow from Hatchet to worker */} + + + {/* Worker box */} + + {/* Gear icon */} + + + + + Worker {i + 1} + + + ); + })} + + + {/* Legend */} +
+ {SOURCES.map((src, i) => ( +
+ + {src.label} +
+ ))} +
+ + ); +}; + +export default EventDrivenDiagram; diff --git a/frontend/docs/components/EventDrivenDiagramWrapper.tsx b/frontend/docs/components/EventDrivenDiagramWrapper.tsx new file mode 100644 index 0000000000..9f83c7bf1e --- /dev/null +++ b/frontend/docs/components/EventDrivenDiagramWrapper.tsx @@ -0,0 +1,7 @@ +import dynamic from "next/dynamic"; + +const EventDrivenDiagram = dynamic(() => import("./EventDrivenDiagram"), { + ssr: false, +}); + +export default EventDrivenDiagram; diff --git a/frontend/docs/components/FanoutDiagram.tsx b/frontend/docs/components/FanoutDiagram.tsx new file mode 100644 index 0000000000..b5d95a80ae --- /dev/null +++ b/frontend/docs/components/FanoutDiagram.tsx @@ -0,0 +1,223 @@ +import React from "react"; + +const FanoutDiagram: React.FC = () => { + const childYPositions = [40, 110, 180, 270]; + const childLabels = ["Child 1", "Child 2", "Child 3", "Child N"]; + + return ( +
+
+ + + + + + + + + + + + + + + + + + + {/* Parent Task Box */} + + + Parent Task + + + spawn(input) + + + {/* Fan-out lines from parent to children */} + {childYPositions.map((cy, i) => ( + + ))} + + {/* Child boxes */} + {childLabels.slice(0, 3).map((label, i) => ( + + + + {label} + + + ))} + + {/* Ellipsis between Child 3 and Child N */} + + ... + + + {/* Child N */} + + + Child N + + + {/* Converge lines from children to results */} + {childYPositions.map((cy, i) => ( + + ))} + + {/* Results Box */} + + + Collect Results + + + await all children + + +
+
+ ); +}; + +export default FanoutDiagram; diff --git a/frontend/docs/components/LongWaitDiagram.tsx b/frontend/docs/components/LongWaitDiagram.tsx new file mode 100644 index 0000000000..abcd2ec175 --- /dev/null +++ b/frontend/docs/components/LongWaitDiagram.tsx @@ -0,0 +1,379 @@ +import React, { useState, useEffect } from "react"; + +type WaitType = "sleep" | "event"; + +const LongWaitDiagram: React.FC = () => { + const [waitType, setWaitType] = useState("sleep"); + const [phase, setPhase] = useState(0); // 0=running, 1=waiting, 2=resumed, 3=complete + + useEffect(() => { + const durations = [1200, 2400, 1200, 1200]; + const timer = setTimeout(() => { + setPhase((prev) => (prev + 1) % 4); + }, durations[phase]); + return () => clearTimeout(timer); + }, [phase]); + + const nodeW = 130; + const nodeH = 50; + const rx = 10; + + // Positions + const taskX = 30; + const taskY = 80; + const waitX = 220; + const waitY = 80; + const resumeX = 440; + const resumeY = 80; + const completeX = 600; + const completeY = 80; + + const isWaiting = phase === 1; + const isResumed = phase === 2; + const isDone = phase === 3; + + const waitLabel = waitType === "sleep" ? "Sleep 24h" : "Wait for Event"; + const waitSublabel = + waitType === "sleep" ? "durable pause" : "external signal"; + const triggerLabel = waitType === "sleep" ? "time elapsed" : "event received"; + + return ( +
+ {/* Toggle */} +
+ Wait type: + {(["sleep", "event"] as WaitType[]).map((type) => ( + + ))} + + {isDone + ? "complete!" + : isResumed + ? "resuming..." + : isWaiting + ? "waiting..." + : "running..."} + +
+ + {/* Diagram */} +
+ + + + + + + + + + + + + + + + + + + {/* Task box */} + + + Task Runs + + + do work + + + {/* Edge: Task -> Wait */} + + + {/* Wait box - larger, distinctive */} + + {/* Pause icon */} + {isWaiting && ( + + + + + )} + + {waitLabel} + + {!isWaiting && ( + + {waitSublabel} + + )} + + {/* Trigger label below wait box */} + + {isResumed || isDone ? triggerLabel : ""} + + + {/* Edge: Wait -> Resume */} + + + {/* Resume box */} + + + Resume + + + continue work + + + {/* Edge: Resume -> Complete */} + + + {/* Complete box */} + + + Complete + + + return result + + +
+
+ ); +}; + +export default LongWaitDiagram; diff --git a/frontend/docs/components/LongWaitDiagramWrapper.tsx b/frontend/docs/components/LongWaitDiagramWrapper.tsx new file mode 100644 index 0000000000..476dd7f715 --- /dev/null +++ b/frontend/docs/components/LongWaitDiagramWrapper.tsx @@ -0,0 +1,7 @@ +import dynamic from "next/dynamic"; + +const LongWaitDiagram = dynamic(() => import("./LongWaitDiagram"), { + ssr: false, +}); + +export default LongWaitDiagram; diff --git a/frontend/docs/components/PatternComparison.tsx b/frontend/docs/components/PatternComparison.tsx new file mode 100644 index 0000000000..353f279426 --- /dev/null +++ b/frontend/docs/components/PatternComparison.tsx @@ -0,0 +1,192 @@ +import React from "react"; + +interface ComparisonRow { + label: string; + workflow: string; + durable: string; +} + +interface PatternComparisonProps { + rows: ComparisonRow[]; + recommendation?: "workflow" | "durable" | "both"; + recommendationText?: string; +} + +const PatternComparison: React.FC = ({ + rows, + recommendation = "workflow", + recommendationText, +}) => { + return ( +
+
+ {/* Workflows column */} +
+
+
+ + + + + + + +
+ + Workflows (DAGs) + + {recommendation === "workflow" && ( + + recommended + + )} +
+
+ {rows.map((row, i) => ( +
+
+ {row.label} +
+
{row.workflow}
+
+ ))} +
+
+ + {/* Durable column */} +
+
+
+ + + + + + +
+ + Durable Workflows + + {recommendation === "durable" && ( + + recommended + + )} +
+
+ {rows.map((row, i) => ( +
+
+ {row.label} +
+
{row.durable}
+
+ ))} +
+
+
+ + {/* Recommendation footer */} + {recommendationText && ( +
+ {recommendationText} +
+ )} +
+ ); +}; + +export default PatternComparison; diff --git a/frontend/docs/components/PipelineDiagram.tsx b/frontend/docs/components/PipelineDiagram.tsx new file mode 100644 index 0000000000..65457854af --- /dev/null +++ b/frontend/docs/components/PipelineDiagram.tsx @@ -0,0 +1,147 @@ +import React from "react"; + +const PipelineDiagram: React.FC = () => { + // Layout: + // Task A (standalone) + // Task B -> Task D -> Task E (pipeline) + // Task C (standalone) + const nodes = [ + { id: "a", label: "Task A", x: 30, y: 40 }, + { id: "b", label: "Task B", x: 30, y: 130 }, + { id: "c", label: "Task C", x: 30, y: 220 }, + { id: "d", label: "Task D", x: 300, y: 130 }, + { id: "e", label: "Task E", x: 560, y: 130 }, + ]; + + const nodeWidth = 140; + const nodeHeight = 50; + const nodeRx = 10; + + const edges = [ + { from: "b", to: "d" }, + { from: "d", to: "e" }, + ]; + + const getNode = (id: string) => nodes.find((n) => n.id === id)!; + + return ( +
+
+ + + + + + + + + + + + + + + {/* Nodes */} + {nodes.map((node) => ( + + + + {node.label} + + + ))} + + {/* Edges (rendered after nodes so lines appear on top) */} + {edges.map(({ from, to }, i) => { + const f = getNode(from); + const t = getNode(to); + const startX = f.x + nodeWidth + 2; + const startY = f.y + nodeHeight / 2; + const endX = t.x - 2; + const endY = t.y + nodeHeight / 2; + const midX = (startX + endX) / 2; + + return ( + + ); + })} + + {/* Parallel label and dashed box around A, B, C */} + + parallel + + + +
+
+ ); +}; + +export default PipelineDiagram; diff --git a/frontend/docs/components/RAGPipelineDiagram.tsx b/frontend/docs/components/RAGPipelineDiagram.tsx new file mode 100644 index 0000000000..cb5c87afc3 --- /dev/null +++ b/frontend/docs/components/RAGPipelineDiagram.tsx @@ -0,0 +1,309 @@ +import React, { useState, useEffect } from "react"; + +const STAGES = [ + { id: "ingest", label: "Ingest", color: "#818cf8" }, + { id: "chunk", label: "Chunk", color: "#38bdf8" }, + { id: "embed", label: "Embed", color: "#fbbf24" }, + { id: "index", label: "Index", color: "#34d399" }, + { id: "query", label: "Query", color: "#f472b6" }, +] as const; + +type StageId = (typeof STAGES)[number]["id"]; + +const StageIcon: React.FC<{ id: StageId; color: string; size?: number }> = ({ + id, + color, + size = 16, +}) => { + const props = { + width: size, + height: size, + viewBox: "0 0 24 24", + fill: "none", + stroke: color, + strokeWidth: "2", + strokeLinecap: "round" as const, + strokeLinejoin: "round" as const, + }; + + switch (id) { + case "ingest": + // Download/import arrow + return ( + + + + + + ); + case "chunk": + // Scissors + return ( + + + + + + + + ); + case "embed": + // Grid/vector + return ( + + + + + + + ); + case "index": + // Database + return ( + + + + + + ); + case "query": + // Search + return ( + + + + + ); + } +}; + +const RAGPipelineDiagram: React.FC = () => { + const [activeStage, setActiveStage] = useState(0); + + useEffect(() => { + const interval = setInterval(() => { + setActiveStage((prev) => (prev + 1) % STAGES.length); + }, 1500); + return () => clearInterval(interval); + }, []); + + const stageWidth = 72; + const gap = 16; + const totalWidth = STAGES.length * stageWidth + (STAGES.length - 1) * gap; + const startX = (440 - totalWidth) / 2; + + return ( +
+ + {/* Connecting arrows */} + {STAGES.slice(0, -1).map((_, i) => { + const fromX = startX + i * (stageWidth + gap) + stageWidth; + const toX = startX + (i + 1) * (stageWidth + gap); + const y = 70; + const isActive = i === activeStage || i + 1 === activeStage; + + return ( + + + + {/* Animated dot traveling along the arrow */} + {i === activeStage && activeStage < STAGES.length - 1 && ( + + + + )} + + ); + })} + + {/* Stage boxes */} + {STAGES.map((stage, i) => { + const x = startX + i * (stageWidth + gap); + const y = 40; + const isActive = i === activeStage; + + return ( + + {/* Glow */} + {isActive && ( + + + + )} + {/* Box */} + + {/* Icon */} + + + + {/* Label */} + + {stage.label} + + + ); + })} + + {/* Fan-out indicator under chunk stage */} + + + fan-out to N chunks + + + + + {/* Rate limit indicator under embed stage */} + + + rate-limited API + + + + + {/* Retry indicator under index stage */} + + + retries on failure + + + + + + {/* Progress indicator */} +
+ {STAGES.map((stage, i) => ( +
+ ))} +
+
+ ); +}; + +export default RAGPipelineDiagram; diff --git a/frontend/docs/components/RAGPipelineDiagramWrapper.tsx b/frontend/docs/components/RAGPipelineDiagramWrapper.tsx new file mode 100644 index 0000000000..545f5fd0fc --- /dev/null +++ b/frontend/docs/components/RAGPipelineDiagramWrapper.tsx @@ -0,0 +1,7 @@ +import dynamic from "next/dynamic"; + +const RAGPipelineDiagram = dynamic(() => import("./RAGPipelineDiagram"), { + ssr: false, +}); + +export default RAGPipelineDiagram; diff --git a/frontend/docs/components/WorkflowComparison.tsx b/frontend/docs/components/WorkflowComparison.tsx new file mode 100644 index 0000000000..cb42b87ac2 --- /dev/null +++ b/frontend/docs/components/WorkflowComparison.tsx @@ -0,0 +1,370 @@ +import React, { useState } from "react"; + +type Mode = "workflows" | "durable"; + +const WorkflowComparison: React.FC = () => { + const [active, setActive] = useState("workflows"); + + const data = { + workflows: { + label: "Workflows (DAGs)", + color: "indigo", + items: [ + { + icon: ( + + + + + + + + ), + title: "Structure", + desc: "DAG of tasks with declared dependencies", + }, + { + icon: ( + + + + + + ), + title: "State", + desc: "Cached between tasks automatically", + }, + { + icon: ( + + + + + ), + title: "Pausing", + desc: "Declarative conditions on task definitions", + }, + { + icon: ( + + + + ), + title: "Recovery", + desc: "Re-runs failed tasks; completed tasks are skipped", + }, + { + icon: ( + + + + + ), + title: "Slots", + desc: "Each task holds a slot while running", + }, + ], + }, + durable: { + label: "Durable Workflows", + color: "emerald", + items: [ + { + icon: ( + + + + + + + + ), + title: "Structure", + desc: "Long-running function with checkpoints", + }, + { + icon: ( + + + + + + ), + title: "State", + desc: "Stored in a durable event log", + }, + { + icon: ( + + + + + ), + title: "Pausing", + desc: "Inline SleepFor and WaitForEvent calls", + }, + { + icon: ( + + + + + + + ), + title: "Recovery", + desc: "Replays from last checkpoint automatically", + }, + { + icon: ( + + + + + + ), + title: "Slots", + desc: "Freed during waits — no wasted compute", + }, + ], + }, + }; + + const current = data[active]; + const isWorkflows = active === "workflows"; + + return ( +
+ {/* Toggle */} +
+ {(["workflows", "durable"] as Mode[]).map((mode) => ( + + ))} +
+ + {/* Cards */} +
+
+ {current.items.map((item, i) => ( +
+
+ {item.icon} +
+
+
+ {item.title} +
+
{item.desc}
+
+
+ ))} +
+ + {/* Best-for footer */} +
+ Best for: + {isWorkflows + ? "Predictable multi-step pipelines, ETL, CI/CD, and any workflow with a known shape" + : "Long waits, human-in-the-loop, large fan-outs, and complex procedural logic"} +
+
+
+ ); +}; + +export default WorkflowComparison; diff --git a/frontend/docs/components/WorkflowComparisonWrapper.tsx b/frontend/docs/components/WorkflowComparisonWrapper.tsx new file mode 100644 index 0000000000..657f46af31 --- /dev/null +++ b/frontend/docs/components/WorkflowComparisonWrapper.tsx @@ -0,0 +1,7 @@ +import dynamic from "next/dynamic"; + +const WorkflowComparison = dynamic(() => import("./WorkflowComparison"), { + ssr: false, +}); + +export default WorkflowComparison; diff --git a/frontend/docs/components/WorkflowDiagram.tsx b/frontend/docs/components/WorkflowDiagram.tsx new file mode 100644 index 0000000000..2459b46d2e --- /dev/null +++ b/frontend/docs/components/WorkflowDiagram.tsx @@ -0,0 +1,264 @@ +import React from "react"; + +const WorkflowDiagram: React.FC = () => { + const nodeW = 130; + const nodeH = 46; + const rx = 10; + + return ( +
+
+ + + + + + + + + + + + + + + + + + + {/* Workflow label */} + + WORKFLOW + + + {/* Dashed container */} + + + {/* --- Row 1: Task A → Task B --- */} + {/* Task A */} + + + Task A + + + {/* Edge A → B */} + + + {/* Task B */} + + + Task B + + + {/* --- Fan out: B → C and B → D --- */} + {/* Edge B → C */} + + + {/* Edge B → D */} + + + {/* Task C */} + + + Task C + + + {/* Task D */} + + + Task D + + + {/* --- Both converge to Result --- */} + {/* Edges C → Result, D → Result drawn before Result box so they don't overlap text */} + + {/* Annotations */} + + start + + + depends on A + + + parallel + + + parallel + + + {/* Vertical dashed line between sequential and parallel sections */} + + + fan-out + + + {/* Labels for sections */} + + sequential + + +
+
+ ); +}; + +export default WorkflowDiagram; diff --git a/frontend/docs/pages/home/_meta.js b/frontend/docs/pages/home/_meta.js index be9674e5d2..402e3abe23 100644 --- a/frontend/docs/pages/home/_meta.js +++ b/frontend/docs/pages/home/_meta.js @@ -6,6 +6,19 @@ export default { index: "🪓 Welcome", architecture: "Architecture", "guarantees-and-tradeoffs": "Guarantees & Tradeoffs", + "--workflows": { + title: "Workflows", + type: "separator", + }, + "workflows-overview": "Workflows (DAGs)", + + "durable-workflows-overview": "Durable Workflows", + "--patterns-and-use-cases": { + title: "Patterns and Use Cases", + type: "separator", + }, + patterns: "Patterns", + "use-cases": "Use Cases", "--quickstart": { title: "Setup", type: "separator", diff --git a/frontend/docs/pages/home/durable-workflows-overview.mdx b/frontend/docs/pages/home/durable-workflows-overview.mdx new file mode 100644 index 0000000000..bbd6e8b830 --- /dev/null +++ b/frontend/docs/pages/home/durable-workflows-overview.mdx @@ -0,0 +1,87 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import DurableWorkflowDiagram from "@/components/DurableWorkflowDiagramWrapper"; + +# Durable Workflows + +A **durable workflow** is a long-running task that stores its progress in a durable event log. If the task is interrupted (such as a crash, a deployment, or a scaling event), it replays from its last checkpoint instead of starting over. This makes durable workflows ideal for tasks that require deterministic behavior and outcomes. + + + +## How Durable Execution Works + + + +### Task runs and checkpoints + +As a durable task executes, each call to `SleepFor`, `WaitForEvent`, `WaitFor`, or `Memo` creates a checkpoint in the durable event log. These checkpoints record the task's progress. + +### Worker slot is freed during waits + +When a durable task enters a long wait (or sleep), the worker slot is released. The task is not consuming compute resources while waiting, unlike a regular task that holds its slot for the entire duration. + +### Task resumes from checkpoint + +If the task is interrupted or the wait completes, Hatchet replays the event log up to the last checkpoint and resumes execution from there. Completed operations are not re-executed. + + + +## The Durable Context + +Durable tasks receive a `DurableContext` instead of a regular `Context`. This extends the standard context with methods for durable execution: + +| Method | Purpose | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`SleepFor(duration)`** | Pause for a fixed duration. Respects the original sleep time on restart; if interrupted after 23 of 24 hours, only sleeps 1 more hour | +| **`WaitForEvent(key, expr)`** | Wait for an external event by key, with optional [CEL filter](https://github.com/google/cel-spec) expression on the payload | +| **`WaitFor(conditions)`** | General-purpose wait accepting any combination of sleep conditions, event conditions, or or-groups. `SleepFor` and `WaitForEvent` are convenience wrappers around this method | +| **`Memo(function)`** | Run functions whose outputs are memoized based on the input arguments | + +## Determinism Rules + +Durable tasks **must be deterministic**: they must perform the same sequence of operations on every replay. This is what allows Hatchet to safely skip already-completed work. + + + +### WIP + +### Never change the order of operations + +If your task calls `SleepFor` then `WaitForEvent`, don't later swap their order. Existing checkpoints depend on the original sequence. Changing it breaks replay. + +### WIP + + + + + If your workflow can be expressed as a [DAG](/home/dags), prefer regular + workflows. DAGs are inherently deterministic and don't require you to think + about replay safety. Use durable workflows when you need inline `SleepFor`, + `WaitForEvent`, or logic that can't be expressed as a static graph. + + +## When to Use Durable Workflows + +| Scenario | Why Durable? | +| --------------------------------- | ---------------------------------------------------------------------- | +| **Long waits** (hours/days) | Worker slots are freed during waits; no wasted compute | +| **Human-in-the-loop** | Wait for approval events without holding resources | +| **Multi-step with inline pauses** | `SleepFor` and `WaitForEvent` let you express complex procedural flows | +| **Large fan-out with collection** | Spawn many children and wait for results without holding a slot | +| **Crash-resilient pipelines** | Automatically resume from checkpoints after failures | + +## Next Steps + + + + Pause tasks for exact durations with crash-safe timing guarantees. + + + Wait for external signals with key matching and CEL filter expressions. + + + Compare with regular DAG workflows and understand when to use each. + + + See the long waits pattern in action with interactive diagrams. + + diff --git a/frontend/docs/pages/home/index.mdx b/frontend/docs/pages/home/index.mdx index d5273878ed..9f08e976ba 100644 --- a/frontend/docs/pages/home/index.mdx +++ b/frontend/docs/pages/home/index.mdx @@ -8,6 +8,8 @@ You write simple functions, called [tasks](./home/your-first-task), in Python, T Hatchet handles scheduling, complex assignment, fault tolerance, and observability so you can focus on building your application as you scale. +## Patterns + ## Use-Cases While Hatchet is a general-purpose orchestration platform, it's particularly well-suited for: diff --git a/frontend/docs/pages/home/patterns/_meta.js b/frontend/docs/pages/home/patterns/_meta.js new file mode 100644 index 0000000000..538d8f1302 --- /dev/null +++ b/frontend/docs/pages/home/patterns/_meta.js @@ -0,0 +1,7 @@ +export default { + fanout: "Fanout", + "pre-determined-pipelines": "Pre-Determined Pipelines", + branching: "Branching", + cycles: "Cycles", + "long-waits": "Long Waits", +}; diff --git a/frontend/docs/pages/home/patterns/branching.mdx b/frontend/docs/pages/home/patterns/branching.mdx new file mode 100644 index 0000000000..4679f1ce8d --- /dev/null +++ b/frontend/docs/pages/home/patterns/branching.mdx @@ -0,0 +1,91 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import BranchingDiagram from "@/components/BranchingDiagramWrapper"; +import PatternComparison from "@/components/PatternComparison"; + +# Branching + +**Branching** lets your workflow take different paths based on conditions evaluated at runtime. A task can inspect the output of a parent task, wait for an external event, or sleep for a duration, and then decide whether to run, skip, or cancel downstream tasks. Only the relevant branch executes, while the other is skipped. + + + +## How It Works + + + +### A task produces output + +An upstream task runs and returns a result (for example, a score, a flag, or a status value). + +### Conditions evaluate the output + +Downstream tasks declare conditions using `skip_if`, `wait_for`, or `cancel_if`. Hatchet evaluates these conditions against parent outputs, incoming events, or sleep timers. + +### One branch executes + +The branch whose condition is satisfied runs. + + + +## Types of Conditions + +| Condition | Behavior | +| -------------------- | ---------------------------------------------------------------------------------- | +| **Parent condition** | Evaluate a CEL expression against a parent task's output (e.g. `value > 50`) | +| **Event condition** | Wait for an external event, optionally filtered by a CEL expression on its payload | +| **Sleep condition** | Wait for a specified duration before continuing | + +## Condition Operators + +| Operator | Effect | +| ----------- | ----------------------------------------------------------------------------------- | +| `wait_for` | Task waits until the condition is satisfied before starting | +| `skip_if` | Task is skipped if the condition evaluates to true | +| `cancel_if` | Task and its downstream dependents are cancelled if the condition evaluates to true | + + + Branching is built using [Conditional Workflows](/home/conditional-workflows). + See that page for full code examples covering parent conditions, event + conditions, sleep conditions, or groups, and a complete branching workflow + walkthrough. + + + + A task cancelled by `cancel_if` behaves like any other cancellation, and all + downstream tasks are cancelled as well. Use `skip_if` when you want downstream + tasks to still run and inspect the skip status. + + +## In Workflows vs Durable Workflows (WIP) + +## Use Cases + + + + Read a feature flag in an upstream task and route the workflow down + different code paths based on its value. + + + Pause workflow execution until a human approves or rejects, then branch + accordingly. + + + Route data through different processing paths based on experiment + assignment. + + + Branch into a fallback path when an upstream task signals a degraded state. + + + +## Next Steps + +- [Conditional Workflows](/home/conditional-workflows): full guide to conditions, or groups, and operators +- [DAG Workflows](/home/dags): define task dependencies that branching builds on +- [Fanout](/home/patterns/fanout): dynamically spawn tasks instead of choosing between fixed branches +- [Pre-Determined Pipelines](/home/patterns/pre-determined-pipelines): fixed-structure pipelines without branching diff --git a/frontend/docs/pages/home/patterns/cycles.mdx b/frontend/docs/pages/home/patterns/cycles.mdx new file mode 100644 index 0000000000..5c3f004e36 --- /dev/null +++ b/frontend/docs/pages/home/patterns/cycles.mdx @@ -0,0 +1,78 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import CycleDiagram from "@/components/CycleDiagramWrapper"; +import PatternComparison from "@/components/PatternComparison"; + +# Cycles + +A **cycle** is a workflow pattern where a task re-runs itself repeatedly until a condition is met. Since Hatchet workflows are DAGs (no circular dependencies), cycles are implemented by having a task spawn a new child run of itself, effectively creating an iterative loop through child task spawning. + + + +## How It Works + + + +### Task performs work + +The task executes its logic: making an API call, running an LLM inference, checking a status, or processing a batch of data. + +### Evaluate a termination condition + +After each execution, the task checks whether the loop should continue. This could be a result quality threshold, a maximum iteration count, or an external signal. + +### Re-spawn or complete + +If the condition is not met, the task spawns a new child run of itself with updated input. If the condition is met, the task returns its final result. + + + + + Cycles use [Procedural Child Spawning](/home/child-spawning) under the hood; + each iteration is a new child task run. This gives you full observability into + every iteration in the Hatchet dashboard. + + +## When to Use Cycles + +| Pattern | Use Cycles When... | +| ------------------------ | ------------------------------------------------------------------------------- | +| **Polling** | You need to check an external system until a resource is ready | +| **Iterative Refinement** | An AI agent refines its output over multiple passes until quality is sufficient | +| **Pagination** | You process paginated API results one page at a time | +| **Retry with Backoff** | Built-in retries aren't sufficient and you need custom retry logic with state | +| **Agent Loops** | An AI agent reasons, acts, observes, and loops until a goal is achieved | + + + Always include a termination condition (max iterations, timeout, or quality + threshold) to prevent infinite loops. Each iteration consumes a worker slot, + so unbounded cycles can exhaust your worker capacity. + + +## In Workflows vs Durable Workflows (WIP) + +## Use Cases + + + + Build agents that reason, take action, observe results, and loop until they + achieve their goal. + + + Check an external service repeatedly until a resource is ready or a status + changes. + + + Process data in successive passes, refining results until convergence. + + + Implement application-specific retry strategies with state carried between + attempts. + + + +## Next Steps + +- [Procedural Child Spawning](/home/child-spawning): the mechanism that powers cycle iteration +- [Retry Policies](/home/retry-policies): built-in retry support for simpler retry scenarios +- [Branching](/home/patterns/branching): combine cycles with conditional logic +- [Fanout](/home/patterns/fanout): spawn parallel tasks instead of sequential iterations diff --git a/frontend/docs/pages/home/patterns/fanout.mdx b/frontend/docs/pages/home/patterns/fanout.mdx new file mode 100644 index 0000000000..cfd63cd6d0 --- /dev/null +++ b/frontend/docs/pages/home/patterns/fanout.mdx @@ -0,0 +1,96 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import FanoutDiagram from "@/components/FanoutDiagram"; +import PatternComparison from "@/components/PatternComparison"; + +# Fanout + +The **fanout pattern** lets a parent task dynamically spawn N child tasks in parallel, wait for all of them to complete, and optionally collect their results. It is the go-to approach when the number of parallel units of work is determined at runtime (for example, processing every item in a list, running inference on a batch of inputs, or sending notifications to a set of users). + + + +## How It Works + + + +### Define parent and child tasks + +The parent task contains the spawning logic. The child task defines the unit of work to be performed for each item. + +### Parent spawns N children at runtime + +When the parent task executes, it iterates over its input data and spawns one child task per item. All children run in parallel across your worker fleet. + +### Collect results + +The parent may choose to await all child completions and aggregate their results into a single return value. + + + + + Fanout is built on top of [Procedural Child Spawning](/home/child-spawning). + See that page for full code examples on defining tasks, spawning children, + parallel execution, and error handling. + + + + When fanning out to large numbers of children, make sure your workers have + enough slots to handle the concurrent load. See [Worker Slots](/home/workers) + for details on configuring slot capacity. + + +## In Workflows vs Durable Workflows + + + +## Use Cases + + + + Process thousands of items (images, documents, records) in parallel with + automatic load distribution across workers. + + + Fan out to map tasks, then reduce results in a downstream DAG step. + + + Send the same prompt to multiple LLM providers concurrently and pick the + best response. + + + Fan out to send notifications to N users while respecting per-provider rate + limits. + + + +## Next Steps + +- [Procedural Child Spawning](/home/child-spawning): learn the full child task API +- [DAG Workflows](/home/dags): combine fanout with declarative task dependencies +- [Concurrency Control](/home/concurrency): limit how many children run simultaneously +- [Rate Limits](/home/rate-limits): throttle child task execution rates diff --git a/frontend/docs/pages/home/patterns/long-waits.mdx b/frontend/docs/pages/home/patterns/long-waits.mdx new file mode 100644 index 0000000000..68791ecad3 --- /dev/null +++ b/frontend/docs/pages/home/patterns/long-waits.mdx @@ -0,0 +1,75 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import LongWaitDiagram from "@/components/LongWaitDiagramWrapper"; +import PatternComparison from "@/components/PatternComparison"; + +# Long Waits + +A **long wait** is a workflow pattern where a task pauses execution for an extended period (hours, days, or even weeks) and then resumes exactly where it left off. Hatchet's durable tasks make this possible without holding a worker slot open: the task may be evicted during the wait and re-scheduled when the wait completes. + + + +## How It Works + + + +### Task performs initial work + +The durable task runs its logic up to the point where it needs to wait, preparing data, making API calls, or setting up state. + +### Task enters a long wait + +The task calls a durable sleep (for a fixed duration) or waits for a durable event (an external signal). During this time, the worker slot is freed, and the task is not consuming resources. + +### Task resumes and completes + +When the sleep expires or the event arrives, Hatchet re-schedules the task on an available worker. The task picks up exactly where it left off, with all prior state intact. + + + + + Long waits require **durable tasks**. Regular tasks cannot pause and resume + across worker restarts. See [Durable Sleep](/home/durable-sleep) and [Durable + Events](/home/durable-events) for API details. + + +## Types of Long Waits + +| Type | Mechanism | Resumes When... | +| ----------------- | -------------------- | ------------------------------------------------------------------------ | +| **Durable Sleep** | `SleepFor(duration)` | The specified duration has elapsed | +| **Durable Event** | `WaitForEvent(key)` | An external event matching the key (and optional CEL filter) is received | + + + Durable sleeps are **exact**. If a task sleeps for 24 hours and is interrupted + after 23 hours, it will only sleep for 1 more hour on restart, not another + full 24 hours. This is what makes durable sleep different from a regular sleep + call. + + +## Use Cases + + + + Send a follow-up email or push notification hours or days after an initial + action. + + + Pause a workflow until a human approves, rejects, or provides input via an + external event. + + + Implement drip campaigns, trial expiration checks, or periodic status polls + with precise timing. + + + Wait for a webhook, payment confirmation, or third-party API callback before + proceeding. + + + +## Next Steps + +- [Durable Sleep](/home/durable-sleep): pause for a fixed duration with exact timing guarantees +- [Durable Events](/home/durable-events): wait for external signals with optional CEL filters +- [Conditional Workflows](/home/conditional-workflows): combine long waits with branching logic +- [Cycles](/home/patterns/cycles): use long waits inside iterative loops diff --git a/frontend/docs/pages/home/patterns/pre-determined-pipelines.mdx b/frontend/docs/pages/home/patterns/pre-determined-pipelines.mdx new file mode 100644 index 0000000000..063e70cf6c --- /dev/null +++ b/frontend/docs/pages/home/patterns/pre-determined-pipelines.mdx @@ -0,0 +1,72 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import PipelineDiagram from "@/components/PipelineDiagram"; +import PatternComparison from "@/components/PatternComparison"; + +# Pre-Determined Pipelines + +A **pre-determined pipeline** is a workflow where the sequence of tasks and their dependencies are defined ahead of time as a Directed Acyclic Graph (DAG). Unlike [fanout](/home/patterns/fanout), where children are spawned dynamically at runtime, pipelines have a fixed structure that is known before execution begins. + + + +## How It Works + + + +### Declare a workflow + +Define a named workflow that acts as the container for your pipeline. This is the entry point for running the entire pipeline. + +### Define tasks with dependencies + +Each task in the pipeline specifies its parent tasks. Hatchet uses these dependencies to build a DAG and determines the execution order automatically. Tasks without dependencies run first; tasks with parents wait until all parents complete. + +### Pass data between tasks + +Tasks can read the outputs of their parent tasks through the context object. This lets you thread data through the pipeline without external state. + + + + + Pre-determined pipelines are built using [DAG Workflows](/home/dags). See that + page for full code examples on defining workflows, adding tasks with + dependencies, accessing parent outputs, and running workflows. + + +## When to Use Pipelines vs Fanout + +| | Pre-Determined Pipelines | Fanout | +| ---------------- | -------------------------------------------------- | ----------------------------------------- | +| **Structure** | Fixed at definition time | Dynamic at runtime | +| **Task count** | Known ahead of time | Determined by input data | +| **Dependencies** | Explicit parent-child DAG | Parent spawns N identical children | +| **Best for** | ETL, data processing stages, multi-step transforms | Batch processing, parallel map operations | + +## In Workflows vs Durable Workflows (WIP) + +## Use Cases + + + + Extract data, transform it through multiple stages, and load it into a + destination, each stage as a task with clear dependencies. + + + Build, test, and deploy in sequence with parallel test suites that fan back + into a deploy step. + + + Parse, validate, enrich, and index documents through a fixed sequence of + processing stages. + + + Chain LLM calls with validation gates (generate, evaluate, refine) where + each step depends on the previous. + + + +## Next Steps + +- [DAG Workflows](/home/dags): full guide to defining workflows with task dependencies +- [Fanout](/home/patterns/fanout): dynamically spawn tasks at runtime instead +- [Concurrency Control](/home/concurrency): limit concurrent task execution within a pipeline +- [Procedural Child Spawning](/home/child-spawning): combine pipelines with dynamic child tasks diff --git a/frontend/docs/pages/home/use-cases/_meta.js b/frontend/docs/pages/home/use-cases/_meta.js new file mode 100644 index 0000000000..510a58e324 --- /dev/null +++ b/frontend/docs/pages/home/use-cases/_meta.js @@ -0,0 +1,6 @@ +export default { + "ai-agents": "AI Agents", + "rag-and-indexing": "RAG & Data Indexing", + "batch-processing": "Batch Processing", + "event-driven": "Event-Driven Systems", +}; diff --git a/frontend/docs/pages/home/use-cases/ai-agents.mdx b/frontend/docs/pages/home/use-cases/ai-agents.mdx new file mode 100644 index 0000000000..6d56a17ffd --- /dev/null +++ b/frontend/docs/pages/home/use-cases/ai-agents.mdx @@ -0,0 +1,100 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import AgentLoopDiagram from "@/components/AgentLoopDiagramWrapper"; + +# AI Agents + +AI agents follow a **reason-act-observe** loop that can run for minutes or hours. This page covers how to structure agent workflows in Hatchet so they survive interruptions, stream output to users, and don't exhaust resources. + + + +## What Makes Agents Different + + + +### Long-running loops + +An agent loop may iterate dozens of times, calling an LLM, parsing tool calls, executing tools, and evaluating results. Durable execution checkpoints each step, so if a worker restarts mid-loop the agent resumes from its last checkpoint instead of starting over. + +### Streaming output + +Agents typically need to stream LLM tokens to a frontend as they're generated. Use `put_stream()` in your task to emit chunks, and `subscribe_to_stream()` on the client to receive them. + +### Concurrency and cancellation + +`CANCEL_IN_PROGRESS` concurrency cancels stale agent runs when a user sends a new message. Rate limits prevent exceeding LLM provider API quotas. Priority queues let interactive agent tasks run ahead of batch work. + + + +## Key Features + +| Feature | What it does for agents | +| --------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| **[Durable Execution](/home/durable-workflows-overview)** | Checkpoints tool calls, LLM responses, and loop iterations so agent state survives crashes | +| **[Streaming](/home/streaming)** | Stream LLM tokens from workers to frontends in real-time | +| **[Durable Events](/home/durable-events)** | Pause for human-in-the-loop feedback without holding worker slots | +| **[Durable Sleep](/home/durable-sleep)** | Schedule agent retries or delayed actions with exact timing | +| **[Concurrency](/home/concurrency)** | Cancel stale runs, limit parallel agents per user | +| **[Rate Limits](/home/rate-limits)** | Stay within LLM provider API rate limits across all workers | +| **[Child Spawning](/home/child-spawning)** | Agents dynamically spawn sub-agents or tool-call tasks | +| **[Cancellation](/home/cancellation)** | Stop long-running agents on user request | + + + The agent loop pattern is a [Cycle](/home/patterns/cycles). Each iteration + spawns a new child run via `RunChild`. Combined with [Durable + Execution](/home/durable-workflows-overview), completed iterations survive + crashes and slots are freed between iterations. + + +## Typical Agent Flow + + + +### Receive user input + +A trigger (API call, webhook, or event) starts the agent workflow with the user's message and conversation context. + +### Reasoning loop + +The agent enters a durable task loop: call the LLM, parse tool calls, execute tools via child tasks, observe results, and decide whether to continue or respond. + +### Stream response + +As the LLM generates its final response, tokens are streamed through Hatchet to the frontend in real-time. + +### Wait for next input + +The agent uses `WaitForEvent` to pause until the user sends another message, freeing the worker slot for other work. + + + + + Always set a **timeout** and **max iteration count** on agent loops. Without + bounds, an agent can loop indefinitely. See [Timeouts](/home/timeouts) for + configuration. + + +## Related Patterns + + + + The core loop pattern behind agent reasoning, where a task re-spawns itself + until a goal is met. + + + Pause agents for human feedback or scheduled retries without holding worker + slots. + + + Agents that spawn parallel tool calls or sub-agent tasks. + + + Route agent behavior based on LLM tool call decisions or user preferences. + + + +## Next Steps + +- [Durable Workflows](/home/durable-workflows-overview): understand checkpointing and replay +- [Streaming](/home/streaming): set up real-time LLM output streaming +- [Concurrency Control](/home/concurrency): configure CANCEL_IN_PROGRESS for chat agents +- [Child Spawning](/home/child-spawning): spawn tool-call tasks from agent loops diff --git a/frontend/docs/pages/home/use-cases/batch-processing.mdx b/frontend/docs/pages/home/use-cases/batch-processing.mdx new file mode 100644 index 0000000000..467dd955c1 --- /dev/null +++ b/frontend/docs/pages/home/use-cases/batch-processing.mdx @@ -0,0 +1,111 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import BatchProcessingDiagram from "@/components/BatchProcessingDiagramWrapper"; + +# Batch Processing + +Batch processing involves running the same operation across a large set of items, such as images, documents, records, or API calls. This page covers how to structure batch workloads as Hatchet workflows with fan-out, retry, and concurrency control. + + + +## Core Challenges + + + +### Parallelism + +A parent task fans out to one child per item. Hatchet distributes these across available workers. Adding more workers increases throughput without code changes. + +### Partial failure + +Each item is an independent task. If one fails, Hatchet retries just that item; the rest continue. You can also bulk-retry all failed items from the dashboard. + +### Resource control + +Concurrency limits prevent overwhelming your infrastructure. Rate limits protect external APIs. Priority queues let urgent batches run ahead of lower-priority work. + + + +## Key Features + +| Feature | What it does for batch processing | +| -------------------------------------------------------- | ----------------------------------------------------------------------- | +| **[Child Spawning](/home/child-spawning)** | Fan out to one task per item with automatic distribution across workers | +| **[Bulk Run](/home/bulk-run)** | Trigger thousands of tasks in a single API call | +| **[Retry Policies](/home/retry-policies)** | Retry failed items individually without restarting the batch | +| **[Bulk Retries](/home/bulk-retries-and-cancellations)** | Re-run all failed items from the dashboard | +| **[Concurrency](/home/concurrency)** | Limit how many items process simultaneously | +| **[Rate Limits](/home/rate-limits)** | Throttle external API calls across all workers | +| **[Priority](/home/priority)** | Urgent batches jump ahead of lower-priority work | +| **[Autoscaling](/home/autoscaling-workers)** | Scale workers up during batch processing, down when idle | + + + The batch processing pattern is [Fanout](/home/patterns/fanout) applied at + scale. For fixed multi-stage processing (e.g., validate → transform → load), + combine with [Pre-Determined + Pipelines](/home/patterns/pre-determined-pipelines). + + +## Architecture + + + +### Trigger the batch + +Start a parent workflow with the batch input (a list of item IDs, file paths, or records). This can come from an API call, event, cron schedule, or the dashboard. + +### Fan out to workers + +The parent task iterates over the input and spawns one child task per item. For very large batches (10,000+ items), use `BulkRunChild` for optimized dispatching. + +### Process items + +Each child task processes its item independently, calling external APIs, transforming data, and writing results. Failed items are retried according to your retry policy. + +### Collect results + +The parent awaits all children and aggregates results. You can see the status of every item in real-time in the Hatchet dashboard. + + + + + For batches with thousands of items, use **durable workflows** so the parent + task doesn't hold a worker slot while waiting for all children to complete. + See [Durable Workflows](/home/durable-workflows-overview) for details. + + +## Common Batch Patterns + +| Pattern | Description | +| ------------------------- | ----------------------------------------------------------------------------------- | +| **Image processing** | Resize, transcode, or analyze images in parallel across workers | +| **Data enrichment** | Enrich records by calling external APIs (geocoding, company info, email validation) | +| **Report generation** | Generate per-customer reports in parallel, then aggregate into a summary | +| **Database migrations** | Process and migrate records in batches with retry and progress tracking | +| **Notification delivery** | Send emails, SMS, or push notifications to a user list with rate limiting | + +## Related Patterns + + + + The core pattern behind batch processing, spawning N children from a parent. + + + Chain batch processing with multi-stage transforms in a DAG. + + + A specialized batch processing use case for document indexing pipelines. + + + Process paginated results one page at a time with iterative child spawning. + + + +## Next Steps + +- [Child Spawning](/home/child-spawning): learn the fan-out API for batch processing +- [Bulk Run](/home/bulk-run): trigger large batches efficiently +- [Concurrency Control](/home/concurrency): limit concurrent item processing +- [Rate Limits](/home/rate-limits): protect external APIs during batch operations diff --git a/frontend/docs/pages/home/use-cases/event-driven.mdx b/frontend/docs/pages/home/use-cases/event-driven.mdx new file mode 100644 index 0000000000..1c6db19e0c --- /dev/null +++ b/frontend/docs/pages/home/use-cases/event-driven.mdx @@ -0,0 +1,110 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import EventDrivenDiagram from "@/components/EventDrivenDiagramWrapper"; + +# Event-Driven Systems + +Event-driven systems react to webhooks, application events, cron schedules, and API calls. This page covers how Hatchet routes events to tasks with concurrency control and spike protection. + + + +## Core Challenges + + + +### Multiple trigger types + +Webhooks, cron schedules, application events, and API calls can all trigger Hatchet tasks. Define your task handlers once and attach whichever triggers you need. + +### Spike protection + +When a burst of events arrives, concurrency controls queue excess work instead of overwhelming your workers. `CANCEL_NEWEST` drops stale events; `GROUP_ROUND_ROBIN` ensures fair processing across sources. + +### Routing and delivery + +Hatchet matches events to registered task triggers, handles deduplication and filtering, and distributes work to available workers. Failed tasks are retried according to your retry policy. + + + +## Key Features + +| Feature | What it does for event-driven systems | +| -------------------------------------------------------------- | ------------------------------------------------------------------- | +| **[Event Triggers](/home/run-on-event)** | React to application events by name with optional payload filtering | +| **[Webhooks](/home/webhooks)** | Receive external HTTP webhooks and route them to tasks | +| **[Cron Triggers](/home/cron-runs)** | Schedule recurring tasks with cron expressions | +| **[Scheduled Runs](/home/scheduled-runs)** | Trigger one-time future execution at a specific time | +| **[Concurrency](/home/concurrency)** | Spike protection with CANCEL_NEWEST or CANCEL_OLDEST strategies | +| **[Rate Limits](/home/rate-limits)** | Throttle downstream processing regardless of event volume | +| **[Inter-Service Triggering](/home/inter-service-triggering)** | Trigger tasks across different services and worker pools | +| **[Additional Metadata](/home/additional-metadata)** | Tag events with metadata for filtering and observability | + + + Hatchet supports all common triggering patterns in a single system. See [Ways + of Running Tasks](/home/running-tasks) for the complete list of trigger types + and when to use each. + + +## Architecture + + + +### Events arrive + +Events come from multiple sources: HTTP webhooks from external services, application events emitted by your code, cron schedules, or direct API calls. + +### Hatchet routes to tasks + +Each event is matched to registered task triggers. A single event can trigger multiple tasks. Hatchet handles deduplication, filtering, and queuing. + +### Workers process tasks + +Tasks are distributed to available workers. Concurrency controls prevent overload. Failed tasks are retried according to your retry policy. + +### Chain further work + +Task outputs can trigger further events, spawn child tasks, or start downstream workflows. Complex event-driven pipelines are composed from individual tasks. + + + + + When processing high-volume events, always configure **concurrency controls** + to prevent worker saturation. Without limits, a burst of 10,000 events will + attempt to run 10,000 tasks simultaneously. + + +## Common Event Patterns + +| Pattern | Description | +| ------------------------------- | ----------------------------------------------------------------------------- | +| **Webhook processing** | Receive Stripe, GitHub, or Twilio webhooks and process them reliably | +| **User action handlers** | React to user sign-ups, purchases, or content uploads with async workflows | +| **Scheduled maintenance** | Run database cleanup, cache invalidation, or health checks on a cron schedule | +| **Cross-service orchestration** | Coordinate workflows across microservices without direct coupling | +| **Real-time data sync** | Keep systems in sync by reacting to change events from databases or APIs | + +## Related Patterns + + + + Route event-driven workflows down different paths based on event payload. + + + A single event triggers parallel processing across multiple workers. + + + Pause a workflow until an external event arrives, then resume. + + + Chain event-triggered tasks into multi-stage processing pipelines. + + + +## Next Steps + +- [Event Triggers](/home/run-on-event): trigger tasks from application events +- [Webhooks](/home/webhooks): receive and process external webhooks +- [Cron Triggers](/home/cron-runs): set up recurring scheduled tasks +- [Concurrency Control](/home/concurrency): configure spike protection diff --git a/frontend/docs/pages/home/use-cases/rag-and-indexing.mdx b/frontend/docs/pages/home/use-cases/rag-and-indexing.mdx new file mode 100644 index 0000000000..9e61604961 --- /dev/null +++ b/frontend/docs/pages/home/use-cases/rag-and-indexing.mdx @@ -0,0 +1,113 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import RAGPipelineDiagram from "@/components/RAGPipelineDiagramWrapper"; + +# RAG & Data Indexing + +RAG and indexing pipelines share a common shape: ingest documents, split them into chunks, generate embeddings, and write to a vector database. This page covers how to structure these pipelines as Hatchet workflows. + + + +## Core Challenges + + + +### Fan-out to many documents + +When new data arrives, a parent task fans out to child tasks, one per document or chunk. Hatchet distributes these across your workers. The number of children is determined at runtime by the input data. + +### Rate-limiting embedding APIs + +Embedding providers (OpenAI, Cohere, Voyage) enforce rate limits. Hatchet's rate limiting throttles embedding tasks across all workers to stay within provider quotas. + +### Retrying without re-processing + +If a task fails (embedding API timeout, database write error), Hatchet retries just that task, not the entire pipeline. Completed chunks stay indexed. + + + +## Key Features + +| Feature | What it does for RAG | +| ------------------------------------------ | ------------------------------------------------------------------------------ | +| **[Child Spawning](/home/child-spawning)** | Fan out to one task per document/chunk; process thousands in parallel | +| **[Rate Limits](/home/rate-limits)** | Throttle embedding API calls across all workers to stay within provider limits | +| **[Retry Policies](/home/retry-policies)** | Retry failed chunks individually without re-processing the whole pipeline | +| **[Concurrency](/home/concurrency)** | Fair scheduling with GROUP_ROUND_ROBIN for multi-tenant indexing | +| **[DAG Workflows](/home/dags)** | Chain pipeline stages (ingest → chunk → embed → index) with clear dependencies | +| **[Event Triggers](/home/run-on-event)** | Start indexing automatically when new documents arrive | +| **[Cron Triggers](/home/cron-runs)** | Schedule periodic re-indexing or incremental updates | +| **[Bulk Run](/home/bulk-run)** | Trigger indexing for thousands of documents in a single API call | + + + RAG pipelines are a natural fit for [Pre-Determined + Pipelines](/home/patterns/pre-determined-pipelines). The stages (ingest, + chunk, embed, index) are fixed and map directly to a DAG. Use + [Fanout](/home/patterns/fanout) within the chunking stage to parallelize. + + +## Pipeline Architecture + + + +### Ingest + +A trigger (event, cron, or API call) starts the pipeline with document references. The ingest task fetches raw content from your data source (S3, database, API). + +### Chunk + +The parent task fans out to child tasks, one per document. Each child splits its document into chunks using your preferred strategy (fixed-size, semantic, recursive). + +### Embed + +Each chunk task calls your embedding provider. Hatchet rate-limits these calls across all workers to respect API quotas. Failed embeddings are retried individually. + +### Index + +Embeddings are written to your vector database (Pinecone, Weaviate, pgvector). Hatchet retries on transient database errors without re-computing embeddings. + +### Query (optional) + +A separate workflow handles user queries, generating query embeddings, searching the vector database, and passing context to the LLM. + + + + + When fanning out to many chunks, ensure your workers have enough slots or use + [Concurrency Control](/home/concurrency) to limit how many run simultaneously. + + +## Multi-Tenant Indexing + +For SaaS applications where multiple tenants share the same pipeline: + +- **GROUP_ROUND_ROBIN concurrency** distributes scheduling fairly so no single tenant monopolizes workers +- **Additional metadata** tags each run with a tenant ID for filtering in the dashboard +- **Priority queues** allow higher-priority indexing jobs to run ahead of lower-priority ones + +## Related Patterns + + + + The fixed-stage DAG pattern that RAG pipelines are built on. + + + Parallelize document and chunk processing across your worker fleet. + + + Implement incremental indexing that re-crawls until all changes are + processed. + + + General-purpose batch processing patterns that apply to indexing workloads. + + + +## Next Steps + +- [DAG Workflows](/home/dags): define multi-stage pipelines with task dependencies +- [Rate Limits](/home/rate-limits): configure rate limiting for embedding APIs +- [Child Spawning](/home/child-spawning): fan out to per-document tasks +- [Concurrency Control](/home/concurrency): fair scheduling for multi-tenant indexing diff --git a/frontend/docs/pages/home/workflows-overview.mdx b/frontend/docs/pages/home/workflows-overview.mdx new file mode 100644 index 0000000000..d8cf339085 --- /dev/null +++ b/frontend/docs/pages/home/workflows-overview.mdx @@ -0,0 +1,67 @@ +import { Callout, Cards, Steps } from "nextra/components"; +import WorkflowDiagram from "@/components/WorkflowDiagram"; + +# Workflows (DAGs) + +A **workflow** is a set of tasks connected by dependencies. You define the shape of work upfront (which tasks run, in what order, and what depends on what) and Hatchet handles execution, retries, and observability for you. + +Workflows in Hatchet are **Directed Acyclic Graphs (DAGs)**: each task is a node, dependencies are edges, and there are no circular paths. This structure makes workflows predictable, debuggable, and easy to reason about. + + + +## What a Workflow Gives You + + + +### Declarative task dependencies + +Define which tasks depend on which. Hatchet automatically executes tasks in the right order and parallelizes independent tasks. + +### Automatic retries and error handling + +Each task has configurable [retry policies](/home/retry-policies) and [timeouts](/home/timeouts). If a task fails, Hatchet retries it without re-running already-completed tasks. + +### Full observability + +Every task execution is tracked in the Hatchet dashboard, including inputs, outputs, durations, and errors. You can see exactly where a workflow succeeded or failed. + +### Cached intermediate results + +Task outputs are stored and passed to downstream tasks. If a workflow is partially complete when a failure occurs, completed tasks don't need to re-run. + + + +## Anatomy of a Workflow + +| Concept | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Task** | An atomic unit of work. A function that takes input and returns output | +| **Dependency** | A parent-child relationship between tasks. A child task waits for its parents to complete | +| **Input** | Data passed to the workflow when it's triggered | +| **Output** | Data returned by each task, accessible to downstream tasks | +| **Trigger** | How a workflow starts: [direct run](/home/run-with-results), [event](/home/run-on-event), [cron](/home/cron-runs), [schedule](/home/scheduled-runs), or [webhooks](/home/webhooks) | + + + Workflows can be as simple as a single task or as complex as dozens of tasks + with branching, fan-out, and conditional logic. Start simple and compose as + needed. + + +## Next Steps + + + + Learn how durable workflows checkpoint, pause, and resume across failures. + + + Define multi-task workflows with dependencies, parent outputs, and parallel + execution. + + + Dynamically create child tasks at runtime for fan-out, loops, and agent + patterns. + + + Explore common workflow patterns: fanout, branching, cycles, and more. + + diff --git a/frontend/docs/styles/global.css b/frontend/docs/styles/global.css index 26369c23a8..efb8c45e53 100644 --- a/frontend/docs/styles/global.css +++ b/frontend/docs/styles/global.css @@ -256,7 +256,7 @@ h3 { background-color: var(--background) !important; } -nav { +nav:not(.nextra-toc) { background-color: var(--background) !important; } diff --git a/frontend/docs/theme.config.tsx b/frontend/docs/theme.config.tsx index cb89069176..bf0de9f2f3 100644 --- a/frontend/docs/theme.config.tsx +++ b/frontend/docs/theme.config.tsx @@ -175,6 +175,9 @@ const config = { `https://github.com/hatchet-dev/hatchet/issues/new`, }, footer: false, + toc: { + backToTop: true, + }, sidebar: { defaultMenuCollapseLevel: 2, toggleButton: true, diff --git a/frontend/snippets/generate.py b/frontend/snippets/generate.py index ab89e50d0f..93f152755b 100644 --- a/frontend/snippets/generate.py +++ b/frontend/snippets/generate.py @@ -302,7 +302,7 @@ def write_doc_index_to_app() -> None: for filename in glob.iglob(path, recursive=True): with open(filename) as f: - content = f.read().replace("export default ", "") + content = f.read().replace("export default ", "").strip().rstrip(";") parsed_meta = cast( dict[str, Any], json.loads(content, cls=JavaScriptObjectDecoder) )