feat: handle metas changing case

Author: portwatcher

README.md

@@ -104,16 +104,31 @@ When a URL is submitted via the `/generate` endpoint:
 git clone <repository-url>
 cd prerender-url-shortener
 ```
-2.  **Create a `.env` file** in the project root with your configuration. See `.env.example` for a template (if one exists, otherwise define the following):
+2.  **Create a `.env` file** in the project root with your configuration. See `.env.example` for a template (if one exists). Current configuration options:
 
 ```env
+# Required
 DATABASE_URL="postgres://user:password@host:port/dbname?sslmode=disable"
-SERVER_PORT=":8080" # Optional, defaults to :8080
-ALLOWED_DOMAINS="example.com,another.org" # Optional, comma-separated, empty means allow all
-ROD_BIN_PATH="" # Optional, path to Chrome/Chromium binary if not in system PATH or for specific version
-RENDER_WORKER_COUNT="3" # Optional, number of background rendering workers, defaults to 3
+
+# Server
+SERVER_PORT=":8080"                       # Optional, default :8080
+ALLOWED_DOMAINS="example.com,another.org" # Optional, comma-separated; empty means allow all
+
+# Renderer / Rod
+ROD_BIN_PATH=""                           # Optional, path to Chrome/Chromium binary
+RENDER_WORKER_COUNT="3"                   # Optional, default 3 workers
+RENDER_TIMEOUT_SECONDS="90"               # Optional, default 90s overall per-page timeout
+
+# Prerender stabilization
+META_WAIT_TIMEOUT_SECONDS="20"            # Optional, default 20s max wait for meta stabilization
+META_STABLE_CONSECUTIVE_CHECKS="3"        # Optional, default 3 equal reads for stability
+PRERENDER_READY_MARKER="data-prerender-ready=\"true\""  # Optional marker that, if present in HTML, ends waiting early
 ```
 
+Notes:
+- PRERENDER_READY_MARKER: leave it as default or set to an empty value to disable marker-based early exit. The renderer still returns once og:image stabilizes or after a brief loop if no og:image is present.
+- Without a ready marker: behavior is generic — waits for load + network idle; then only loops for og:image stabilization if an og:image exists.
+
 3.  **Install dependencies:**
 ```bash
 go mod tidy
@@ -167,4 +182,4 @@ docker buildx build --platform linux/amd64,linux/arm64 -t <your_username>/preren
 
 # For a local build (current platform only)
 docker build -t prerender-url-shortener .
-``` 
+``` 

internal/config/config.go

@@ -11,12 +11,16 @@ import (
 // Config holds the application configuration.
 // We'll use struct tags for environment variable loading.
 type Config struct {
-	ServerPort           string `env:"SERVER_PORT,default=:8080"`
-	DatabaseURL          string `env:"DATABASE_URL,required"`
-	RodBinPath           string `env:"ROD_BIN_PATH"`                      // Optional, if not in default PATH
-	AllowedDomains       string `env:"ALLOWED_DOMAINS"`                   // Comma-separated list of allowed domains
-	RenderWorkerCount    int    `env:"RENDER_WORKER_COUNT,default=3"`     // Number of render workers
-	RenderTimeoutSeconds int    `env:"RENDER_TIMEOUT_SECONDS,default=90"` // Timeout for Rod rendering in seconds
+    ServerPort           string `env:"SERVER_PORT,default=:8080"`
+    DatabaseURL          string `env:"DATABASE_URL,required"`
+    RodBinPath           string `env:"ROD_BIN_PATH"`                      // Optional, if not in default PATH
+    AllowedDomains       string `env:"ALLOWED_DOMAINS"`                   // Comma-separated list of allowed domains
+    RenderWorkerCount    int    `env:"RENDER_WORKER_COUNT,default=3"`     // Number of render workers
+    RenderTimeoutSeconds int    `env:"RENDER_TIMEOUT_SECONDS,default=90"` // Timeout for Rod rendering in seconds
+    // Prerender stabilization settings
+    MetaWaitTimeoutSeconds       int    `env:"META_WAIT_TIMEOUT_SECONDS,default=20"`          // Max time to wait for metas to stabilize
+    MetaStableConsecutiveChecks  int    `env:"META_STABLE_CONSECUTIVE_CHECKS,default=3"`     // Number of consecutive equal reads to consider stable
+    PrerenderReadyMarker         string `env:"PRERENDER_READY_MARKER,default=data-prerender-ready=\"true\""` // String to search in HTML as a ready signal
 }
 
 var AppConfig *Config
@@ -35,7 +39,10 @@ func LoadConfig() error {
 	AppConfig.RodBinPath = getEnv("ROD_BIN_PATH", "")
 	AppConfig.AllowedDomains = getEnv("ALLOWED_DOMAINS", "") // Empty means allow all
 	AppConfig.RenderWorkerCount = getEnvInt("RENDER_WORKER_COUNT", 3)
-	AppConfig.RenderTimeoutSeconds = getEnvInt("RENDER_TIMEOUT_SECONDS", 90)
+    AppConfig.RenderTimeoutSeconds = getEnvInt("RENDER_TIMEOUT_SECONDS", 90)
+    AppConfig.MetaWaitTimeoutSeconds = getEnvInt("META_WAIT_TIMEOUT_SECONDS", 20)
+    AppConfig.MetaStableConsecutiveChecks = getEnvInt("META_STABLE_CONSECUTIVE_CHECKS", 3)
+    AppConfig.PrerenderReadyMarker = getEnv("PRERENDER_READY_MARKER", "data-prerender-ready=\"true\"")
 
 	if AppConfig.DatabaseURL == "" {
 		log.Fatal("DATABASE_URL environment variable is required")

internal/renderer/renderer.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"log"
 	"prerender-url-shortener/internal/config"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/go-rod/rod"
@@ -102,7 +104,7 @@ func renderWithRod(url string) (string, error) {
 		return "", fmt.Errorf("failed to create page for %s: %w", url, err)
 	}
 	log.Printf("Rod: Page created successfully for URL: %s", url)
-	//nolint:errcheck
+
 	defer func() {
 		log.Printf("Rod: Closing page for URL: %s", url)
 		page.MustClose() // MustClose panics on error, no return to check.
@@ -121,7 +123,6 @@ func renderWithRod(url string) (string, error) {
 	// Wait for network to be almost idle, this is a good indicator for SPAs
 	// Using a timeout to prevent indefinite blocking
 	log.Printf("Rod: Waiting for network to be almost idle for URL: %s (timeout: 30s)", url)
-	//nolint:errcheck
 	page.Timeout(30 * time.Second).WaitNavigation(proto.PageLifecycleEventNameNetworkAlmostIdle)()
 	log.Printf("Rod: Network almost idle wait completed for URL: %s", url)
 
@@ -130,12 +131,91 @@ func renderWithRod(url string) (string, error) {
 	time.Sleep(2 * time.Second)
 	log.Printf("Rod: Additional wait completed for URL: %s", url)
 
-	log.Printf("Rod: Extracting HTML content for URL: %s", url)
-	html, err := page.HTML()
+    // Wait for metas (e.g., og:image) to reach their final state or a ready marker
+	log.Printf("Rod: Waiting for meta stabilization for URL: %s", url)
+	finalHTML, err := waitForMetaFinalization(page)
 	if err != nil {
-		return "", fmt.Errorf("failed to get HTML content for %s: %w", url, err)
+		log.Printf("Rod: Meta stabilization wait ended with error for URL: %s: %v. Returning current HTML.", url, err)
+		// Best-effort fallback to current HTML
+		html, hErr := page.HTML()
+		if hErr != nil {
+			return "", fmt.Errorf("failed to get HTML content after meta wait for %s: %w", url, hErr)
+		}
+		return html, nil
 	}
-	log.Printf("Rod: Successfully extracted HTML content for URL: %s (length: %d characters)", url, len(html))
+	log.Printf("Rod: Meta stabilization complete for URL: %s (length: %d characters)", url, len(finalHTML))
+	return finalHTML, nil
+}
+
+// waitForMetaFinalization polls the page HTML until either:
+// - The configured ready marker is present in the HTML, or
+// - The og:image meta content is stable for N consecutive checks (only if an og:image exists),
+// or times out based on config.
+func waitForMetaFinalization(page *rod.Page) (string, error) {
+	timeout := time.Duration(config.AppConfig.MetaWaitTimeoutSeconds) * time.Second
+	stableTarget := config.AppConfig.MetaStableConsecutiveChecks
+	if stableTarget < 1 {
+		stableTarget = 1
+	}
+
+    // Regex to extract meta content attributes
+    // Matches: <meta property="og:image" ... content="...">
+    // Note: We intentionally ignore twitter:image for stabilization. The renderer remains generic
+    // and will only loop for stability when an og:image is present.
+    ogRe := regexp.MustCompile(`(?i)<meta[^>]+property=["']og:image["'][^>]*content=["']([^"']+)["'][^>]*>`) //nolint:lll
+
+	deadline := time.Now().Add(timeout)
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+    var lastOG string
+    stableCount := 0
+    emptyCount := 0
+
+	readyMarker := strings.TrimSpace(config.AppConfig.PrerenderReadyMarker)
+
+	for {
+		// Fetch current HTML
+		html, err := page.HTML()
+		if err != nil {
+			return "", fmt.Errorf("failed to get HTML during meta wait: %w", err)
+		}
+        // html captured for checks below
+
+		// If a custom ready marker is present, we're done
+		if readyMarker != "" && strings.Contains(html, readyMarker) {
+			return html, nil
+		}
 
-	return html, nil
+        // Extract og:image
+        og := ""
+        if m := ogRe.FindStringSubmatch(html); len(m) > 1 {
+            og = m[1]
+        }
+
+        // Only loop for stability when og:image exists
+        if og != "" {
+            if og == lastOG {
+                stableCount++
+            } else {
+                stableCount = 1
+            }
+            lastOG = og
+            if stableCount >= stableTarget {
+                return html, nil
+            }
+        } else {
+            // If no og:image is present repeatedly, don't wait the full timeout.
+            // This keeps the renderer generic while avoiding long waits for pages without OG metadata.
+            emptyCount++
+            if emptyCount >= 3 { // ~1.5s given 500ms tick
+                return html, nil
+            }
+        }
+
+        if time.Now().After(deadline) {
+            return html, fmt.Errorf("meta wait timeout after %v", timeout)
+        }
+        <-ticker.C
+    }
 }