Archiver: fix race condition, dead code path, and remove spin-wait

internal/pkg/archiver/general/body.go

@@ -47,6 +47,7 @@ func processBody(u *models.URL, disableAssetsCapture, domainsCrawl bool, maxHops
 		if err := connutil.CopyWithTimeout(io.Discard, u.GetResponse().Body); err != nil {
 			return err
 		}
+		return nil
 	}
 
 	// Get a buffer from the pool for MIME type detection

internal/pkg/archiver/headless/archiver.go

@@ -327,11 +327,13 @@ func archivePage(warcClient *warc.CustomHTTPClient, item *models.Item, seed *mod
 	// Navigate to the URL
 	logger.Debug("navigating to URL")
 
-	go router.Run()
+	ready := make(chan struct{})
+	go func() {
+		close(ready) // router is now running and handlers are active
+		router.Run()
+	}()
 
-	// Wait for the router to start to avoid race condition in rod
-	// The race happens between router.Run() initializing events and page.Navigate() triggering events.
-	time.Sleep(100 * time.Millisecond)
+	<-ready
 
 	err = page.Navigate(item.GetURL().String())
 	if err != nil {

internal/pkg/archiver/ratelimiter/ratelimiter.go

@@ -59,8 +59,10 @@ func (tb *tokenBucket) Wait() {
 			tb.mu.Unlock()
 			return
 		}
-		tb.mu.Unlock()
-		time.Sleep(50 * time.Millisecond) // adjust as needed
+		waitDuration := time.Duration(float64(time.Second) / tb.refillRate)
+        tb.mu.Unlock()
+        time.Sleep(waitDuration)
+        tb.mu.Lock()
 	}
 }
 

pkg/models/item_dedupe.go

@@ -38,10 +38,10 @@ func flattenTree(root *Item) []*Item {
 	var nodes []*Item
 	var traverse func(node *Item)
 	traverse = func(node *Item) {
-		nodes = append(nodes, node)
 		if node == nil {
 			return
 		}
+		nodes = append(nodes, node)
 		for _, child := range node.GetChildren() {
 			traverse(child)
 		}