Order local queue URLs by number of hops and timestamp (#490)

* feat: order fresh URLs by number of hops and timestamp. This should allow us to more effectively crawl "in order" when using the local queue.

* add index for hops and timestamp. Should improve performance for new databases.

Author: NGTmeaty

internal/pkg/source/lq/query.sql

@@ -1,6 +1,7 @@
 -- name: GetFreshURLs :many
 SELECT * FROM urls
 WHERE status = 'FRESH'
+ORDER BY hops ASC, timestamp ASC
 LIMIT ?;
 
 -- name: ClaimThisURL :exec

internal/pkg/source/lq/schema.sql

@@ -8,3 +8,4 @@ CREATE TABLE IF NOT EXISTS urls (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS urls_value ON urls (value); -- for deduplication
 CREATE INDEX IF NOT EXISTS urls_status ON urls (status); -- for queueing
+CREATE INDEX IF NOT EXISTS urls_hops_timestamp ON urls (hops ASC, timestamp ASC); -- for sorting by crawl depth and time