behaviors: optionally collect outlinks while looping

This solves one of the existing limitations of our outlink gathering
system: we run it once, after all behaviours have completed. For
some interactive pages, particularly single-page apps with paginated
data, it means that we'll completely miss that content since it won't
be in the DOM anymore by the time we get around to it.

In a previous PR, #433, I made the outlink gathering logic reusable
by ensuring it's possible for us to dynamically call the outlink
gathering function at any time. In addition, in #429, I made it
possible for behaviours to return outlinks to Brozzler; if the
behaviour chooses to return outlinks, then Brozzler will add them to
the set it extracts after behavours complete.

This branch uses both of those by introducing new functionality in
behaviours. We always inject the outlink gathering code before running
behaviours, so they can now run it at will. It also introduces two new
behaviour parameters:

* `extractOutlinks` - If set to `true`, then the behaviour script will
  call the outlink gathering logic and return any outlinks it harvested
  to Brozzler. Defaults to `false`.
* `extractOutlinksInLoop` - If set to `true`, then the behaviour script
  will gather outlinks every iteration of the loop. This combines great
  with `repeatSameElement`, since it means the behaviour script can
  click a `next` pagination button and then immediately gather whatever
  new outlinks have appeared on the page. Defaults to `false`.

Author: mistydemeo

brozzler/browser.py

@@ -769,8 +769,7 @@ def navigate_to_page(self, page_url, timeout=300):
         self.send_to_chrome(method="Page.navigate", params={"url": page_url})
         self._wait_for(lambda: self.websock_thread.got_page_load_event, timeout=timeout)
 
-    def extract_outlinks(self, timeout=60) -> frozenset[str]:
-        self.logger.info("extracting outlinks")
+    def inject_outlink_extractor(self, timeout=60):
         self.websock_thread.expect_result(self._command_id.peek())
         js = brozzler.jinja2_environment().get_template("extract-outlinks.js").render()
         # This defines the method but doesn't extract outlinks yet
@@ -782,8 +781,14 @@ def extract_outlinks(self, timeout=60) -> frozenset[str]:
         self._wait_for(
             lambda: self.websock_thread.received_result(msg_id), timeout=timeout
         )
-        self.websock_thread.expect_result(self._command_id.peek())
 
+    def extract_outlinks(self, timeout=60) -> frozenset[str]:
+        self.logger.info("extracting outlinks")
+
+        # Define the outlink extractor first before extracting
+        self.inject_outlink_extractor(timeout=timeout)
+
+        self.websock_thread.expect_result(self._command_id.peek())
         # Now we actually do outlink extraction
         msg_id = self.send_to_chrome(
             method="Runtime.evaluate",
@@ -879,6 +884,9 @@ def url(self, timeout=30):
         return message["result"]["result"]["value"]
 
     def run_behavior(self, behavior_script, timeout=900) -> frozenset[str]:
+        # Inject the outlink extractor so it's available to behaviors
+        self.inject_outlink_extractor(timeout=timeout)
+
         self.send_to_chrome(
             method="Runtime.evaluate",
             suppress_logging=True,

brozzler/js-templates/umbraBehavior.js.j2

@@ -27,6 +27,7 @@ class UmbraBehavior {
         this.intervalId = null;
         this.intervalTimeMs = {{interval or 300}};
         this.index = 0;
+        this.outlinks = [];
     }
 
     simpleIntervalFunc() {
@@ -43,6 +44,8 @@ class UmbraBehavior {
         var didSomething = false;
         var somethingLeftAbove = false;
         var somethingLeftBelow = false;
+        var extractOutlinks = this.actions[k].extractOutlinks ? this.actions[k].extractOutlinks : false;
+        var extractOutlinksInLoop = this.actions[k].extractOutlinksInLoop ? this.actions[k].extractOutlinksInLoop : false;
 
         var documents = [];
         documents[0] = document;
@@ -121,6 +124,12 @@ class UmbraBehavior {
                             childSelectors = documents[j].querySelectorAll(childSelector);
                         }
                     }
+                    // If we were asked to extract outlinks within the loop,
+                    // repeat this every time we iterate
+                    if (extractOutlinks && extractOutlinksInLoop) {
+                        this.outlinks = this.outlinks.concat(__brzl_compileOutlinks(window));
+                    }
+
                     didSomething = true;
                     break;
                 } else if (where > 0) {
@@ -136,6 +145,12 @@ class UmbraBehavior {
             }
         }
 
+        // If we were asked not to extract outlinks within the loop,
+        // do this once after we've finished looping
+        if (extractOutlinks && !extractOutlinksInLoop) {
+            this.outlinks = this.outlinks.concat(__brzl_compileOutlinks(window));
+        }
+
         if (!didSomething) {
             if (somethingLeftBelow || ( (window.scrollY + window.innerHeight) < document.documentElement.scrollHeight)) {
                 window.scrollBy(0, 200);
@@ -221,7 +236,12 @@ var umbraBehavior = new UmbraBehavior( {{actions|json}} );
 
 // Called from outside of this script.
 var umbraBehaviorFinished = function() {
-    return umbraBehavior.isFinished();
+    let outlinkStrings = umbraBehavior.outlinks.map(el => el.toString());
+
+    return {
+        finished: umbraBehavior.isFinished(),
+        outlinks: Array.from(outlinkStrings),
+    };
 };
 
 umbraBehavior.start();