webtext: --save flag

Author: chapmanjacobd

pdm.lock

@@ -1,4 +1,5 @@
 import argparse, re
+from pathlib import Path
 
 from bs4 import BeautifulSoup, NavigableString
 
@@ -10,6 +11,7 @@
 def parse_args():
     parser = argparse.ArgumentParser(prog="library extract-text", usage=usage.extract_text)
     parser.add_argument("--skip-links", action="store_true")
+    parser.add_argument("--save", action="store_true")
 
     parser.add_argument("--cookies", help="path to a Netscape formatted cookies file")
     parser.add_argument("--cookies-from-browser", metavar="BROWSER[+KEYRING][:PROFILE][::CONTAINER]")
@@ -99,12 +101,21 @@ def extract_text() -> None:
         web.load_selenium(args)
     try:
         for url in arg_utils.gen_paths(args):
+            output_lines = []
             for s in iterables.return_unique(get_text)(args, url):
                 if s is None:
                     break
 
-                printing.pipe_print(s)
+                if args.save:
+                    output_lines.append(s)
+                else:
+                    printing.pipe_print(s)
 
+            if args.save:
+                save_path = web.url_to_local_path(url)
+                Path(save_path).parent.mkdir(exist_ok=True, parents=True)
+                with open(save_path, "w") as f:
+                    f.writelines(s + "\n" for s in output_lines)
     finally:
         if args.selenium:
             web.quit_selenium(args)

xklb/scripts/mining/extract_text.py

@@ -305,7 +305,19 @@ def quit_selenium(args):
             pass
 
 
-def set_output_path(url, output_path, output_prefix, relative, response):
+def url_to_local_path(url):
+    base_path = "."
+    parsed_url = urlparse(url)
+    relative_path = parsed_url.netloc + "/" + parsed_url.path.lstrip("/")
+    base_path = os.path.dirname(relative_path)
+
+    filename = url.split("/")[-1]
+    output_path = os.path.join(base_path, filename)
+    output_path = path_utils.clean_path(output_path.encode())
+    return output_path
+
+
+def gen_output_path_from_response(url, output_path, output_prefix, relative, response):
     if output_path is None:
         content_d = response.headers.get("Content-Disposition")
         if content_d:
@@ -341,7 +353,7 @@ def download_url(
 
     remote_size = nums.safe_int(r.headers.get("Content-Length"))
 
-    output_path = set_output_path(url, output_path, output_prefix, relative, r)
+    output_path = gen_output_path_from_response(url, output_path, output_prefix, relative, r)
     if output_path == ".":
         log.warning("Skipping directory %s", url)
         return