wip: allocate-torrents

Author: chapmanjacobd

pdm.lock

@@ -50,9 +50,11 @@ deluxe = [
   "PyExifTool",
   "pymcdm",
   "pyvirtualdisplay",
+  "qbittorrent-api",
   "scikit-learn",
   "selenium-wire",
   "subliminal",
+  "torrentool",
   "tqdm",
   "wordllama>=0.2.7.post0",
   "xattr",

pyproject.toml

@@ -24,7 +24,8 @@
         "nicotine_import": "Import paths from nicotine+",
         "places_import": "Import places of interest (POIs)",
         "row_add": "Add arbitrary data to SQLite",
-        "computer_add": "Add computer info to SQLite",
+        "computers_add": "Add computer info to SQLite",
+        "torrents_add": "Add torrent info to SQLite",
     },
     "Text subcommands": {
         "cluster_sort": "Sort text and images by similarity",
@@ -73,10 +74,13 @@
         "process_text": "Shrink documents to HTML+AV1 image format (requires Calibre)",
         "images_to_pdf": "Convert folders of images into image PDFs",
         "pdf_edit": "Apply brightness, contrast, saturation, and sharpness adjustments to PDFs",
+        "torrents_start": "Start torrents (qBittorrent-nox)",
+        "torrents_stop": "Stop torrents (qBittorrent-nox)",
     },
     "Multi-database subcommands": {
         "merge_dbs": "Merge SQLite databases",
         "copy_play_counts": "Copy play history",
+        "allocate_torrents": "Use computers.db and torrents.db to allocate torrents",
     },
     "Filesystem Database subcommands": {
         "disk_usage": "Show disk usage",
@@ -121,6 +125,7 @@
         "gallery_update": "Update online gallery media",
         "links_update": "Update a link-scraping database",
         "reddit_update": "Update reddit media",
+        "computers_update": "Update computer stats",
     },
     "Misc subcommands": {
         "export_text": "Export HTML files from SQLite databases",
@@ -152,9 +157,10 @@ def print_help(parser) -> None:
 
 
 modules = {
-    "xklb.createdb.computer_add.computer_add": ["pc-add", "ssh-add"],
-    "xklb.createdb.fs_add.fs_add": ["x", "extract"],
-    "xklb.createdb.fs_add.fs_update": ["xu"],
+    "xklb.createdb.computers_add.computers_add": ["computer-add", "pc-add", "ssh-add"],
+    "xklb.createdb.computers_add.computers_update": ["computer-update", "pc-update", "ssh-update"],
+    "xklb.createdb.fs_add.fs_add": ["filesystem-add", "x", "extract"],
+    "xklb.createdb.fs_add.fs_update": ["filesystem-update", "xu"],
     "xklb.createdb.gallery_add.gallery_add": ["gdl-add", "ga"],
     "xklb.createdb.gallery_add.gallery_update": ["gdl-update", "gu"],
     "xklb.createdb.hn_add.hacker_news_add": ["hn-add"],
@@ -171,6 +177,7 @@ def print_help(parser) -> None:
     "xklb.createdb.tabs_add.tabs_add": [],
     "xklb.createdb.tabs_add.tabs_shuffle": [],
     "xklb.createdb.tildes.tildes": [],
+    "xklb.createdb.torrents_add.torrents_add": ["torrent-add"],
     "xklb.createdb.tube_add.tube_add": ["ta", "dladd", "da"],
     "xklb.createdb.tube_add.tube_update": ["dlupdate", "tu"],
     "xklb.createdb.web_add.web_add": ["web-dir-add"],
@@ -218,10 +225,13 @@ def print_help(parser) -> None:
     "xklb.mediafiles.process_text.process_text": ["text-process"],
     "xklb.mediafiles.images_to_pdf.images_to_pdf": ["images2pdf"],
     "xklb.mediafiles.pdf_edit.pdf_edit": [],
+    "xklb.mediafiles.torrents_start.torrents_start": ["torrent-start"],
+    "xklb.mediafiles.torrents_stop.torrents_stop": ["torrent-stop"],
     "xklb.misc.dedupe_czkawka.czkawka_dedupe": ["dedupe-czkawka"],
     "xklb.misc.export_text.export_text": [],
     "xklb.multidb.copy_play_counts.copy_play_counts": [],
     "xklb.multidb.merge_dbs.merge_dbs": ["merge-db"],
+    "xklb.multidb.allocate_torrents.allocate_torrents": [],
     "xklb.playback.links_open.links_open": ["open-links"],
     "xklb.playback.play_actions.media": [],
     "xklb.playback.play_actions.filesystem": ["fs", "open"],

tests/createdb/test_computer_add.py

@@ -52,7 +52,7 @@ def get_uptime_info():
 
 
 def get_system_name():
-    ignore_text = ['To Be Filled By O.E.M.']
+    ignore_text = ["To Be Filled By O.E.M."]
     info_strings = []
 
     if IS_LINUX:
@@ -217,21 +217,45 @@ def get_mem_info():
     }
 
 
+def find_mount_point(path):
+    if not os.path.islink(path):
+        path = os.path.abspath(path)
+    elif os.path.islink(path) and os.path.lexists(os.readlink(path)):
+        path = os.path.realpath(path)
+
+    while not os.path.ismount(path):
+        path = os.path.dirname(path)
+        if os.path.islink(path) and os.path.lexists(os.readlink(path)):
+            path = os.path.realpath(path)
+    return path
+
+
+def dev_name(path):
+    dev = os.stat(path).st_dev
+    major, minor = os.major(dev), os.minor(dev)
+    link = os.readlink(f"/sys/dev/block/{major}:{minor}")
+    device = os.path.basename(link)
+    return device
+
+
 def get_mounts():
     disk_io = psutil.disk_io_counters(perdisk=True)
 
     mounts = {}
     for partition in psutil.disk_partitions():
-        if partition.mountpoint == os.sep or partition.mountpoint.startswith(("/boot",)):
+        if partition.mountpoint in [os.sep, "/var", "/etc", "/usr"] or partition.mountpoint.startswith(
+            ("/boot", "/sysroot")
+        ):
             continue
 
         try:
             usage = psutil.disk_usage(partition.mountpoint)
         except PermissionError:  # CD-ROM, etc
+            print(f"PermissionError: Skipping {partition.mountpoint}", file=sys.stderr)
             continue
         else:
             mapped_device = (
-                os.path.realpath(partition.device) if partition.device.startswith('/dev/mapper/') else partition.device
+                os.path.realpath(partition.device) if partition.device.startswith("/dev/mapper/") else partition.device
             )
             device_key = mapped_device.replace("\\", "/").split("/")[-1]  # without /dev/ prefix
             io = disk_io[device_key]

tests/createdb/test_computers_add.py

@@ -0,0 +1,97 @@
+import concurrent.futures, json, os
+from pathlib import Path
+
+import paramiko
+
+from xklb import usage
+from xklb.mediadb import db_playlists
+from xklb.utils import arggroups, argparse_utils, consts, db_utils, objects, remote_processes
+from xklb.utils.log_utils import log
+
+
+def parse_args(action, usage):
+    parser = argparse_utils.ArgumentParser(usage=usage)
+    arggroups.debug(parser)
+
+    arggroups.database(parser)
+    if action == consts.SC.computers_add:
+        parser.add_argument("hostnames", nargs="+", help="List of hostnames to connect to")
+    args = parser.parse_args()
+    arggroups.args_post(args, parser, create_db=True)
+    return args
+
+
+def gather_system_info(hostname):
+    with paramiko.SSHClient() as ssh:
+        ssh.load_system_host_keys()
+        ssh.set_missing_host_key_policy(paramiko.WarningPolicy())
+        ssh.connect(hostname)
+
+        tempdir = remote_processes.ssh_tempdir(ssh) or "."
+        xklb_dir = os.path.join(tempdir, "xklb")
+        r = remote_processes.cmd(ssh, "mkdir", "-p", xklb_dir)
+
+        with ssh.open_sftp() as sftp:
+            src = Path(__file__).parent / "computer_info.py"
+            dst = Path(xklb_dir) / "computer_info.py"
+            sftp.put(bytes(src), bytes(dst))
+
+        r = remote_processes.cmd(ssh, "python3", dst)
+        data = json.loads(r.stdout)
+        return data
+    raise RuntimeError
+
+
+def log_warning_if_same_free_space(computer_info, disks):
+    seen_free_spaces = set()
+    for i, disk in enumerate(disks):
+        free_space = disk["free"]
+        if free_space in seen_free_spaces:
+            log.warning(
+                "%s mount %s has the same free space as another disk! You should open a ticket with this info: %s",
+                computer_info["path"],
+                disk["path"],
+                disks,
+            )
+        else:
+            seen_free_spaces.add(free_space)
+
+
+def computer_add(args, hostnames):
+    with concurrent.futures.ThreadPoolExecutor() as ex:
+        future_to_hostname = {ex.submit(gather_system_info, hostname): hostname for hostname in hostnames}
+
+        for future in concurrent.futures.as_completed(future_to_hostname):
+            hostname = future_to_hostname[future]
+            try:
+                computer_info = future.result()
+            except Exception:
+                log.exception(hostname)
+                if args.verbose >= consts.LOG_DEBUG:
+                    raise
+            else:
+                disks = computer_info.pop("disks")
+                log.debug(computer_info)
+
+                log_warning_if_same_free_space(computer_info, disks)
+
+                computer_info["path"] = hostname
+                playlists_id = db_playlists._add(args, objects.dict_filter_bool(computer_info))
+                for disk in disks:
+                    disk = disk | {"playlists_id": playlists_id}
+                    args.db["media"].insert(disk, pk=["playlists_id", "path"], alter=True, replace=True)
+
+    if not args.db["media"].detect_fts():
+        db_utils.optimize(args)
+
+
+def computers_add():
+    args = parse_args(consts.SC.computers_add, usage.computers_add)
+
+    computer_add(args, args.hostnames)
+
+
+def computers_update():
+    args = parse_args(consts.SC.computers_update, usage.computers_update)
+
+    computer_add(args, [d["path"] for d in args.db.query("select path from playlists")])