computers-add: better handling of bind mounts

Author: chapmanjacobd

library/createdb/computers_add.py

@@ -13,9 +13,12 @@ def parse_args(action, usage):
 
     arggroups.database(parser)
     if action == consts.SC.computers_add:
+        parser.add_argument("--ignore-mounts", nargs="+", default=[], help="List of mountpoints to ignore")
         parser.add_argument("hostnames", nargs="+", help="List of hostnames to connect to")
     args = parser.parse_args()
     arggroups.args_post(args, parser, create_db=True)
+
+    args.ignore_mounts = [s.rstrip(os.sep) for s in args.ignore_mounts]
     return args
 
 
@@ -44,14 +47,25 @@ def gather_system_info(hostname):
 
 def log_warning_if_same_free_space(computer_info, disks):
     seen_free_spaces = set()
+    seen_devices = set()
     for i, disk in enumerate(disks):
+        device = disk["device"]
+        if device in seen_devices:
+            log.warning(
+                "Skipping already seen device %s... any bind mounts? %s",
+                device,
+                [d for d in disks if d["device"] == device],
+            )
+            continue
+        seen_devices.add(device)
+
         free_space = disk["free"]
         if free_space in seen_free_spaces:
             log.warning(
-                "%s mount %s has the same free space as another disk! You should open a ticket with this info: %s",
-                computer_info["path"],
+                "%s mount %s has the same free space as another disk! You are lucky! Or not and you should open a ticket with this info: %s",
+                computer_info["node"],
                 disk["path"],
-                disks,
+                [d for d in disks if d["free"] == free_space],
             )
         else:
             seen_free_spaces.add(free_space)
@@ -73,6 +87,7 @@ def computer_add(args, hostnames):
                 disks = computer_info.pop("disks")
                 log.debug(computer_info)
 
+                disks = [d for d in disks if d["path"] not in args.ignore_mounts]
                 log_warning_if_same_free_space(computer_info, disks)
 
                 computer_info["path"] = hostname

library/createdb/getty_add.py

@@ -17,32 +17,32 @@ def parse_args():
 
 
 def activity_stream_extract(args, json_data):
-    assert json_data['type'] == 'OrderedCollectionPage'
+    assert json_data["type"] == "OrderedCollectionPage"
 
     data = []
-    if 'orderedItems' in json_data:
-        for item in json_data['orderedItems']:
-            for k in ['id', 'created', 'endTime']:
+    if "orderedItems" in json_data:
+        for item in json_data["orderedItems"]:
+            for k in ["id", "created", "endTime"]:
                 item.pop(k)
-            obj = item.pop('object')
+            obj = item.pop("object")
 
-            type_ = item.pop('type')
-            if type_ == 'Delete':
+            type_ = item.pop("type")
+            if type_ == "Delete":
                 with args.db.conn:
-                    args.db["activity_stream"].delete_where("path = ?", [obj.get('id')])
-            elif type_ == 'Update':
+                    args.db["activity_stream"].delete_where("path = ?", [obj.get("id")])
+            elif type_ == "Update":
                 continue  # TODO: implement in-band Update mechanism
-            elif type_ not in ['Create']:
+            elif type_ not in ["Create"]:
                 raise
 
             obj_info = {
-                'path': obj.get('id'),
-                'type': obj.get('type'),
-                **{k: v for k, v in obj.items() if k not in ['id', 'type']},
+                "path": obj.get("id"),
+                "type": obj.get("type"),
+                **{k: v for k, v in obj.items() if k not in ["id", "type"]},
             }
             data.append(obj_info)
             if item:
-                print('item', item)
+                print("item", item)
 
     else:
         raise
@@ -68,23 +68,24 @@ def activity_stream_fetch(url):
 
     return r.json()
 
+
 def update_activity_stream(args):
-    current_page = int(args.db.pop('select max(page) from activity_stream') or 0) + 1
+    current_page = int(args.db.pop("select max(page) from activity_stream") or 0) + 1
 
     next_page_url = f"https://data.getty.edu/museum/collection/activity-stream/page/{current_page}"
     while next_page_url:
         log.debug("Fetching %s...", next_page_url)
 
         page_data = activity_stream_fetch(next_page_url)
         if page_data:
-            current_page = int(page_data['id'].split('/')[-1])
+            current_page = int(page_data["id"].split("/")[-1])
 
             activities = activity_stream_extract(args, page_data)
             args.db["activity_stream"].insert_all(
                 [{"page": current_page, **activity} for activity in activities], alter=True, replace=True  # pk="id",
             )
 
-            next_page_url = page_data.get('next', {}).get('id')
+            next_page_url = page_data.get("next", {}).get("id")
         else:
             break
 
@@ -94,6 +95,7 @@ def getty_add():
 
     update_activity_stream(args)
 
+
 # https://data.getty.edu/museum/collection/group/ee294bfc-bbe5-42b4-95b2-04872b802bfe
 # https://data.getty.edu/museum/collection/object/08eaed9f-1354-4817-8aed-1db49e893a03
 # https://data.getty.edu/museum/collection/document/37194afd-905c-43df-9f28-baacdd91062a