Merge pull request #4971 from sgibson91/cilogon-cleanup

Cilogon cleanup

Author: sgibson91

deployer/README.md

@@ -369,9 +369,9 @@ This allows us to validate that all required values are present and have the cor
 ### The `cilogon-client` sub-command for CILogon OAuth client management
 Deployer sub-command for managing CILogon clients for 2i2c hubs.
 
-#### `cilogon-client create/delete/get/get-all/update`
+#### `cilogon-client create/delete/get/get-all/update/cleanup`
 
-create/delete/get/get-all/update/ CILogon clients using the 2i2c administrative client provided by CILogon.
+create/delete/get/get-all/update/cleanup CILogon clients using the 2i2c administrative client provided by CILogon.
 
 ### The `exec` sub-command for executing shells and debugging commands
 

deployer/commands/cilogon.py

@@ -13,10 +13,12 @@
 - `delete` a CILogon client application when a hub is removed or changes auth methods
 - `get` details about an existing hub CILogon client
 - `get-all` existing 2i2c CILogon client applications
+- `cleanup` duplicated CILogon applications
 """
 
 import base64
 import json
+from collections import Counter
 from pathlib import Path
 
 import requests
@@ -27,6 +29,8 @@
 from deployer.cli_app import cilogon_client_app
 from deployer.utils.file_acquisition import (
     build_absolute_path_to_hub_encrypted_config_file,
+    find_absolute_path_to_cluster_file,
+    get_cluster_names_list,
     get_decrypted_file,
     persist_config_in_encrypted_file,
     remove_jupyterhub_hub_config_key_from_encrypted_file,
@@ -264,52 +268,19 @@ def get_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None):
     return client_details
 
 
-def delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None):
-    """Deletes the client associated with the id.
-
-    Args:
-        id (str): Id of the client to delete
+def delete_client(admin_id, admin_secret, client_id=None):
+    """Deletes a CILogon client.
 
     Returns status code if response.ok
     or None if the `delete` request returned a status code not in the range 200-299.
 
     See: https://github.com/ncsa/OA4MP/blob/HEAD/oa4mp-server-admin-oauth2/src/main/scripts/oidc-cm-scripts/cm-delete.sh
     """
-    config_filename = build_absolute_path_to_hub_encrypted_config_file(
-        cluster_name, hub_name
-    )
-
-    if not client_id:
-        if Path(config_filename).is_file():
-            client_id = load_client_id_from_file(config_filename)
-            # Nothing to do if no client has been found
-            if not client_id:
-                print_colour(
-                    "No `client_id` to delete was provided and couldn't find any in `config_filename`",
-                    "red",
-                )
-                return
-        else:
-            print_colour(
-                f"No `client_id` to delete was provided and couldn't find any {config_filename} file",
-                "red",
-            )
-            return
+    if client_id is None:
+        print("Deleting a CILogon client for unknown ID")
     else:
-        if not stored_client_id_same_with_cilogon_records(
-            admin_id,
-            admin_secret,
-            cluster_name,
-            hub_name,
-            client_id,
-        ):
-            print_colour(
-                "CILogon records are different than the client app stored in the configuration file. Consider updating the file.",
-                "red",
-            )
-            return
+        print(f"Deleting the CILogon client details for {client_id}...")
 
-    print(f"Deleting the CILogon client details for {client_id}...")
     headers = build_request_headers(admin_id, admin_secret)
     response = requests.delete(build_request_url(client_id), headers=headers, timeout=5)
     if not response.ok:
@@ -318,19 +289,6 @@ def delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None
 
     print_colour("Done!")
 
-    # Delete client credentials from config file also if file exists
-    if Path(config_filename).is_file():
-        print(f"Deleting the CILogon client details from the {config_filename} also...")
-        key = "CILogonOAuthenticator"
-        try:
-            remove_jupyterhub_hub_config_key_from_encrypted_file(config_filename, key)
-        except KeyError:
-            print_colour(f"No {key} found to delete from {config_filename}", "yellow")
-            return
-        print_colour(f"CILogonAuthenticator config removed from {config_filename}")
-        if not Path(config_filename).is_file():
-            print_colour(f"Empty {config_filename} file also deleted.", "yellow")
-
 
 def get_all_clients(admin_id, admin_secret):
     print("Getting all existing CILogon client applications...")
@@ -344,8 +302,67 @@ def get_all_clients(admin_id, admin_secret):
         return
 
     clients = response.json()
-    for c in clients["clients"]:
-        print(c)
+    return [c for c in clients["clients"]]
+
+
+def find_duplicated_clients(clients):
+    """Determine duplicated CILogon clients by comparing client names
+
+    Args:
+        clients (list[dict]): A list of dictionaries containing information about
+            the existing CILogon clients. Generated by get_all_clients function.
+
+    Returns:
+        list: A list of duplicated client names
+    """
+    client_names = [c["name"] for c in clients]
+    client_names_count = Counter(client_names)
+    return [k for k, v in client_names_count.items() if v > 1]
+
+
+def find_orphaned_clients(clients):
+    """Find CILogon clients for which an associated cluster or hub no longer
+    exists and can safely be deleted.
+
+    Args:
+        clients (list[dict]): A list of existing CILogon client info
+
+    Returns:
+        list[dict]: A list of 'orphaned' CILogon clients which don't have an
+            associated cluster or hub, which can be deleted
+    """
+    clients_to_be_deleted = []
+    clusters = get_cluster_names_list()
+
+    for client in clients:
+        cluster = next((cl for cl in clusters if cl in client["name"]), "")
+
+        if cluster:
+            cluster_config_file = find_absolute_path_to_cluster_file(cluster)
+            with open(cluster_config_file) as f:
+                cluster_config = yaml.load(f)
+
+            hub = next(
+                (
+                    hub["name"]
+                    for hub in cluster_config["hubs"]
+                    if hub["name"] in client["name"]
+                ),
+                "",
+            )
+
+            if not hub:
+                print(
+                    f"A hub pertaining to client {client['name']} does NOT exist. Marking client for deletion."
+                )
+                clients_to_be_deleted.append(client)
+        else:
+            print(
+                f"A cluster pertaining to client {client['name']} does NOT exist. Marking client for deletion."
+            )
+            clients_to_be_deleted.append(client)
+
+    return clients_to_be_deleted
 
 
 def get_2i2c_cilogon_admin_credentials():
@@ -415,7 +432,13 @@ def get(
 def get_all():
     """Retrieve details about all existing 2i2c CILogon clients."""
     admin_id, admin_secret = get_2i2c_cilogon_admin_credentials()
-    get_all_clients(admin_id, admin_secret)
+    clients = get_all_clients(admin_id, admin_secret)
+    for c in clients:
+        print(c)
+
+    # Our plan with CILogon only permits 100 clients, so provide feedback on that
+    # number here. Change this if our plan updates.
+    print_colour(f"{len(clients)} / 100 clients used", "yellow")
 
 
 @cilogon_client_app.command()
@@ -432,7 +455,123 @@ def delete(
         """,
     ),
 ):
-    """Delete an existing CILogon client. This deletes both the CILogon client application,
-    and the client credentials from the configuration file."""
+    """
+    Delete an existing CILogon client. This deletes both the CILogon client application,
+    and the client credentials from the configuration file.
+    """
+    config_filename = build_absolute_path_to_hub_encrypted_config_file(
+        cluster_name, hub_name
+    )
     admin_id, admin_secret = get_2i2c_cilogon_admin_credentials()
+
+    if not client_id:
+        if Path(config_filename).is_file():
+            client_id = load_client_id_from_file(config_filename)
+            # Nothing to do if no client has been found
+            if not client_id:
+                print_colour(
+                    "No `client_id` to delete was provided and couldn't find any in `config_filename`",
+                    "red",
+                )
+                return
+        else:
+            print_colour(
+                f"No `client_id` to delete was provided and couldn't find any {config_filename} file",
+                "red",
+            )
+            return
+    else:
+        if not stored_client_id_same_with_cilogon_records(
+            admin_id,
+            admin_secret,
+            cluster_name,
+            hub_name,
+            client_id,
+        ):
+            print_colour(
+                "CILogon records are different than the client app stored in the configuration file. Consider updating the file.",
+                "red",
+            )
+            return
+
     delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id)
+
+    # Delete client credentials from config file also if file exists
+    if Path(config_filename).is_file():
+        print(f"Deleting the CILogon client details from the {config_filename} also...")
+        key = "CILogonOAuthenticator"
+        try:
+            remove_jupyterhub_hub_config_key_from_encrypted_file(config_filename, key)
+        except KeyError:
+            print_colour(f"No {key} found to delete from {config_filename}", "yellow")
+            return
+        print_colour(f"CILogonAuthenticator config removed from {config_filename}")
+        if not Path(config_filename).is_file():
+            print_colour(f"Empty {config_filename} file also deleted.", "yellow")
+
+
+@cilogon_client_app.command()
+def cleanup(
+    delete: bool = typer.Option(
+        False, help="Proceed with deleting duplicated CILogon apps"
+    )
+):
+    """Identify duplicated CILogon clients and which ID is being actively used in config,
+    and optionally delete unused duplicates.
+
+    Args:
+        delete (bool, optional): Delete unused duplicate CILogon apps. Defaults to False.
+    """
+    clients_to_be_deleted = []
+
+    admin_id, admin_secret = get_2i2c_cilogon_admin_credentials()
+    clients = get_all_clients(admin_id, admin_secret)
+    duplicated_clients = find_duplicated_clients(clients)
+
+    # Cycle over each duplicated client name
+    for duped_client in duplicated_clients:
+        # Finds all the client IDs associated with a duplicated name
+        ids = [c["client_id"] for c in clients if c["name"] == duped_client]
+
+        # Establish the cluster and hub name from the client name and build the
+        # absolute path to the encrypted hub values file
+        cluster_name, hub_name = duped_client.split("-")
+        config_filename = build_absolute_path_to_hub_encrypted_config_file(
+            cluster_name, hub_name
+        )
+
+        with get_decrypted_file(config_filename) as decrypted_path:
+            with open(decrypted_path) as f:
+                secret_config = yaml.load(f)
+
+        if (
+            "CILogonOAuthenticator"
+            not in secret_config["jupyterhub"]["hub"]["config"].keys()
+        ):
+            print(
+                f"Hub {hub_name} on cluster {cluster_name} doesn't use CILogonOAuthenticator."
+            )
+        else:
+            # Extract the client ID *currently in use* from the encrypted config and remove it from the list of IDs
+            config_client_id = secret_config["jupyterhub"]["hub"]["config"][
+                "CILogonOAuthenticator"
+            ]["client_id"]
+            ids.remove(config_client_id)
+
+        clients_to_be_deleted.extend(
+            [{"client_name": duped_client, "client_id": id} for id in ids]
+        )
+
+        # Remove the duplicated clients from the client list
+        clients = [c for c in clients if c["name"] != duped_client]
+
+    orphaned_clients = find_orphaned_clients(clients)
+    clients_to_be_deleted.extend(orphaned_clients)
+
+    print_colour("CILogon clients to be deleted...")
+    for c in clients_to_be_deleted:
+        print(c)
+
+    if delete:
+        for c in clients_to_be_deleted:
+            delete_client(admin_id, admin_secret, client_id=c["client_id"])

deployer/utils/file_acquisition.py

@@ -248,3 +248,12 @@ def get_all_cluster_yaml_files():
         for path in CONFIG_CLUSTERS_PATH.glob("**/cluster.yaml")
         if "templates" not in path.as_posix()
     }
+
+
+def get_cluster_names_list():
+    """
+    Returns a list of all the clusters currently listed under config/clusters
+    """
+    return [
+        d.name for d, _, _ in CONFIG_CLUSTERS_PATH.walk() if "templates" not in str(d)
+    ]