Merge pull request #17 from GDC-ConsumerEdge/cli

feat: Add a cli to run cluster health validator locally

Author: benfogel

CODEOWNERS

@@ -1,2 +1,2 @@
 # Root code owner
-* @mike-ensor
+* @mike-ensor @benfogel

README.md

@@ -2,10 +2,14 @@
 
 The cluster health validator is a service that runs in-cluster and reports an
 aggregated signal of platform and workload health. The health is reported both
-as a status as a Kubernetes and asa prometheus metric. This can be used during
+as a status as a Kubernetes and as a prometheus metric. This can be used during
 cluster provisioning to signal to completion of the pre-staging process or as a
 continual sanity check of the state of a cluster.
 
+Alternatively, the cluster health validator can run locally, useful for local
+troubleshooting or to use during the cluster provisioning process without
+requiring an in-cluster component. 
+
 ## Installation
 
 This project uses a CRD and operator, and requires Cluster-Level access. The project can be deployed as a `RootSync` config-sync object with the following configuration. NOTE: Production use should clone the repo, make it private and use the `token` appraoch to authentiate to private repo.
@@ -86,3 +90,47 @@ IMAGE_TAG=gcr.io/${PROJECT_ID}/cluster-health-validator:1.0.0
 docker build -t ${IMAGE_TAG} .
 docker push ${IMAGE_TAG}
 ```
+
+## Local Usage
+
+```
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r app/requirements.txt
+
+python3 app --help
+usage: app [-h] [--health-check HEALTH_CHECK [HEALTH_CHECK ...]] [-v | -q] [-w] [-i INTERVAL] [-t TIMEOUT]
+
+options:
+  -h, --help            show this help message and exit
+  --health-check HEALTH_CHECK [HEALTH_CHECK ...]
+                        Set a health check to perform. For health checks requiring parameters, pass them in a key=value format as additional arguments. Example: --health-check
+                        checkvirtualmachines namespace=vm-workloads count=3
+  -v, --verbose         increase output verbosity; -vv for max verbosity
+  -q, --quiet           output errors only
+  -w, --wait            wait for health checks to pass before exiting
+  -i INTERVAL, --interval INTERVAL
+                        interval to poll passing health checks
+  -t TIMEOUT, --timeout TIMEOUT
+                        Overall timeout for health checks to pass
+```
+
+Examples:
+
+```
+# Run the default health checks (CheckNodes, CheckRootSyncs, CheckRobinCluster)
+python3 app
+
+# Run customized health checks
+python3 app --health-check checknodes \
+            --health-check checkrobincluster \
+            --health-check checkrootsyncs \
+            --health-check checkgooglegrouprbac \
+            --health-check checkvirtualmachines namespace=vm-workloads count=3 \
+            --health-check checkdatavolumes namespace=vm-workloads count=3
+
+# Run default health checks and wait until all health checks pass.
+#   Timeout after 1 hour if health checks don't pass
+python3 app --wait --interval 60 --timeout 3600
+
+```

app/__main__.py

@@ -0,0 +1,152 @@
+import argparse
+import logging
+import sys
+import time
+from check_data_volumes import CheckDataVolumes
+from check_google_group_rbac import CheckGoogleGroupRBAC
+from check_nodes import CheckNodes
+from check_robin_cluster import CheckRobinCluster
+from check_root_syncs import CheckRootSyncs
+from check_virtual_machines import CheckVirtualMachines
+from check_vmruntime import CheckVMRuntime
+from kubernetes import config
+
+health_check_map = {
+    CheckGoogleGroupRBAC.__name__.lower(): CheckGoogleGroupRBAC,
+    CheckNodes.__name__.lower(): CheckNodes,
+    CheckRobinCluster.__name__.lower(): CheckRobinCluster,
+    CheckRootSyncs.__name__.lower(): CheckRootSyncs,
+    CheckVMRuntime.__name__.lower(): CheckVMRuntime,
+    CheckDataVolumes.__name__.lower(): CheckDataVolumes,
+    CheckVirtualMachines.__name__.lower(): CheckVirtualMachines,
+}
+
+default_health_checks = [
+    CheckNodes.__name__.lower(),
+    CheckRobinCluster.__name__.lower(),
+    CheckRootSyncs.__name__.lower()
+]
+
+config.load_config()
+logging.basicConfig(stream=sys.stdout)
+logger = logging.getLogger('main')
+
+def run_health_checks(args):
+    checks = []
+
+    if args.health_check is None:
+        # use default health checks
+        logger.info('No health checks specified, using default health checks: ' + ', '.join(default_health_checks))
+        checks = [health_check_map[check_name]() for check_name in default_health_checks]
+    else:
+        for health_check in args.health_check:
+            if len(health_check) == 0:
+                logger.error('No health check specified')
+                return 1
+
+            check_name = health_check[0].lower()
+
+            if check_name not in health_check_map:
+                logger.error('Unknown health check specified: ' + health_check)
+                return 1
+            
+             
+            if len(health_check) > 1:
+                # Health check includes named parameters that need to be passed
+                check_args = {}
+
+                for parameter in health_check[1:]:
+                    if "=" not in parameter:
+                        logger.error('Invalid parameter specified: ' + parameter + '. Parameters must be in the format key=value')
+                        return 1
+
+                    key, value = parameter.split("=")
+                    check_args[key] = value
+
+                checks.append(health_check_map[check_name](check_args))
+            else:
+                checks.append(health_check_map[check_name]())
+
+    failed_health_checks = []
+
+    for check in checks:
+        try:
+            if not check.is_healthy():
+                failed_health_checks.append(check.__class__.__name__)
+        except Exception:
+            failed_health_checks.append(check.__class__.__name__)
+
+
+    if len(failed_health_checks) > 0:
+        for failure in failed_health_checks:
+            logger.error('Health check failed: ' + failure)
+        return 1
+    
+    logger.info('All health checks passed!')
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        '--health-check',
+        action='append',
+        help='''Set a health check to perform.
+                For health checks requiring parameters, pass them in a key=value format as additional arguments.
+                Example: --health-check checkvirtualmachines namespace=vm-workloads count=3''',
+        nargs='+')
+    verbosity_mutex = parser.add_mutually_exclusive_group()
+    verbosity_mutex.add_argument(
+        '-v', '--verbose',
+        action='count',
+        default=0,
+        help='increase output verbosity; -vv for max verbosity')
+    verbosity_mutex.add_argument(
+        '-q', '--quiet',
+        action='store_true',
+        help='output errors only')
+
+    parser.add_argument(
+        '-w', '--wait',
+        action='store_true',
+        help='wait for health checks to pass before exiting')
+
+    parser.add_argument(
+        '-i', '--interval',
+        type=int,
+        default=60,
+        help='interval to poll passing health checks')
+
+    parser.add_argument(
+        '-t', '--timeout',
+        type=int,
+        default=3600,
+        help='Overall timeout for health checks to pass')
+
+    args = parser.parse_args()
+    if args.quiet:
+        logger.setLevel(logging.ERROR)
+    elif args.verbose == 1:
+        logger.setLevel(logging.INFO)
+    elif args.verbose >= 2:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.WARNING)
+
+    if (args.wait):
+        # Poll continuously unless all health checks pass
+        max_loops = int(args.timeout / args.interval)
+        for i in range(max_loops):
+            if run_health_checks(args) == 0:
+                return 0
+
+            time.sleep(args.interval)
+
+        logger.error('Timed out waiting for health checks to pass')
+        return 1
+    else:
+        return run_health_checks(args)
+
+if __name__ == '__main__':
+    sys.exit(main())