Adding collector providing the total of vCPU of all workers

README.md

@@ -93,6 +93,7 @@ For more flexibility, use:
   - `METRICS_UTILITY_CRC_INGRESS_URL`
   - `METRICS_UTILITY_CRC_SSO_URL`
   - `METRICS_UTILITY_OPTIONAL_CCSP_REPORT_SHEETS`
+  - `METRICS_UTILITY_MANDATORY_COLLECTORS`
   - `METRICS_UTILITY_OPTIONAL_COLLECTORS`
   - `METRICS_UTILITY_ORGANIZATION_FILTER`
   - `METRICS_UTILITY_PRICE_PER_NODE`

metrics_utility/automation_controller_billing/collectors.py

@@ -1,8 +1,11 @@
 import json
+import logging
 import os
 import os.path
 import platform
 
+from datetime import datetime, timezone
+
 import distro
 
 from awx.conf.license import get_license
@@ -11,9 +14,15 @@
 from django.db import connection
 from django.utils.timezone import now, timedelta
 from django.utils.translation import gettext_lazy as _
+from kubernetes import client
+from kubernetes import config as kube_config
 
 from metrics_utility.base import CsvFileSplitter, register
+from metrics_utility.exceptions import MetricsException, MissingRequiredEnvVar
+
 
+logging.basicConfig(format='%(asctime)s(+%(relativeCreated)d): %(message)s', level=logging.WARNING)
+logger = logging.getLogger(__name__)
 
 """
 This module is used to define metrics collected by
@@ -33,6 +42,10 @@ def something(since):
 """
 
 
+def get_mandatory_collectors():
+    return os.environ.get('METRICS_UTILITY_MANDATORY_COLLECTORS', 'job_host_summary').split(',')
+
+
 def get_optional_collectors():
     return os.environ.get('METRICS_UTILITY_OPTIONAL_COLLECTORS', 'main_jobevent').split(',')
 
@@ -205,6 +218,9 @@ def yaml_and_json_parsing_functions():
 
 @register('job_host_summary', '1.2', format='csv', description=_('Data for billing'), fnc_slicing=daily_slicing)
 def job_host_summary_table(since, full_path, until, **kwargs):
+    if 'job_host_summary' not in get_mandatory_collectors():
+        return None
+
     # TODO: controler needs to have an index on main_jobhostsummary.modified
     prepend_query = """
         -- Define function for parsing field out of yaml encoded as text
@@ -464,3 +480,56 @@ def main_host_table(since, full_path, until, **kwargs):
     return _copy_table(
         table='main_host', query=f'COPY ({query}) TO STDOUT WITH CSV HEADER', path=full_path, prepend_query=yaml_and_json_parsing_functions()
     )
+
+
+@register('total_workers_vcpu', '1.0', format='json', description=_('Total workers vCPU'))
+def total_workers_vcpu(since, full_path, until, **kwargs):
+    if 'total_workers_vcpu' not in get_optional_collectors():
+        return None
+
+    cluster_name = os.environ.get('METRICS_UTILITY_ANSIBLE_SAAS_CLUSTER_NAME')
+    if not cluster_name:
+        logger.error('environment variable METRICS_UTILITY_ANSIBLE_SAAS_CLUSTER_NAME is not set')
+        raise MissingRequiredEnvVar('environment variable METRICS_UTILITY_ANSIBLE_SAAS_CLUSTER_NAME is not set')
+
+    now = datetime.now(timezone.utc)
+
+    info = {'cluster_name': cluster_name, 'timestamp': now.isoformat(), 'nodes': []}
+
+    # If METRICS_UTILITY_VCPU_COUNT_ENABLED is not set or not set to true then it returns 1
+    vcpu_count_enabled_str = os.environ.get('METRICS_UTILITY_VCPU_COUNT_ENABLED')
+    vcpu_count_enabled = False
+    if vcpu_count_enabled_str and (vcpu_count_enabled_str.lower() == 'true'):
+        vcpu_count_enabled = True
+    if not vcpu_count_enabled:
+        return {'cluster_name': info['cluster_name'], 'total_workers_vcpu': '1'}
+
+    try:
+        kube_config.load_incluster_config()
+    except kube_config.ConfigException:
+        try:
+            kube_config.load_kube_config()
+        except kube_config.ConfigException as e:
+            logger.error(f'Could not configure Kubernetes Python client ERROR: {e}')
+            raise MetricsException(f'Could not configure Kubernetes Python client ERROR: {e}')
+
+    # Create a CoreV1Api client
+    api_instance = client.CoreV1Api()
+
+    nodes = api_instance.list_node()
+
+    total_workers_vcpu = 0
+    for node_info in nodes.items:
+        for resource, value in node_info.status.capacity.items():
+            if resource == 'cpu':
+                info['nodes'].append({node_info.metadata.name: int(value)})
+                total_workers_vcpu += int(value)
+
+    info['total_workers_vcpu'] = total_workers_vcpu
+
+    logger_info = logging.getLogger(__name__)
+    logger_info.setLevel(logging.INFO)
+
+    logger_info.info(json.dumps(info, indent=2))
+
+    return {'cluster_name': info['cluster_name'], 'total_workers_vcpu': info['total_workers_vcpu']}

metrics_utility/base/collection.py

@@ -25,6 +25,7 @@ def __init__(self, collector, fnc_collecting):
         self.version = fnc_collecting.__insights_analytics_version__
 
         self.data_type = fnc_collecting.__insights_analytics_type__
+
         self.filename = f'{self.key}.{self.data_type}'
         # either since/until or full sync(if enabled)
         self.since = None  # set by Collector._create_collections()

metrics_utility/management/validation.py

@@ -45,7 +45,8 @@
         'managed_nodes_by_organizations',
     },
 }
-VALID_COLLECTORS = {'main_host', 'main_jobevent', 'main_indirectmanagednodeaudit'}
+VALID_OPTIONAL_COLLECTORS = {'main_host', 'main_jobevent', 'main_indirectmanagednodeaudit', 'total_workers_vcpu'}
+VALID_MANDATORY_COLLECTORS = {'job_host_summary'}
 VALID_SHIP_TARGET_BUILD = {'directory', 's3', 'controller_db'}
 VALID_SHIP_TARGET_GATHER = {'directory', 's3', 'crc'}
 
@@ -222,18 +223,33 @@ def validate_collectors(errors):
     Environment Variables:
         METRICS_UTILITY_OPTIONAL_COLLECTORS (str, optional): Comma-separated
             list of collector names. Defaults to 'main_jobevent' if not set.
+        METRICS_UTILITY_MANDATORY_COLLECTORS (str, optional): Comma-separated
+            list of collector names. Defaults to 'main_jobevent' if not set.
 
     Notes:
-        - The set of valid collectors is defined by the global variable
-          VALID_COLLECTORS.
+        - The set of valid optional collectors is defined by the global variable
+          VALID_OPTIONAL_COLLECTORS.
+        - The set of valid mandatory collectors is defined by the global variable
+          VALID_MANDATORY_COLLECTORS.
         - Error messages include the invalid collector names and the list of
           valid values.
     """
+    collectors_env_var = os.environ.get('METRICS_UTILITY_MANDATORY_COLLECTORS')
+    if collectors_env_var:
+        collectors = collectors_env_var.split(',')
+        if collectors:
+            invalid = set(collectors) - VALID_MANDATORY_COLLECTORS
+            if invalid:
+                errors.append(
+                    f'Invalid METRICS_UTILITY_MANDATORY_COLLECTORS: \
+                              {", ".join(invalid)}. Valid values: {", ".join(VALID_MANDATORY_COLLECTORS)}'
+                )
+
     collectors = os.environ.get('METRICS_UTILITY_OPTIONAL_COLLECTORS', 'main_jobevent').split(',')
     if collectors:
-        invalid = set(collectors) - VALID_COLLECTORS
+        invalid = set(collectors) - VALID_OPTIONAL_COLLECTORS
         if invalid:
-            errors.append(f'Invalid METRICS_UTILITY_OPTIONAL_COLLECTORS: {", ".join(invalid)}. Valid values: {", ".join(VALID_COLLECTORS)}')
+            errors.append(f'Invalid METRICS_UTILITY_OPTIONAL_COLLECTORS: {", ".join(invalid)}. Valid values: {", ".join(VALID_OPTIONAL_COLLECTORS)}')
 
 
 def validate_ship_target(errors, method):