Skip to content

Commit 0ea9ad0

Browse files
AlexSCoreyhimdel
AlexSCorey
authored and
himdel
committed
library.collectors: move collectors to library
* make sure none of them read environment variables anymore * make sure there are no assumptions about files being present somewhere in the filesystem * db= is always a psycopg connection handle copy_table now supports output_dir or output_file, and optional params= for query params Issue: AAP-54476
1 parent 489b1ce commit 0ea9ad0

16 files changed

Lines changed: 1083 additions & 66 deletions

‎metrics_utility/library/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
from . import collectors, dataframes, extractors, instants, package, reports, storage
2+
from .csv_file_splitter import CsvFileSplitter
23
from .utils import last_gather, lock, save_last_gather, tempdir
34

45

56
__all__ = [
7+
'CsvFileSplitter',
68
'collectors',
79
'dataframes',
810
'extractors',

‎metrics_utility/library/collectors.py‎

Lines changed: 0 additions & 66 deletions
This file was deleted.

‎metrics_utility/library/collectors/__init__.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
from .config import config
2+
from .execution_environments import execution_environments
3+
from .job_host_summary import job_host_summary
4+
from .job_host_summary_service import job_host_summary_service
5+
from .main_host import main_host
6+
from .main_indirectmanagednodeaudit import main_indirectmanagednodeaudit
7+
from .main_jobevent import main_jobevent
8+
from .main_jobevent_service import main_jobevent_service
9+
from .total_workers_vcpu import total_workers_vcpu
10+
from .unified_jobs import unified_jobs
11+
12+
13+
__all__ = [
14+
'config',
15+
'execution_environments',
16+
'job_host_summary',
17+
'job_host_summary_service',
18+
'main_host',
19+
'main_indirectmanagednodeaudit',
20+
'main_jobevent',
21+
'main_jobevent_service',
22+
'total_workers_vcpu',
23+
'unified_jobs',
24+
]

‎metrics_utility/library/collectors/config.py‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import json
2+
import os
3+
import platform
4+
5+
from importlib.metadata import PackageNotFoundError, version
6+
7+
import distro
8+
9+
from django.utils.dateparse import parse_datetime
10+
11+
from .util import collector
12+
13+
14+
# controller settings we collect
15+
SETTINGS = [
16+
'AUTHENTICATION_BACKENDS',
17+
'INSTALL_UUID',
18+
'LICENSE',
19+
'LOG_AGGREGATOR_ENABLED',
20+
'LOG_AGGREGATOR_LOGGERS',
21+
'LOG_AGGREGATOR_TYPE',
22+
'PENDO_TRACKING_STATE',
23+
'SUBSCRIPTION_USAGE_MODEL',
24+
'SYSTEM_UUID',
25+
'TOWER_URL_BASE',
26+
]
27+
28+
29+
@collector
30+
def config(*, db=None, billing_provider_params={}):
31+
settings = _get_controller_settings(db, keys=SETTINGS)
32+
license_info = settings.get('LICENSE', {})
33+
34+
return {
35+
# settings
36+
'authentication_backends': settings.get('AUTHENTICATION_BACKENDS'),
37+
'controller_url_base': settings.get('TOWER_URL_BASE'),
38+
'external_logger_enabled': settings.get('LOG_AGGREGATOR_ENABLED'),
39+
'external_logger_type': settings.get('LOG_AGGREGATOR_TYPE'),
40+
'install_uuid': settings.get('INSTALL_UUID'),
41+
'instance_uuid': settings.get('SYSTEM_UUID'),
42+
'logging_aggregators': settings.get('LOG_AGGREGATOR_LOGGERS'),
43+
'pendo_tracking': settings.get('PENDO_TRACKING_STATE'),
44+
'subscription_usage_model': settings.get('SUBSCRIPTION_USAGE_MODEL'),
45+
# license
46+
'account_number': license_info.get('account_number'),
47+
'automated_instances': license_info.get('automated_instances'),
48+
'automated_since': license_info.get('automated_since'),
49+
'compliant': license_info.get('compliant'),
50+
'current_instances': license_info.get('current_instances'),
51+
'date_expired': license_info.get('date_expired'),
52+
'date_warning': license_info.get('date_warning'),
53+
'free_instances': license_info.get('free_instances', 0),
54+
'grace_period_remaining': license_info.get('grace_period_remaining'),
55+
'license_date': license_info.get('license_date'),
56+
'license_expiry': license_info.get('time_remaining', 0),
57+
'license_type': license_info.get('license_type', 'UNLICENSED'),
58+
'pool_id': license_info.get('pool_id'),
59+
'product_name': license_info.get('product_name'),
60+
'satellite': license_info.get('satellite'),
61+
'sku': license_info.get('sku'),
62+
'subscription_id': license_info.get('subscription_id'),
63+
'subscription_name': license_info.get('subscription_name'),
64+
'support_level': license_info.get('support_level'),
65+
'total_licensed_instances': license_info.get('instance_count', 0),
66+
'trial': license_info.get('trial'),
67+
'usage': license_info.get('usage'),
68+
'valid_key': license_info.get('valid_key'),
69+
# versions & config
70+
'billing_provider_params': billing_provider_params,
71+
'controller_version': _get_controller_version(db) or _version('awx'),
72+
'metrics_utility_version': version('metrics-utility'), # version from setup.cfg
73+
'platform': {
74+
'dist': distro.linux_distribution(),
75+
'release': platform.release(),
76+
'system': platform.system(),
77+
'type': _get_install_type(),
78+
},
79+
}
80+
81+
82+
def _version(package):
83+
try:
84+
return version(package)
85+
except PackageNotFoundError:
86+
return None
87+
88+
89+
def _get_install_type():
90+
if os.getenv('container') == 'oci':
91+
return 'openshift'
92+
93+
if os.getenv('KUBERNETES_SERVICE_PORT'):
94+
return 'k8s'
95+
96+
return 'traditional'
97+
98+
99+
def _get_controller_settings(db, keys):
100+
settings = {}
101+
with db.cursor() as cursor:
102+
cursor.execute(f"SELECT key, value FROM conf_setting WHERE key IN ('{"', '".join(keys)}')")
103+
for key, value in cursor.fetchall():
104+
if value:
105+
settings[key] = json.loads(value, object_hook=_datetime_hook)
106+
return settings
107+
108+
109+
def _get_controller_version(db):
110+
"""Get AWX/Controller version from the main_instance DB table."""
111+
sql = """
112+
SELECT version
113+
FROM main_instance
114+
WHERE enabled = true
115+
AND version IS NOT NULL
116+
AND version != ''
117+
ORDER BY last_seen DESC
118+
LIMIT 1
119+
"""
120+
with db.cursor() as cursor:
121+
cursor.execute(sql)
122+
result = cursor.fetchone()
123+
if result and result[0]:
124+
return result[0]
125+
return None
126+
127+
128+
def _datetime_hook(d):
129+
new_d = {}
130+
for key, value in d.items():
131+
try:
132+
new_d[key] = parse_datetime(value)
133+
except TypeError:
134+
new_d[key] = value
135+
return new_d

‎metrics_utility/library/collectors/execution_environments.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
from .util import collector, copy_table
2+
3+
4+
@collector
5+
def execution_environments(*, db=None, output_dir=None):
6+
query = """
7+
SELECT
8+
id,
9+
created,
10+
modified,
11+
description,
12+
image,
13+
managed,
14+
created_by_id,
15+
credential_id,
16+
modified_by_id,
17+
organization_id,
18+
name,
19+
pull
20+
FROM main_executionenvironment
21+
"""
22+
23+
return copy_table(db=db, table='main_executionenvironment', query=query, output_dir=output_dir)

‎metrics_utility/library/collectors/job_host_summary.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
from .util import collector, copy_table
2+
3+
4+
@collector
5+
def job_host_summary(*, db=None, since=None, until=None, output_dir=None):
6+
where = ' AND '.join(
7+
[
8+
f"main_jobhostsummary.modified >= '{since.isoformat()}'",
9+
f"main_jobhostsummary.modified < '{until.isoformat()}'",
10+
]
11+
)
12+
13+
# TODO: controler needs to have an index on main_jobhostsummary.modified
14+
query = f"""
15+
WITH
16+
filtered_hosts AS (
17+
SELECT DISTINCT main_jobhostsummary.host_id
18+
FROM main_jobhostsummary
19+
WHERE {where}
20+
),
21+
hosts_variables AS (
22+
SELECT
23+
filtered_hosts.host_id,
24+
CASE
25+
WHEN (metrics_utility_is_valid_json(main_host.variables))
26+
THEN main_host.variables::jsonb->>'ansible_host'
27+
ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_host' )
28+
END AS ansible_host_variable,
29+
CASE
30+
WHEN (metrics_utility_is_valid_json(main_host.variables))
31+
THEN main_host.variables::jsonb->>'ansible_connection'
32+
ELSE metrics_utility_parse_yaml_field(main_host.variables, 'ansible_connection' )
33+
END AS ansible_connection_variable
34+
FROM filtered_hosts
35+
LEFT JOIN main_host ON main_host.id = filtered_hosts.host_id
36+
)
37+
SELECT
38+
main_jobhostsummary.id,
39+
main_jobhostsummary.created,
40+
main_jobhostsummary.modified,
41+
main_jobhostsummary.host_name,
42+
main_jobhostsummary.host_id as host_remote_id,
43+
hosts_variables.ansible_host_variable,
44+
hosts_variables.ansible_connection_variable,
45+
main_jobhostsummary.changed,
46+
main_jobhostsummary.dark,
47+
main_jobhostsummary.failures,
48+
main_jobhostsummary.ok,
49+
main_jobhostsummary.processed,
50+
main_jobhostsummary.skipped,
51+
main_jobhostsummary.failed,
52+
main_jobhostsummary.ignored,
53+
main_jobhostsummary.rescued,
54+
main_unifiedjob.created AS job_created,
55+
main_jobhostsummary.job_id AS job_remote_id,
56+
main_unifiedjob.unified_job_template_id AS job_template_remote_id,
57+
main_unifiedjob.name AS job_template_name,
58+
main_inventory.id AS inventory_remote_id,
59+
main_inventory.name AS inventory_name,
60+
main_organization.id AS organization_remote_id,
61+
main_organization.name AS organization_name,
62+
main_unifiedjobtemplate_project.id AS project_remote_id,
63+
main_unifiedjobtemplate_project.name AS project_name
64+
FROM main_jobhostsummary
65+
-- connect to main_job, that has connections into inventory and project
66+
LEFT JOIN main_job ON main_jobhostsummary.job_id = main_job.unifiedjob_ptr_id
67+
-- get project name from project_options
68+
LEFT JOIN main_unifiedjobtemplate AS main_unifiedjobtemplate_project ON main_unifiedjobtemplate_project.id = main_job.project_id
69+
-- get inventory name from main_inventory
70+
LEFT JOIN main_inventory ON main_inventory.id = main_job.inventory_id
71+
-- get job name from main_unifiedjob
72+
LEFT JOIN main_unifiedjob ON main_unifiedjob.id = main_jobhostsummary.job_id
73+
-- get organization name from main_organization
74+
LEFT JOIN main_organization ON main_organization.id = main_unifiedjob.organization_id
75+
-- get variables from precomputed hosts_variables
76+
LEFT JOIN hosts_variables ON hosts_variables.host_id = main_jobhostsummary.host_id
77+
WHERE {where}
78+
ORDER BY main_jobhostsummary.modified ASC
79+
"""
80+
81+
return copy_table(db=db, table='main_jobhostsummary', query=query, prepend_query=True, output_dir=output_dir)

0 commit comments

Comments
 (0)