From 6ae8ddc764cc42e27189a1c1c76f7776c1e9398d Mon Sep 17 00:00:00 2001 From: Jorick van der Hoeven Date: Wed, 19 Oct 2022 09:19:10 -0400 Subject: [PATCH 1/3] Add some basic typing to the cli.py file This adds typing hings to the cli.py file to allow us to leverage mypy analysis of the file and identify any type errors. --- henry/cli.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/henry/cli.py b/henry/cli.py index 240d946..0cf8e6d 100755 --- a/henry/cli.py +++ b/henry/cli.py @@ -8,7 +8,7 @@ from henry.modules import fetcher -def main(): +def main() -> None: parser = setup_cli() user_input = parse_input(parser) @@ -19,16 +19,16 @@ def main(): elif user_input.command == "vacuum": vacuum.Vacuum.run(user_input) else: - parser.error() + parser.error("Unable to parse cli commands") -def setup_cli(): +def setup_cli() -> argparse.ArgumentParser: parser = create_parser() setup_subparsers(parser) return parser -def create_parser(): +def create_parser() -> argparse.ArgumentParser: help_file = os.path.join(os.path.dirname(henry.__file__), ".support_files/help.rtf") with open(help_file, "r", encoding="unicode_escape") as myfile: description = myfile.read() @@ -49,14 +49,14 @@ def create_parser(): return parser -def setup_subparsers(parser): +def setup_subparsers(parser: argparse.ArgumentParser) -> None: subparsers = parser.add_subparsers(dest="command", help=argparse.SUPPRESS) setup_pulse_subparser(subparsers) setup_analyze_subparser(subparsers) setup_vacuum_subparser(subparsers) -def setup_pulse_subparser(subparsers): +def setup_pulse_subparser(subparsers: argparse._SubParsersAction) -> None: pulse_parser = subparsers.add_parser( "pulse", help="pulse help", usage="henry pulse [global options]" ) @@ -72,7 +72,7 @@ def setup_pulse_subparser(subparsers): ) -def setup_analyze_subparser(subparsers): +def setup_analyze_subparser(subparsers: argparse._SubParsersAction) -> None: analyze_parser = subparsers.add_parser( "analyze", help="analyze help", usage="henry analyze" ) @@ -165,7 +165,7 @@ def setup_analyze_subparser(subparsers): add_common_arguments(analyze_explores) -def setup_vacuum_subparser(subparsers): +def setup_vacuum_subparser(subparsers: argparse._SubParsersAction) -> None: vacuum_parser = subparsers.add_parser( "vacuum", help="vacuum help", usage="henry vacuum" ) @@ -221,7 +221,7 @@ def setup_vacuum_subparser(subparsers): add_common_arguments(vacuum_explores) -def add_common_arguments(parser: argparse.ArgumentParser): +def add_common_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--save", action="store_true", @@ -237,7 +237,7 @@ def add_common_arguments(parser: argparse.ArgumentParser): parser.add_argument("--section", type=str, default="Looker", help=argparse.SUPPRESS) -def parse_input(parser: argparse.ArgumentParser): +def parse_input(parser: argparse.ArgumentParser) -> fetcher.Input: args = vars(parser.parse_args()) return fetcher.Input(**args) From 7bc9637d0ac79fb031bded44212c3ea1ddba341f Mon Sep 17 00:00:00 2001 From: Jorick van der Hoeven Date: Wed, 19 Oct 2022 09:28:08 -0400 Subject: [PATCH 2/3] Update Pulse to be compatible with Looker API 4.0 The pulse command was using outdated 3.0 API calls to Looker and the try_connection sdk command is broken in the Looker 4.0 sdk so we need to catch the exception. This also updates the queries to run with the latest Looker back end model and using the system_activity explores instead of the i__looker explore. --- henry/commands/pulse.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/henry/commands/pulse.py b/henry/commands/pulse.py index 424eaca..63b7410 100644 --- a/henry/commands/pulse.py +++ b/henry/commands/pulse.py @@ -2,7 +2,7 @@ from textwrap import fill from typing import Sequence, cast -from looker_sdk import models +from looker_sdk.sdk.api40 import models from looker_sdk.error import SDKError from henry.modules import exceptions, fetcher, spinner @@ -40,16 +40,20 @@ def check_db_connections(self): for connection in db_connections: assert connection.dialect assert isinstance(connection.name, str) - resp = self.sdk.test_connection( - connection.name, - models.DelimSequence(connection.dialect.connection_tests), - ) - results = list(filter(lambda r: r.status == "error", resp)) - errors = [f"- {fill(cast(str, e.message), width=100)}" for e in results] + try: + resp = self.sdk.test_connection( + connection.name, + models.DelimSequence(connection.dialect.connection_tests), + ) + results = list(filter(lambda r: r.status == "error", resp)) + errors = [f"- {fill(cast(str, e.message), width=100)}" for e in results] + except SDKError: + results = [] + errors = ["API JSONDecode Error"] resp = self.sdk.run_inline_query( "json", models.WriteQuery( - model="i__looker", + model="system__activity", view="history", fields=["history.query_run_count"], filters={"history.connection_name": connection.name}, @@ -76,7 +80,7 @@ def check_dashboard_performance(self): "30 seconds in the last 7 days" ) request = models.WriteQuery( - model="i__looker", + model="system__activity", view="history", fields=["dashboard.title, query.count"], filters={ @@ -99,7 +103,7 @@ def check_dashboard_errors(self): "\bTest 3/6: Checking for dashboards with erroring queries in the last 7 days" # noqa: B950 ) request = models.WriteQuery( - model="i__looker", + model="system__activity", view="history", fields=["dashboard.title", "history.query_run_count"], filters={ @@ -120,7 +124,7 @@ def check_explore_performance(self): """Prints a list of the slowest running explores.""" print("\bTest 4/6: Checking for the slowest explores in the past 7 days") request = models.WriteQuery( - model="i__looker", + model="system__activity", view="history", fields=["query.model", "query.view", "history.average_runtime"], filters={ @@ -148,7 +152,7 @@ def check_schedule_failures(self): """Prints a list of schedules that have failed in the past 7 days.""" print("\bTest 5/6: Checking for failing schedules") request = models.WriteQuery( - model="i__looker", + model="system__activity", view="scheduled_plan", fields=["scheduled_job.name", "scheduled_job.count"], filters={ @@ -166,6 +170,9 @@ def check_schedule_failures(self): def check_legacy_features(self): """Prints a list of enabled legacy features.""" print("\bTest 6/6: Checking for enabled legacy features") - lf = list(filter(lambda f: f.enabled, self.sdk.all_legacy_features())) - legacy_features = [{"Feature": cast(str, f.name)} for f in lf] + try: + lf = list(filter(lambda f: f.enabled, self.sdk.all_legacy_features())) + legacy_features = [{"Feature": cast(str, f.name)} for f in lf] + except SDKError: + legacy_features = [["Unable to pull legacy features due to SDK error"]] self._tabularize_and_print(legacy_features) From 4c0add05b2389edb14d1ad2c4ed20de1e51b0606 Mon Sep 17 00:00:00 2001 From: Jorick van der Hoeven Date: Wed, 19 Oct 2022 09:30:03 -0400 Subject: [PATCH 3/3] Update fetcher queries to use system_activity explore --- henry/modules/fetcher.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/henry/modules/fetcher.py b/henry/modules/fetcher.py index 190eeb2..e9c78ee 100644 --- a/henry/modules/fetcher.py +++ b/henry/modules/fetcher.py @@ -124,14 +124,14 @@ def get_used_models(self) -> Dict[str, int]: resp = self.sdk.run_inline_query( "json", models.WriteQuery( - model="i__looker", + model="system__activity", view="history", fields=["history.query_run_count, query.model"], filters={ "history.created_date": self.timeframe, "query.model": "-system^_^_activity, -i^_^_looker", "history.query_run_count": ">0", - "user.dev_mode": "No", + "user.dev_branch_name": "NULL", }, limit="5000", ), @@ -177,7 +177,7 @@ def get_used_explores( resp = self.sdk.run_inline_query( "json", models.WriteQuery( - model="i__looker", + model="system__activity", view="history", fields=["query.view", "history.query_run_count"], filters={ @@ -225,13 +225,13 @@ def get_used_explore_fields( resp = self.sdk.run_inline_query( "json", models.WriteQuery( - model="i__looker", + model="system__activity", view="history", fields=[ "query.model", "query.view", "query.formatted_fields", - "query.formatted_filters", + "query.filters", "history.query_run_count", ], filters={ @@ -263,7 +263,7 @@ def get_used_explore_fields( # and a dimension/measure, it's listed in both query.formatted_fields # and query.formatted_filters. The recorded variable keeps track of # this, so that no double counting occurs. - filters = row["query.formatted_filters"] + filters = row["query.filters"] if filters: parsed_filters = re.findall(r"(\w+\.\w+)+", filters) for f in parsed_filters: