Merge branch 'main' into bugfix/fix-test-scale-to-zero

Author: brettmthompson

.flake8

@@ -19,6 +19,7 @@ fcn_exclude_functions =
     re,
     logging,
     LOGGER,
+    BASIC_LOGGER,
     os,
     json,
     pytest,

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
       - id: detect-secrets
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.9
+    rev: v0.11.10
     hooks:
       - id: ruff
       - id: ruff-format
@@ -49,7 +49,7 @@ repos:
   #     - id: renovate-config-validator
 
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.25.1
+    rev: v8.26.0
     hooks:
       - id: gitleaks
 

conftest.py

@@ -2,14 +2,19 @@
 import os
 import pathlib
 import shutil
+import datetime
+import traceback
 
 import shortuuid
+from _pytest.runner import CallInfo
+from _pytest.reports import TestReport
 from pytest import (
     Parser,
     Session,
     FixtureRequest,
     FixtureDef,
     Item,
+    Collector,
     Config,
     CollectReport,
 )
@@ -18,8 +23,15 @@
 from pytest_testconfig import config as py_config
 
 from utilities.constants import KServeDeploymentType
+from utilities.database import Database
 from utilities.logger import separator, setup_logging
-
+from utilities.must_gather_collector import (
+    set_must_gather_collector_directory,
+    set_must_gather_collector_values,
+    get_must_gather_collector_dir,
+    collect_rhoai_must_gather,
+    get_base_dir,
+)
 
 LOGGER = logging.getLogger(name=__name__)
 BASIC_LOGGER = logging.getLogger(name="basic")
@@ -31,6 +43,7 @@ def pytest_addoption(parser: Parser) -> None:
     runtime_group = parser.getgroup(name="Runtime details")
     upgrade_group = parser.getgroup(name="Upgrade options")
     platform_group = parser.getgroup(name="Platform")
+    must_gather_group = parser.getgroup(name="MustGather")
     cluster_sanity_group = parser.getgroup(name="ClusterSanity")
 
     # AWS config and credentials options
@@ -118,6 +131,12 @@ def pytest_addoption(parser: Parser) -> None:
         "--applications-namespace",
         help="RHOAI/ODH applications namespace",
     )
+    must_gather_group.addoption(
+        "--collect-must-gather",
+        help="Indicate if must-gather should be collected on failure.",
+        action="store_true",
+        default=False,
+    )
 
     # Cluster sanity options
     cluster_sanity_group.addoption(
@@ -205,14 +224,22 @@ def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool
 
 
 def pytest_sessionstart(session: Session) -> None:
-    tests_log_file = session.config.getoption("log_file") or "pytest-tests.log"
+    log_file = session.config.getoption("log_file") or "pytest-tests.log"
+    tests_log_file = os.path.join(get_base_dir(), log_file)
+    LOGGER.info(f"Writing tests log to {tests_log_file}")
     if os.path.exists(tests_log_file):
         pathlib.Path(tests_log_file).unlink()
-
+    if session.config.getoption("--collect-must-gather"):
+        session.config.option.must_gather_db = Database()
     session.config.option.log_listener = setup_logging(
         log_file=tests_log_file,
         log_level=session.config.getoption("log_cli_level") or logging.INFO,
     )
+    must_gather_dict = set_must_gather_collector_values()
+    shutil.rmtree(
+        path=must_gather_dict["must_gather_base_directory"],
+        ignore_errors=True,
+    )
 
 
 def pytest_fixture_setup(fixturedef: FixtureDef[Any], request: FixtureRequest) -> None:
@@ -226,9 +253,23 @@ def pytest_runtest_setup(item: Item) -> None:
     2. Adds `fail_if_missing_dependent_operators` fixture for Serverless tests.
     3. Adds fixtures to enable KServe/model mesh in DSC for model server tests.
     """
-
     BASIC_LOGGER.info(f"\n{separator(symbol_='-', val=item.name)}")
     BASIC_LOGGER.info(f"{separator(symbol_='-', val='SETUP')}")
+    if item.config.getoption("--collect-must-gather"):
+        # set must-gather collection directory:
+        set_must_gather_collector_directory(item=item, directory_path=get_must_gather_collector_dir())
+
+        # At the begining of setup work, insert current epoch time into the database to indicate test
+        # start time
+
+        try:
+            db = item.config.option.must_gather_db
+            db.insert_test_start_time(
+                test_name=f"{item.fspath}::{item.name}",
+                start_time=int(datetime.datetime.now().timestamp()),
+            )
+        except Exception as db_exception:
+            LOGGER.error(f"Database error: {db_exception}. Must-gather collection may not be accurate")
 
     if KServeDeploymentType.SERVERLESS.lower() in item.keywords:
         item.fixturenames.insert(0, "fail_if_missing_dependent_operators")
@@ -252,6 +293,10 @@ def pytest_runtest_call(item: Item) -> None:
 
 def pytest_runtest_teardown(item: Item) -> None:
     BASIC_LOGGER.info(f"{separator(symbol_='-', val='TEARDOWN')}")
+    # reset must-gather collector after each tests
+    py_config["must_gather_collector"]["collector_directory"] = py_config["must_gather_collector"][
+        "must_gather_base_directory"
+    ]
 
 
 def pytest_report_teststatus(report: CollectReport, config: Config) -> None:
@@ -276,10 +321,56 @@ def pytest_sessionfinish(session: Session, exitstatus: int) -> None:
     session.config.option.log_listener.stop()
     if session.config.option.setupplan or session.config.option.collectonly:
         return
-    base_dir = py_config["tmp_base_dir"]
-    LOGGER.info(f"Deleting pytest base dir {base_dir}")
-    shutil.rmtree(path=base_dir, ignore_errors=True)
+    if session.config.getoption("--collect-must-gather"):
+        db = session.config.option.must_gather_db
+        file_path = db.database_file_path
+        LOGGER.info(f"Removing database file path {file_path}")
+        if os.path.exists(file_path):
+            os.remove(file_path)
+        # clean up the empty folders
+    collector_directory = py_config["must_gather_collector"]["must_gather_base_directory"]
+    if os.path.exists(collector_directory):
+        for root, dirs, files in os.walk(collector_directory, topdown=False):
+            for _dir in dirs:
+                dir_path = os.path.join(root, _dir)
+                if not os.listdir(dir_path):
+                    shutil.rmtree(path=dir_path, ignore_errors=True)
+    LOGGER.info(f"Deleting pytest base dir {session.config.option.basetemp}")
+    shutil.rmtree(path=session.config.option.basetemp, ignore_errors=True)
 
     reporter: Optional[TerminalReporter] = session.config.pluginmanager.get_plugin("terminalreporter")
     if reporter:
         reporter.summary_stats()
+
+
+def calculate_must_gather_timer(test_start_time: int) -> int:
+    default_duration = 300
+    if test_start_time > 0:
+        duration = int(datetime.datetime.now().timestamp()) - test_start_time
+        return duration if duration > 60 else default_duration
+    else:
+        LOGGER.warning(f"Could not get start time of test. Collecting must-gather for last {default_duration}s")
+        return default_duration
+
+
+def pytest_exception_interact(node: Item | Collector, call: CallInfo[Any], report: TestReport | CollectReport) -> None:
+    LOGGER.error(report.longreprtext)
+    if node.config.getoption("--collect-must-gather"):
+        test_name = f"{node.fspath}::{node.name}"
+        LOGGER.info(f"Must-gather collection is enabled for {test_name}.")
+
+        try:
+            db = node.config.option.must_gather_db
+            test_start_time = db.get_test_start_time(test_name=test_name)
+        except Exception as db_exception:
+            test_start_time = 0
+            LOGGER.warning(f"Error: {db_exception} in accessing database.")
+
+        try:
+            collect_rhoai_must_gather(
+                since=calculate_must_gather_timer(test_start_time=test_start_time),
+                target_dir=os.path.join(get_must_gather_collector_dir(), "pytest_exception_interact"),
+            )
+
+        except Exception as current_exception:
+            LOGGER.warning(f"Failed to collect logs: {test_name}: {current_exception} {traceback.format_exc()}")

pyproject.toml

@@ -65,6 +65,7 @@ dependencies = [
     "jira>=3.8.0",
     "openshift-python-wrapper>=11.0.50",
     "semver>=3.0.4",
+    "sqlalchemy>=2.0.40",
     "pytest-order>=1.3.0",
     "marshmallow==3.26.1,<4", # this version is needed for pytest-jira
 ]

tests/global_config.py

@@ -3,9 +3,9 @@
 distribution: str = "downstream"
 applications_namespace: str = "redhat-ods-applications"  # overwritten in conftest.py if distribution is upstream
 dsc_name: str = "default-dsc"
+must_gather_base_dir: str = "must-gather-base-dir"
 dsci_name: str = "default-dsci"
 dependent_operators: str = "servicemeshoperator,authorino-operator,serverless-operator"
-
 use_unprivileged_client: bool = True
 
 for _dir in dir():

utilities/constants.py

@@ -272,3 +272,5 @@ class RunTimeConfig:
     },
     "commands": {"GRPC": "vllm_tgis_adapter"},
 }
+
+RHOAI_OPERATOR_NAMESPACE = "redhat-ods-operator"

utilities/database.py

@@ -0,0 +1,53 @@
+import logging
+import os
+
+from sqlalchemy import Integer, String, create_engine
+from sqlalchemy.orm import Mapped, Session, mapped_column
+from sqlalchemy.orm import DeclarativeBase
+from utilities.must_gather_collector import get_base_dir
+
+LOGGER = logging.getLogger(__name__)
+
+TEST_DB = "opendatahub-tests.db"
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class OpenDataHubTestTable(Base):
+    __tablename__ = "OpenDataHubTestTable"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True, nullable=False)
+    test_name: Mapped[str] = mapped_column(String(500))
+    start_time: Mapped[int] = mapped_column(Integer, nullable=False)
+
+
+class Database:
+    def __init__(self, database_file_name: str = TEST_DB, verbose: bool = True) -> None:
+        self.database_file_path = os.path.join(get_base_dir(), database_file_name)
+        self.connection_string = f"sqlite:///{self.database_file_path}"
+        self.verbose = verbose
+        self.engine = create_engine(url=self.connection_string, echo=self.verbose)
+        Base.metadata.create_all(bind=self.engine)
+
+    def insert_test_start_time(self, test_name: str, start_time: int) -> None:
+        with Session(bind=self.engine) as db_session:
+            new_table_entry = OpenDataHubTestTable(test_name=test_name, start_time=start_time)
+            db_session.add(new_table_entry)
+            db_session.commit()
+
+    def get_test_start_time(self, test_name: str) -> int:
+        with Session(bind=self.engine) as db_session:
+            result_row = (
+                db_session.query(OpenDataHubTestTable)
+                .with_entities(OpenDataHubTestTable.start_time)
+                .filter_by(test_name=test_name)
+                .first()
+            )
+            if result_row:
+                start_time_value = result_row[0]
+            else:
+                start_time_value = 0
+                LOGGER.warning(f"No test found with name: {test_name}")
+            return start_time_value

utilities/exceptions.py

@@ -96,6 +96,12 @@ def __str__(self) -> str:
         return f"Failed to log in as user {self.user}."
 
 
+class InvalidArgumentsError(Exception):
+    """Raised when mutually exclusive or invalid argument combinations are passed."""
+
+    pass
+
+
 class ResourceNotReadyError(Exception):
     pass