diff --git a/README.md b/README.md index c38f49b1f..0ee43df27 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,16 @@ To watch teuthology's output, you can: ```bash podman logs -f teuthology ``` +To locate/ display the ``teuthology.log`` log archive of latest test run, you can + +```bash +# To display the logs of latest run +ceph-devstack logs +``` +```bash +# To locate the logs file of latest run +ceph-devstack logs --log-file +``` If you want testnode containers to be replaced as they are stopped and destroyed, you can: diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py index fc0cc8d94..4a3ed162d 100644 --- a/ceph_devstack/__init__.py +++ b/ceph_devstack/__init__.py @@ -104,6 +104,15 @@ def parse_args(args: List[str]) -> argparse.Namespace: "container", help="The container to wait for", ) + parser_logs=subparsers.add_parser( + "logs", help="Display teuthology.log logs of the latest run" + ) + parser_logs.add_argument( + "--log-file", + action="store_true", + default=False, + help="Display teuthology.log file path of the latest run with flag set" + ) return parser.parse_args(args) diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py index ccb02ef83..98e7c3550 100644 --- a/ceph_devstack/cli.py +++ b/ceph_devstack/cli.py @@ -1,8 +1,10 @@ import asyncio import logging import sys - from pathlib import Path +from pydoc import ttypager +import os +from datetime import datetime from ceph_devstack import config, logger, parse_args, VERBOSE from ceph_devstack.requirements import check_requirements @@ -40,6 +42,9 @@ async def run(): return elif args.command == "wait": return await obj.wait(container_name=args.container) + elif args.command == "logs": + ret=teuthology_logs(str(data_path)) + return ret else: await obj.apply(args.command) return 0 @@ -48,3 +53,70 @@ async def run(): sys.exit(asyncio.run(run())) except KeyboardInterrupt: logger.debug("Exiting!") + +def teuthology_logs(data_path:str) -> int: + """ + This function is used to get the teuthology logs of the latest job run. + + Args: + data_path (str): Path to the data directory. + """ + list_runs = [f.path for f in os.scandir(data_path + "/archive") if f.is_dir()] + if len(list_runs) == 0: + logger.error("No runs found!") + return 1 + # Atleast one run directory exist, due to above condition + try: + list_runs.sort(key=lambda x: datetime.strptime(x.split('/')[-1].split('root-')[1].split('-teuthology')[0], '%Y-%m-%d_%H:%M:%S')) + except ValueError as e: + # if the run directory is not in the format root-YYYY-MM-DD_HH:MM:SS-teuthology + logger.error(f"Error parsing date: {e}") + return 1 + latest_run = list_runs[-1] + latest_run_subdir = [f.path for f in os.scandir(latest_run) if f.is_dir()] + if len(latest_run_subdir) == 0: + logger.error("No jobs found!") + return 1 + # check if only one job present, then display logs. Also check if the teuthology.log file exists in the latest run directory + if len(latest_run_subdir) == 1 : + if os.path.exists(latest_run_subdir[0] + "/teuthology.log") and os.path.isfile(latest_run_subdir[0] + "/teuthology.log"): + try: + if config["args"].get("log_file", False): + print(f"Log file path: {latest_run_subdir[0]}/teuthology.log") + return 0 + with open(latest_run_subdir[0] + "/teuthology.log", 'r') as f: + ttypager(f.read()) + return 0 + except : + logger.error("Error while reading teuthology.log!") + return 1 + else: + logger.error("teuthology.log file not found!") + return 1 + + # Multiple jobs present, then display the job ids + print("Jobs present in latest run:") + job_ids=[] + for job in latest_run_subdir: + job_ids.append(job.split('/')[-1]) + print(f"Job id: {job.split('/')[-1]}") + job_id=input("Enter any of the above job id to get logs: ") + # check if the job id is valid + if job_id not in job_ids: + logger.error("Invalid job id!") + return 1 + # check if the teuthology.log file exists in the job directory + if os.path.exists(latest_run +'/'+ job_id +'/teuthology.log') and os.path.isfile(latest_run +'/'+ job_id +'/teuthology.log'): + try: + if config["args"].get("log_file", False): + print(f"Log file path: {latest_run +'/'+ job_id +'/teuthology.log'}") + return 0 + with open(latest_run +"/"+ job_id +"/teuthology.log", 'r') as f: + ttypager(f.read()) + except : + logger.error("Error while reading teuthology.log!!") + return 1 + else: + logger.error("teuthology.log file not found!") + return 1 + return 0 \ No newline at end of file diff --git a/ceph_devstack/resources/test/test_config.toml b/ceph_devstack/resources/test/test_config.toml new file mode 100644 index 000000000..0466321c2 --- /dev/null +++ b/ceph_devstack/resources/test/test_config.toml @@ -0,0 +1,23 @@ +data_dir = "/tmp/ceph-devstack" + +[containers.archive] +image = "python:alpine" + +[containers.beanstalk] +image = "quay.io/ceph-infra/teuthology-beanstalkd:latest" + +[containers.paddles] +image = "quay.io/ceph-infra/paddles:latest" + +[containers.postgres] +image = "quay.io/ceph-infra/teuthology-postgresql:latest" + +[containers.pulpito] +image = "quay.io/ceph-infra/pulpito:latest" + +[containers.testnode] +count = 3 +image = "quay.io/ceph-infra/teuthology-testnode:latest" + +[containers.teuthology] +image = "quay.io/ceph-infra/teuthology-dev:latest" diff --git a/ceph_devstack/resources/test/test_logs.py b/ceph_devstack/resources/test/test_logs.py new file mode 100644 index 000000000..5e382f847 --- /dev/null +++ b/ceph_devstack/resources/test/test_logs.py @@ -0,0 +1,110 @@ +import pytest +import os +from datetime import datetime,timedelta +from ceph_devstack.cli import main +import random +import shutil +import string +import sys +import logging + +''' +Using pytest paramtrization to test logs command with different +combinations of runs, jobs, selection and if view file path flag is set. +Following parameter combination results in 7*2=14 test cases. +''' +@pytest.mark.parametrize("num_runs,num_jobs,selection,teuthology_file_present",[(0,0,0,True),(2,0,0,True),(4,1,0,True),(4,1,0,False),(4,3,2,True),((4,3,2,False)),(3, 2, 3,True)]) +@pytest.mark.parametrize("flag_set",[ True, False]) +def test_teuthology_logs(num_runs:int,num_jobs:int,selection:int, flag_set:bool,teuthology_file_present:bool,capsys:pytest.CaptureFixture,monkeypatch:pytest.MonkeyPatch,caplog:pytest.LogCaptureFixture) -> int: + """ This function tests the 'logs' command of ceph-devstack. + + Creates a directory structure with random logs and runs the 'logs' command. + Checks if the logs are displayed correctly. + Removes the directory structure after the test. + + Args: + num_runs (int): Number of runs to be created. + num_jobs (int): Number of jobs to be created. + selection (int): The job id to be selected. + flag_set (bool): Flag to set log file path. + capsys (fixture): To capture stdout and stderr. + monkeypatch (fixture): To patch the sys.argv for invoking cli with args and stdin for job selection. + caplog (fixture): To capture logs. + """ + logger = logging.getLogger(__name__) + + if flag_set: + monkeypatch.setattr(sys, 'argv', [ sys.argv[0],'-c', 'ceph_devstack/resources/test/test_config.toml', 'logs','--log-file']) + else: + monkeypatch.setattr(sys, 'argv', [ sys.argv[0],'-c', 'ceph_devstack/resources/test/test_config.toml', 'logs']) + monkeypatch.setattr('builtins.input', lambda name: str(selection)) + data_path = '/tmp/ceph-devstack' + try: + os.makedirs(data_path+'/archive', exist_ok=True) + except Exception as e: + logger.error(f"Error creating directory: {e}") + exit(1) + runs_dir={} + if num_runs>0: + for i in range(num_runs): + end = datetime.now() + start = end - timedelta(days=4) + random_date = start + (end - start) * random.random() + random_date=random_date.strftime('%Y-%m-%d_%H:%M:%S') + try: + os.makedirs(data_path+'/archive'+'/root-'+random_date+'-teuthology', exist_ok=True) + except Exception as e: + logger.error(f"Error creating directory: {e}") + exit(1) + random_logs = [] + if num_jobs>0: + for j in range(num_jobs): + try: + os.makedirs(data_path+'/archive'+'/root-'+random_date+'-teuthology/'+str(j), exist_ok=True) + except Exception as e: + logger.error(f"Error creating directory: {e}") + exit(1) + if teuthology_file_present: + try: + with open(data_path+'/archive'+'/root-'+random_date+'-teuthology/'+str(j)+'/teuthology.log', 'w') as f: + random_logs.append(''.join(random.choices(string.ascii_letters, k=200))) + f.write(random_logs[-1]) + except Exception as e: + logger.error(f"Error creating file: {e}") + exit(1) + runs_dir[data_path+'/archive'+'/root-'+random_date+'-teuthology']=random_logs + try: + with pytest.raises(SystemExit) as main_exit: + main() + except Exception as e: + logger.error(f"Error running main: {e}") + exit(1) + + output, err = capsys.readouterr() + try: + shutil.rmtree(data_path+'/archive') + except Exception as e: + logger.error(f"Error removing directory: {e}") + exit(1) + + runs_dir_list=list(runs_dir.keys()) + if num_runs>0: + runs_dir_list.sort(key=lambda x: datetime.strptime(x.split('/')[-1].split('root-')[1].split('-teuthology')[0], '%Y-%m-%d_%H:%M:%S')) + if num_runs < 1: + assert main_exit.value.code == 1 + assert "No runs found!" in caplog.text + elif num_jobs < 1: + assert main_exit.value.code == 1 + assert "No jobs found!" in caplog.text + elif selection not in range(num_jobs): + assert main_exit.value.code == 1 + assert "Invalid job id!" in caplog.text + elif not teuthology_file_present: + assert main_exit.value.code == 1 + assert "teuthology.log file not found!" in caplog.text + elif flag_set: + assert main_exit.value.code == 0 + assert f"Log file path: {runs_dir_list[-1]}/{selection}/teuthology.log" in output + else: + assert main_exit.value.code == 0 + assert runs_dir[runs_dir_list[-1]][int(selection)] in output