Skip to content
17 changes: 6 additions & 11 deletions drishti/handlers/handle_darshan.py
Original file line number Diff line number Diff line change
Expand Up @@ -752,15 +752,10 @@ def handler():
display_thresholds(console)
display_footer(console, insights_start_time, insights_end_time)

filename = '{}.html'.format(args.log_path)
export_html(console, filename)

filename = '{}.svg'.format(args.log_path)
export_svg(console, filename)

filename = '{}-summary.csv'.format(
args.log_path.replace('.darshan', '')
)

export_csv(filename, job['job']['jobid'])
# Export to HTML, SVG, and CSV
input_filename = os.path.basename(args.log_path).replace('.darshan', '')
out_dir = args.export_dir if args.export_dir != "" else os.getcwd()

export_html(console, out_dir, input_filename)
export_svg(console, out_dir, input_filename)
export_csv(out_dir, input_filename, job['job']['jobid'])
28 changes: 9 additions & 19 deletions drishti/handlers/handle_recorder.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

import os
import time

import pandas as pd
from recorder_utils import RecorderReader
from recorder_utils.build_offset_intervals import build_offset_intervals

from drishti.includes.module import *


Expand Down Expand Up @@ -577,23 +579,11 @@ def process_helper(file_map, df_intervals, df_posix_records, fid=None):
display_thresholds(console)
display_footer(console, insights_start_time, insights_end_time)

if args.split_files:
filename = '{}.{}.html'.format(args.log_path, fid)
else:
filename = '{}.html'.format(args.log_path)

export_html(console, filename)

if args.split_files:
filename = '{}.{}.svg'.format(args.log_path, fid)
else:
filename = '{}.svg'.format(args.log_path)

export_svg(console, filename)

if args.split_files:
filename = '{}.{}.summary.csv'.format(args.log_path, fid)
else:
filename = '{}-summary.csv'.format(args.log_path)
export_csv(filename)
# Export to HTML, SVG, and CSV
input_filename = os.path.basename(os.path.dirname(args.log_path))
input_filename = f"{input_filename}.{fid}" if args.split_files else input_filename # Append fid if split_files is enabled
out_dir = args.export_dir if args.export_dir != "" else os.getcwd()

export_html(console, out_dir, input_filename)
export_svg(console, out_dir, input_filename)
export_csv(out_dir, input_filename, job['job']['jobid'])
142 changes: 78 additions & 64 deletions drishti/includes/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -1823,76 +1823,90 @@ def display_footer(console, insights_start_time, insights_end_time):
)
)

def export_html(console, filename):
'''
'''

if args.export_html:
console.save_html(
filename,
theme=set_export_theme(),
clear=False
)
def export_html(console, export_dir, filename):
if not args.export_html:
return

os.makedirs(export_dir, exist_ok=True) # Ensure export directory exists
filepath = os.path.join(export_dir, f"{filename}.html")

def export_svg(console, filename):
if args.export_svg:
console.save_svg(
filename,
title='Drishti',
theme=set_export_theme(),
clear=False
)
console.save_html(
filepath,
theme=set_export_theme(),
clear=False
)


def export_csv(filename, jobid=None):
if args.export_csv:
issues = [
'JOB',
INSIGHTS_STDIO_HIGH_USAGE,
INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE,
INSIGHTS_POSIX_READ_COUNT_INTENSIVE,
INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE,
INSIGHTS_POSIX_READ_SIZE_INTENSIVE,
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE,
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE,
INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE,
INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE,
INSIGHTS_POSIX_REDUNDANT_READ_USAGE,
INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE,
INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE,
INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE,
INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE,
INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE,
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE,
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE,
INSIGHTS_POSIX_HIGH_METADATA_TIME,
INSIGHTS_POSIX_SIZE_IMBALANCE,
INSIGHTS_POSIX_TIME_IMBALANCE,
INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE,
INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE,
INSIGHTS_MPI_IO_NO_USAGE,
INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE,
INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE,
INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE,
INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE,
INSIGHTS_MPI_IO_BLOCKING_READ_USAGE,
INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE,
INSIGHTS_MPI_IO_AGGREGATORS_INTRA,
INSIGHTS_MPI_IO_AGGREGATORS_INTER,
INSIGHTS_MPI_IO_AGGREGATORS_OK
]
if codes:
issues.extend(codes)
def export_svg(console, export_dir, filename):
if not args.export_svg:
return

os.makedirs(export_dir, exist_ok=True) # Ensure export directory exists
filepath = os.path.join(export_dir, f"{filename}.svg")

console.save_svg(
filepath,
title='Drishti',
theme=set_export_theme(),
clear=False
)

detected_issues = dict.fromkeys(issues, False)
detected_issues['JOB'] = jobid

for report in csv_report:
detected_issues[report] = True
def export_csv(export_dir, filename, jobid=None):
if not args.export_csv:
return

issues = [
'JOB',
INSIGHTS_STDIO_HIGH_USAGE,
INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE,
INSIGHTS_POSIX_READ_COUNT_INTENSIVE,
INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE,
INSIGHTS_POSIX_READ_SIZE_INTENSIVE,
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE,
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE,
INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE,
INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE,
INSIGHTS_POSIX_REDUNDANT_READ_USAGE,
INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE,
INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE,
INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE,
INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE,
INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE,
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE,
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE,
INSIGHTS_POSIX_HIGH_METADATA_TIME,
INSIGHTS_POSIX_SIZE_IMBALANCE,
INSIGHTS_POSIX_TIME_IMBALANCE,
INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE,
INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE,
INSIGHTS_MPI_IO_NO_USAGE,
INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE,
INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE,
INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE,
INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE,
INSIGHTS_MPI_IO_BLOCKING_READ_USAGE,
INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE,
INSIGHTS_MPI_IO_AGGREGATORS_INTRA,
INSIGHTS_MPI_IO_AGGREGATORS_INTER,
INSIGHTS_MPI_IO_AGGREGATORS_OK
]
if codes:
issues.extend(codes)

detected_issues = dict.fromkeys(issues, False)
detected_issues['JOB'] = jobid

for report in csv_report:
detected_issues[report] = True


os.makedirs(export_dir, exist_ok=True) # Ensure export directory exists
filepath = os.path.join(export_dir, f"{filename}.csv")

with open(filename, 'w') as f:
w = csv.writer(f)
w.writerow(detected_issues.keys())
w.writerow(detected_issues.values())
with open(filepath, 'w') as f:
w = csv.writer(f)
w.writerow(detected_issues.keys())
w.writerow(detected_issues.values())

7 changes: 7 additions & 0 deletions drishti/includes/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,13 @@
help='Export a CSV with the code of all issues that were triggered'
)

parser.add_argument(
'--export_dir',
default="",
dest='export_dir',
help='Specify the directory prefix for the output files (if any)'
)

parser.add_argument(
'--json',
default=False,
Expand Down