Updates as of 6/28 (end internship status)

README.md

@@ -21,6 +21,21 @@ Viewer provides access to the benchmarks' results.
 
 This code is distributed under the Apache 2.0 license, see the [LICENSE](LICENSE) file.
 
+## Requirements
+- Requires Python 3.6 or later.
+- Python packages 
+    - Matplotlib
+    - Flask 
+    - pandas
+
+
 ## Getting Started with development
 
 There is a specfic document on this at [DEVELOPMENT.md](DEVELOPMENT.md)
+
+## To run the web application
+
+To run the web application, you need to export pythonpath in the following format:
+```export PYTHONPATH="$PYTHONPATH:{directory_to_glideinbenchmark}/glideinbenchmark/src/"```
+then you can run the web application with the following command:
+```python3 glideinbenchmark/src/glideinbenchmark/index.py```

src/glideinbenchmark/factory_parser.py

@@ -17,7 +17,7 @@ def create_new_attr(root, value):
     new_attrib.attrib["const"] = "False"
     new_attrib.attrib["glidein_publish"] = "True"
     new_attrib.attrib["job_publish"] = "False"
-    new_attrib.attrib["parameter"] = "False"
+    new_attrib.attrib["parameter"] = "True"
     new_attrib.attrib["publish"] = "True"
     new_attrib.attrib["type"] = "string"
     new_attrib.attrib["value"] = value

src/glideinbenchmark/index.py

@@ -0,0 +1,32 @@
+from flask import Flask, render_template
+from runner import *
+from viewer import *
+
+app = Flask(__name__)
+app.config['SECRET_KEY'] = 'testing_secret_key'
+@app.route('/')
+def home():
+    return render_template('index.html')
+
+@app.route('/viewer')
+def viewer():
+    return viewer_index()
+
+@app.route('/viewer/histograms')
+def plot_histograms():
+    return histograms()
+
+@app.route('/viewer/histograms/download/<path:filename>', methods=['GET'])
+def download_file_hist(filename):
+    return download_file(filename)
+
+@app.route('/runner')
+def runner():
+    return render_template('runner.html')
+
+@app.route('/runner/config', methods=["GET", "POST"])
+def runner_config_route():
+    return runner_config()
+
+if __name__ == '__main__':
+    app.run(debug=True, host='0.0.0.0')

src/glideinbenchmark/job_log_parsing.py

@@ -0,0 +1,100 @@
+import os
+import xml.etree.ElementTree as ET
+import csv
+
+directory = "../../test/fixtures/job_logs_location/"
+# directory = "/var/log/gwms-factory"
+output_directory = "../../test/fixtures/job_logs_location/output_db"
+
+def find_matching_files(directory):
+    matching_files = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.startswith("job.") and file.endswith(".err"):
+                matching_files.append(os.path.join(root, file))
+    return matching_files
+
+def extract_xml_content(file_path):
+    entry_name = file_path.split("entry_")[1].split("/")[0]
+    factory_name = file_path.split("user_decisionengine")[0].split("/")[-3]
+    with open(file_path, "r") as f:
+        contents = f.readlines()
+        start_index = None
+        end_index = None
+        for i, line in enumerate(contents):
+            if line.strip() == "==== GlideinBenchmark results ====":
+                start_index = i + 1
+            elif line.strip() == "==== GlideinBenchmark end ====":
+                end_index = i
+                break
+        if start_index is not None and end_index is not None:
+            xml_content = "".join(contents[start_index:end_index])
+            # Add the entry and factory lines at the second to last line with correct spacing
+            lines = xml_content.split("\n")
+            second_to_last_line = lines[-3]
+            indent = second_to_last_line[:second_to_last_line.index("<")]
+            xml_content = "\n".join([lines[0]] + [f"{indent}<factory>{factory_name}</factory>", f"{indent}<entry>{entry_name}</entry>"] + lines[1:])
+            return xml_content
+        else:
+            return None
+
+def xml_content_to_csv(xml_content, output_directory, job_name):
+    try:
+        # get the root into an Etree
+        root = ET.fromstring(xml_content)
+        # Find the name associated with the main class
+        name = None
+        csv_file = os.path.join(output_directory, "")  # Add output_directory to csv_file path
+        data = {"job_name": job_name}  # Add job_name as the first column
+        for element in root:
+            if element.tag == "bmk_name":
+                name = element.text
+            if element.text is not None and element.tag != "bmk_name":
+                data[element.tag] = element.text
+
+        # Check if a name is found and print the result
+        if name is None:
+            print("Invalid")
+            return False
+        else:
+            print("Name:", name)
+            csv_file += name + ".csv"
+        # Check if the csv file exists
+        file_exists = os.path.isfile(csv_file)
+        # Read the existing data from the csv file
+        existing_data = []
+        if file_exists:
+            with open(csv_file, 'r') as f:
+                reader = csv.DictReader(f)
+                existing_data = list(reader)
+        # Update the existing row if job_name already exists
+        for row in existing_data:
+            if row["job_name"] == job_name:
+                row.update(data)
+        # Write the content of the xml file to the csv file
+        with open(csv_file, 'w') as f:
+            writer = csv.DictWriter(f, fieldnames=data.keys())
+            writer.writeheader()
+            writer.writerows(existing_data)
+        # Append the data if job_name doesn't exist in the csv file
+        if not any(row["job_name"] == job_name for row in existing_data):
+            with open(csv_file, 'a') as f:
+                writer = csv.DictWriter(f, fieldnames=data.keys())
+                writer.writerow(data)
+    except ET.ParseError:
+        print("The content is not a valid XML file.")
+
+def process_matching_files(directory):
+    matching_files = find_matching_files(directory)
+    for file in matching_files:
+        xml_content = extract_xml_content(file)
+        if xml_content is not None:
+            # remove every value before the last / and add the output directory
+            job_name = os.path.basename(file)[4:-4]
+            # print(xml_content)
+            xml_content_to_csv(xml_content, output_directory, job_name)
+    return directory
+
+if __name__ == "__main__":
+    # Usage
+    process_matching_files(directory)

src/glideinbenchmark/output_plot_creation.py

@@ -0,0 +1,32 @@
+import matplotlib.pyplot as plt
+import pandas as pd
+import os
+
+output_directory = "../../test/fixtures/job_logs_location/output_db"
+
+def create_histogram():
+    # Read the csv files into a single dataframe
+    data = pd.DataFrame()
+    figures = []  # List to store all figures
+    figure_locations = []
+    for file in os.listdir(output_directory):
+        if file.endswith('.csv'):
+            file_path = os.path.join(output_directory, file)
+            figure_locations.append(os.path.abspath(file_path))
+            try:
+                data = pd.read_csv(file_path)
+                # go through the values under the job name and put the results in a histogram
+                results = data['result'].to_list()
+
+                # Create a plot using the job_names as x axis and results as y axis
+                fig, ax = plt.subplots()
+                ax.hist(results, bins=int(max(results)) - int(min(results)) + 1)
+                ax.set_xlabel('Values')
+                ax.set_ylabel('Results')
+                ax.set_title(f'Job Results for {file[:-4]}')
+
+                figures.append(fig)  # Append the figure to the list
+
+            except FileNotFoundError:
+                print(f"File not found: {file_path}")
+    return figures, figure_locations

src/glideinbenchmark/runner.py

@@ -4,3 +4,104 @@
 """
 This App submits targeted benchmarks using the GlideinWMS Factory
 """
+import os
+import subprocess
+
+from flask import Flask, render_template, request
+from glideinbenchmark import factory_parser
+
+# RUN THIS BEFORE RUNNING THE FILE export PYTHONPATH="$PYTHONPATH:/home/swamina7/GlideinBenchmark_webapp/glideinbenchmark/src/"
+
+app = Flask(__name__)
+# xml_file_dir = '../../test/fixtures/etc_gwms-factory/'
+xml_file_dir = '/etc/gwms-factory/'
+script_path = (
+    factory_parser.__file__
+)
+
+app.config['XML_CONFIG'] = xml_file_dir
+
+@app.route("/runner/config", methods=["GET", "POST"])
+def runner_config():
+    """
+    Creates the page controlling the Factory configuration
+    This function handles the runner_config route of the Flask application.
+    The xml_config path is set in the Flask application's configuration before the application is run.
+    """
+    xml_config = app.config['XML_CONFIG']
+
+    if xml_config.endswith(".xml") and os.path.isfile(xml_config):
+        xml_files = [xml_config]
+        xml_config = os.path.join(os.path.dirname(xml_config), "config.d")
+    else:
+        xml_files = []
+    # get a list of all the xml files in the directory
+    if os.path.isdir(xml_config):
+        xml_files += [os.path.join(xml_file_dir,f) for f in os.listdir(xml_config) if f.endswith(".xml")]
+    print(xml_files)
+    # set a dict to store all the entries in
+    entries = {}
+    # Check if there are any xml files in the directory
+    if len(xml_files) == 0:
+        print("No xml files found")
+        return render_template("runner_config.html", entries=[])
+    # go through all the xml files and get the entries in each
+    for xml_file in xml_files:
+        xml_name = os.path.basename(xml_file).split(".")[0]
+        xml_file = os.path.abspath(xml_file)
+        # check if file exists and if not, return empty list
+        if factory_parser.check_file_exists(xml_file) == False:
+            print(xml_file)
+            print("factory location incorrect")
+            return render_template("runner_config.html", entries=[])
+        if request.method == "POST":
+            # get the entries from the current xml file
+            curr_entries = factory_parser.list_entries(xml_file=xml_file)
+            enabled_entries = []
+            disabled_entries = []
+            removed_entries = []
+            # go through each entry and get the status
+            for entry in curr_entries:
+                selected_option = request.form.get(f"{xml_name}_{entry}_status")
+                if selected_option == "enable":
+                    # Code to enable the entry
+                    enabled_entries.append(entry)
+                    print(f"{entry} Enable selected")
+                elif selected_option == "disable":
+                    disabled_entries.append(entry)
+                    # Code to disable the entry
+                    print(f"{entry} Disable selected")
+                elif selected_option == "remove":
+                    removed_entries.append(entry)
+                    # Code to remove the entry
+                    print(f"{entry} Remove selected")
+                else:
+                    print(f"No action taken for {entry}")
+            # get a comma separated list of entries for each type
+            if len(enabled_entries) > 0:
+                enabled_entries_str = ",".join(enabled_entries)
+                subprocess.call(
+                    ["python3", script_path, "-e", enabled_entries_str, xml_file]
+                )
+            if len(disabled_entries) > 0:
+                disabled_entries_str = ",".join(disabled_entries)
+                subprocess.call(
+                    ["python3", script_path, "-d", disabled_entries_str, xml_file]
+                )
+            if len(removed_entries) > 0:
+                removed_entries_str = ",".join(removed_entries)
+                subprocess.call(
+                    ["python3", script_path, "-r", removed_entries_str, xml_file]
+                )
+        curr_entries = factory_parser.list_entry_status(xml_file=xml_file)
+        for entry in curr_entries:
+            # get file name without extension
+            entry_name = xml_name + "_" + entry
+            # add the entry name to the dict with entry name
+            entries[entry_name] = curr_entries[entry]
+        # entries = curr_entries
+    return render_template("runner_config.html", entries=entries)
+
+
+if __name__ == "__main__":
+    app.run(debug=True)