Skip to content

Commit b7adff3

Browse files
ruratarurata
authored
MarsPlot testing (#103)
* added marsplot test and workflow * install ghostscript for testing * different exe name for windows * yml doesn't have else * yml fix * gs name? * gs name2 * add gs path on win * gs path * gs path win * verify * verify2 * verify3 * gs path * gs name * gs alias for windows * gs alias to test only * move alias to inside unittest * move alias again * subprocess change * subprocess2 * test 64c * gs_command in MarsPlot * update github path * path syntax * added exe * changed gs windows * change gs_command * gs_command fix * add debug to test * debug dash * debug double dash * removed hard coded forward slashes * more path fixes * test gs version * gs version2 * version3 * no more check_gs function * no epscrop for windows test * change to pypdf * updated pyproject
1 parent 29c685f commit b7adff3

5 files changed

Lines changed: 603 additions & 50 deletions

File tree

‎.github/workflows/marsplot_test.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: MarsPlot Test Workflow
2+
# Cancel any in-progress job or previous runs
3+
concurrency:
4+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
5+
cancel-in-progress: true
6+
on:
7+
# Trigger the workflow on push to devel branch
8+
push:
9+
branches: [ devel ]
10+
paths:
11+
- 'bin/MarsPlot.py'
12+
- 'tests/test_marsplot.py'
13+
- '.github/workflows/marsplot_test.yml'
14+
# Allow manual triggering of the workflow
15+
workflow_dispatch:
16+
# Trigger on pull requests that modify MarsPlot or tests
17+
pull_request:
18+
branches: [ devel ]
19+
paths:
20+
- 'bin/MarsPlot.py'
21+
- 'tests/test_marsplot.py'
22+
- '.github/workflows/marsplot_test.yml'
23+
jobs:
24+
test:
25+
# Run on multiple OS and Python versions for comprehensive testing
26+
strategy:
27+
matrix:
28+
os: [ubuntu-latest, macos-latest, windows-latest]
29+
python-version: ['3.9', '3.10', '3.11']
30+
runs-on: ${{ matrix.os }}
31+
steps:
32+
# Checkout the repository
33+
- uses: actions/checkout@v3
34+
# Set up the specified Python version
35+
- name: Set up Python ${{ matrix.python-version }}
36+
uses: actions/setup-python@v3
37+
with:
38+
python-version: ${{ matrix.python-version }}
39+
# Install dependencies
40+
- name: Install dependencies
41+
shell: bash
42+
run: |
43+
python -m pip install --upgrade pip
44+
pip install numpy netCDF4 xarray
45+
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
46+
# Install the package in editable mode
47+
- name: Install package
48+
run: pip install -e .
49+
# Set HOME for Windows since it might be used by the script
50+
- name: Set HOME environment variable for Windows
51+
if: runner.os == 'Windows'
52+
shell: pwsh
53+
run: |
54+
echo "HOME=$env:USERPROFILE" >> $env:GITHUB_ENV
55+
# Set up AmesCAP configuration - handle platform differences
56+
- name: Set up AmesCAP configuration
57+
shell: bash
58+
run: |
59+
mkdir -p $HOME/.amescap
60+
cp mars_templates/amescap_profile $HOME/.amescap_profile
61+
# Print out environment info
62+
- name: Show environment info
63+
run: |
64+
python -c "import os, sys, numpy, netCDF4, xarray; print(f'Python: {sys.version}, NumPy: {numpy.__version__}, NetCDF4: {netCDF4.__version__}, xarray: {xarray.__version__}')"
65+
echo "Working directory: $(pwd)"
66+
echo "Home directory: $HOME"
67+
echo "Environment variables: $(env)"
68+
# Free up disk space
69+
- name: Free up disk space
70+
if: runner.os == 'Linux'
71+
run: |
72+
sudo rm -rf /usr/local/lib/android
73+
sudo rm -rf /usr/share/dotnet
74+
sudo rm -rf /opt/ghc
75+
sudo rm -rf /usr/local/share/boost
76+
# Create temporary directory for tests
77+
- name: Create temporary test directory
78+
shell: bash
79+
run: |
80+
mkdir -p $HOME/marsplot_tests
81+
echo "TMPDIR=$HOME/marsplot_tests" >> $GITHUB_ENV
82+
# Run the integration tests with cleanup between tests
83+
- name: Run MarsPlot tests
84+
run: |
85+
python -m unittest -v tests/test_marsplot.py
86+
# Clean up temporary files to avoid disk space issues - OS specific
87+
- name: Clean up temp files (Unix)
88+
if: runner.os != 'Windows' && always()
89+
shell: bash
90+
run: |
91+
rm -rf $HOME/marsplot_tests || true
92+
- name: Clean up temp files (Windows)
93+
if: runner.os == 'Windows' && always()
94+
shell: pwsh
95+
run: |
96+
Remove-Item -Path "$env:USERPROFILE\marsplot_tests" -Recurse -Force -ErrorAction SilentlyContinue

‎bin/MarsPlot.py‎

Lines changed: 36 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import matplotlib
3737
import re # Regular expressions
3838
import numpy as np
39+
from pypdf import PdfReader, PdfWriter
3940
from netCDF4 import Dataset, MFDataset
4041
from warnings import filterwarnings
4142
import matplotlib.pyplot as plt
@@ -74,6 +75,9 @@
7475
import functools
7576
import traceback
7677

78+
import platform
79+
import shutil
80+
7781
def debug_wrapper(func):
7882
"""
7983
A decorator that wraps a function with error handling based on the
@@ -443,9 +447,9 @@ def main():
443447
Ncdf_num = select_range(Ncdf_num, bound)
444448

445449
# Make folder "plots" in cwd
446-
dir_plot_present = os.path.exists(f"{output_path}/plots")
450+
dir_plot_present = os.path.exists(os.path.join(output_path,"plots"))
447451
if not dir_plot_present:
448-
os.makedirs(f"{output_path}/plots")
452+
os.makedirs(os.path.join(output_path,"plots"))
449453

450454
# ============ Update Progress Bar ============
451455
global i_list
@@ -500,70 +504,52 @@ def main():
500504
# PDF basename "Diagnostics":
501505
# e.g., Custom.in -> Diagnostics.pdf, or
502506
# Custom_01.in -> Diagnostics_01.pdf
503-
input_file = (f"{output_path}/"
504-
f"{args.template_file.name}")
505-
basename = input_file.split("/")[-1].split(".")[0].strip()
507+
input_file = (os.path.join(output_path,
508+
f"{args.template_file.name}"))
509+
510+
if platform.system() == "Windows":
511+
basename = input_file.split("\\")[-1].split(".")[0].strip()
512+
else:
513+
basename = input_file.split("/")[-1].split(".")[0].strip()
506514
except:
507515
# Use default PDF basename "Diagnostics".
508516
basename = "Custom"
509517

510518
# Generate PDF name
511519
if basename == "Custom":
512520
# If template name = Custom.in -> Diagnostics.pdf
513-
output_pdf = (f"{output_path}/Diagnostics.pdf")
521+
output_pdf = os.path.join(output_path,"Diagnostics.pdf")
514522
elif basename[0:7] == "Custom_":
515523
# If template name = Custom_XX.in -> Diagnostics_XX.pdf
516-
output_pdf = (f"{output_path}/Diagnostics_{basename[7:9]}.pdf")
524+
output_pdf = os.path.join(output_path,f"Diagnostics_{basename[7:9]}.pdf")
517525
else:
518526
# If template name is NOT Custom.in, use prefix to
519527
# generate PDF name
520-
output_pdf = (f"{output_path}/{basename}.pdf")
528+
output_pdf = os.path.join(output_path,f"{basename}.pdf")
521529

522530
# Add quotes around PDF name (name -> "name")
523-
output_pdf = f'"{output_pdf}"'
531+
output_pdfq = f'"{output_pdf}"'
524532

525533
# Direct gs output to file instead of printing to screen
526-
debug_filename = f"{output_path}/.debug_MCMC_plots.txt"
534+
debug_filename = os.path.join(output_path,f".debug_MCMC_plots.txt")
527535
fdump = open(debug_filename, "w")
528536

529-
try:
530-
# Test whether gs command fails
531-
cmd_txt = (f"gs -sDEVICE=pdfwrite -dNOPAUSE -dBATCH -dSAFER "
532-
f"-dEPSCrop -sOutputFile={output_pdf} {all_fig}")
533-
subprocess.check_call(cmd_txt, shell = True, stdout = fdump,
534-
stderr = fdump)
535-
except subprocess.CalledProcessError:
536-
# On NAS, gs renamed gs.bin, check_call returns nonzero,
537-
# so use cmd_txt instead:
538-
cmd_txt = (f"gs.bin -sDEVICE=pdfwrite -dNOPAUSE -dBATCH "
539-
f"-dSAFER -dEPSCrop -sOutputFile={output_pdf} "
540-
f"{all_fig}")
537+
writer = PdfWriter()
541538

542539
try:
543-
# Test gs command again
544-
subprocess.check_call(cmd_txt, shell = True, stdout = fdump,
545-
stderr = fdump)
546-
# If successful, execute command
547-
subprocess.call(cmd_txt, shell = True, stdout = fdump,
548-
stderr = fdump)
549-
# Delete individual figures
550-
cmd_txt = (f"rm -f {all_fig}")
551-
subprocess.call(cmd_txt, shell = True, stdout = fdump,
552-
stderr = fdump)
553-
# Delete debug_MCMC_plots.txt debug file
554-
cmd_txt = (f'rm -f "{debug_filename}"')
555-
subprocess.call(cmd_txt, shell = True)
556-
# Delete /plots dir only if created in this routine
557-
if not dir_plot_present:
558-
cmd_txt = (f'rm -r "{output_path}"/plots')
559-
subprocess.call(cmd_txt, shell = True)
560-
give_permission(output_pdf)
561-
print(f"{output_pdf} was generated")
540+
for pdf_file in fig_list:
541+
reader = PdfReader(pdf_file)
542+
for page in reader.pages:
543+
writer.add_page(page)
562544

545+
with open(output_pdf, "wb") as f:
546+
writer.write(f)
547+
give_permission(output_pdf)
548+
print(f"{output_pdfq} was generated")
563549
except subprocess.CalledProcessError:
564550
# If gs command fails again, prompt user to try
565551
# generating PNG instead
566-
print("ERROR with ghostscript when merging PDF, please "
552+
print("ERROR with merging PDF, please "
567553
"try a different format, such as PNG.")
568554
if debug:
569555
raise
@@ -1574,7 +1560,7 @@ def make_template():
15741560
"""
15751561
global customFileIN # Will be modified
15761562
global current_version
1577-
newname = f"{output_path}/Custom.in"
1563+
newname = os.path.join(output_path,"Custom.in")
15781564
newname = create_name(newname)
15791565

15801566
customFileIN = open(newname, "w")
@@ -2094,7 +2080,7 @@ def prep_file(var_name, file_type, simuID, sol_array):
20942080
# Specific sol requested (e.g., [2400])
20952081
Sol_num_current = [0]
20962082

2097-
if os.path.isfile(f"{input_paths[simuID]}/{file_type}.nc"):
2083+
if os.path.isfile(os.path.join(f"{input_paths[simuID]}",f"{file_type}.nc")):
20982084
# First check if file on tape without a sol number
20992085
# (e.g., Luca_dust_MY24_dust.nc exists on disk)
21002086
file_has_sol_number = False
@@ -2118,11 +2104,11 @@ def prep_file(var_name, file_type, simuID, sol_array):
21182104
for i in range(0, nfiles):
21192105
if file_has_sol_number:
21202106
# Sol number
2121-
file_list[i] = (f"{input_paths[simuID]}/"
2122-
f"{int(Sol_num_current[i]):05}.{file_type}.nc")
2107+
file_list[i] = (os.path.join(input_paths[simuID],
2108+
f"{int(Sol_num_current[i]):05}.{file_type}.nc"))
21232109
else:
21242110
# No sol number
2125-
file_list[i] = f"{input_paths[simuID]}/{file_type}.nc"
2111+
file_list[i] = os.path.join(input_paths[simuID],f"{file_type}.nc")
21262112

21272113
check_file_tape(file_list[i])
21282114

@@ -2670,8 +2656,8 @@ def fig_save(self):
26702656
sensitive_name = "multi_panel"
26712657

26722658
plt.tight_layout()
2673-
self.fig_name = (f"{output_path}/plots/"
2674-
f"{sensitive_name}.{out_format}")
2659+
self.fig_name = (os.path.join(output_path,"plots",
2660+
f"{sensitive_name}.{out_format}"))
26752661
self.fig_name = create_name(self.fig_name)
26762662
plt.savefig(self.fig_name, dpi=my_dpi)
26772663
if out_format != "pdf":
@@ -4148,7 +4134,7 @@ def fig_save(self):
41484134
sensitive_name = "multi_panel"
41494135

41504136
self.fig_name = (
4151-
f"{output_path}/plots/{sensitive_name}.{out_format}"
4137+
os.path.join(output_path,"plots",f"{sensitive_name}.{out_format}")
41524138
)
41534139
self.fig_name = create_name(self.fig_name)
41544140

‎docs/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ netcdf4==1.6.5
1515
numpy==1.26.2
1616
requests==2.31.0
1717
scipy==1.11.4
18+
pypdf==5.4.0
1819

1920
# Dependencies required by other packages
2021
certifi==2023.11.17

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ dependencies = [
2222
"xarray>=2023.5.0",
2323
"pandas>=2.0.3",
2424
"pyodbc>=4.0.39",
25+
"pypdf==5.4.0",
2526
]
2627

2728
[project.optional-dependencies]

0 commit comments

Comments
 (0)