diff --git a/docs/source/code/hpc_container.def b/docs/source/code/hpc_container.def new file mode 100644 index 000000000..af3ba63ac --- /dev/null +++ b/docs/source/code/hpc_container.def @@ -0,0 +1,92 @@ +Bootstrap: docker +From: ubuntu:26.04 + +%post + export DEBIAN_FRONTEND=noninteractive + set -e + + apt-get update && apt-get install -y \ + tzdata \ + wget \ + unzip \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + curl \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libffi-dev \ + liblzma-dev \ + libegl1 \ + libgl1 \ + libglib2.0-0 \ + libx11-6 \ + libxext6 \ + libxrender1 \ + libxkbcommon-x11-0 \ + libxcb1 \ + libdbus-1-3 \ + libxcb-cursor0 \ + libxcb-xinerama0 \ + libsm6 \ + libice6 \ + libglu1-mesa + + ln -fs /usr/share/zoneinfo/Europe/Helsinki /etc/localtime + dpkg-reconfigure --frontend noninteractive tzdata + + # Clean apt caches + apt-get clean + rm -rf /var/lib/apt/lists/* + + # ------------------------------- + # Install Python 3.13 from source + # ------------------------------- + cd /tmp + wget https://www.python.org/ftp/python/3.13.0/Python-3.13.0.tgz + tar -xf Python-3.13.0.tgz + cd Python-3.13.0 + + ./configure --enable-optimizations + make -j$(nproc) + make altinstall # installs as python3.13 + + # ------------------------------- + # Create virtual environment + # ------------------------------- + /usr/local/bin/python3.13 -m venv /opt/venv + + # Activate and install packages + . /opt/venv/bin/activate + pip install --upgrade pip + pip install spinetoolbox + + deactivate + + # Remove Python build artifacts + cd / + rm -rf /tmp/Python-3.13.0* + + # ------------------------------- + # Install GAMS + # ------------------------------- + cd / + wget https://d37drm4t2jghv5.cloudfront.net/distributions/53.5.1/linux/linux_x64_64_sfx.exe + chmod +x linux_x64_64_sfx.exe + ./linux_x64_64_sfx.exe + + mkdir -p /opt + mv /gams53.5_linux_x64_64_sfx /opt/gams53.5 + + rm linux_x64_64_sfx.exe + +%environment + # Make "python" point to venv python + export PATH=/opt/venv/bin:/opt/gams53.5:$PATH + export QT_QPA_PLATFORM=offscreen diff --git a/docs/source/conf.py b/docs/source/conf.py index d83dffd9e..ef07fe0d4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -95,7 +95,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_static_path = ["code"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. diff --git a/docs/source/examples/gams_on_hpc.rst b/docs/source/examples/gams_on_hpc.rst new file mode 100644 index 000000000..75380ada7 --- /dev/null +++ b/docs/source/examples/gams_on_hpc.rst @@ -0,0 +1,474 @@ +.. _Spine Toolbox on HPC: + +======================================= +Running Spine Toolbox projects on HPC's +======================================= + +.. contents:: + :depth: 1 + :local: + +************ +Introduction +************ + +This tutorial demonstrates how to run Spine Toolbox project involving `GAMS `_ +workflows on a High-Performance Computing (HPC) system using the Slurm scheduler. + +The guide assumes you have access to a Linux-based HPC cluster with a shared filesystem and Slurm +installed. You also need basic familiarity with the Linux command line and a Gams license. This tutorial +can be completed with a demo license for Gams, but for real use cases you need a valid Gams license. + +You will learn the full workflow: + +:: + + Local machine → Cluster login → Upload files → Submit job → Monitor → Retrieve results + +**************************************************** +HPC's with container support (apptainer/singularity) +**************************************************** + +The easiest way to run Spine Toolbox projects involving GAMS is to use *apptainer* containers. Log in to the Login +node of your HPC and check if *apptainer* is available in your HPC with the following command: + +.. code-block:: bash + + apptainer --version + +If this fails, try checking if *apptainer* is available as a module: + +.. code-block:: bash + + module avail apptainer + +If the response is a version number or a list of package names and version numbers, you are good to continue to the +next section. If you see an error message or something like 'module not available', skip the next section and +continue from (`HPC's without container support`_). + +.. Note:: + + If the previous commands failed, you can still try if `singularity` is available with `singularity --version` or + `module avail singularity`. Apptainer was previously called singularity. + + +Building the container +---------------------- + +Apptainer is an open source container platform designed for ease-of-use on shared systems and in high performance +computing (HPC) environments. The container is a single file (.sif), which you can be built by using an Apptainer +image definition (.def) file. + +Building the container requires using Linux or Windows Subsystem for Linux (WSL) on Windows. The following +instructions are for WSL (v2+) on Windows with an Ubuntu distro (tested on Ubuntu 24.04). Please make sure you are have +WSL version 2 or later since version 1 is being phased out as obsolete. If you don't have WSL installed, +please contact your organizations IT department for help. Building Apptainer containers is done using *.def* files. +You can `view and copy hpc_container.def on your own system here <../_static/hpc_container.def>`_. Save +`hpc_container.def` file into a mounted drive (for example, `/mnt/c/users//hpc/hpc_container.sif`) for easier +access. The file installs the following apps into the container: + +- Ubuntu 26.04 +- Python 3.13 +- Spine Toolbox (latest release) +- Gams 53.5 + +.. note:: + + Julia and SpineOpt are not included in this container. If you need them, they can be easily added to the .def + file if you want to run Spine Toolbox projects with SpineOpt tools on an HPC as well. + +To start building the container, open command prompt or powershell on Windows and type + +.. code-block:: bash + + wsl + +Install `Go` + +.. code-block:: bash + + sudo apt install -y golang + +Install `apptainer` by cloning the repo and building from sources + +.. code-block:: bash + + git clone https://github.com/apptainer/apptainer.git + cd apptainer + ./mconfig + make -C builddir + sudo make -C builddir install + +Ensure `fakeroot` is configured + +.. code-block:: bash + + sudo apt install fakeroot uidmap + +Cd to `/mnt/c/users//hpc/` or where ever you saved the **hpc_container.def** file, and build +the container by running + +.. code-block:: bash + + apptainer build --fakeroot hpc_container.sif hpc_container.def + +Why --fakeroot? See https://apptainer.org/docs/user/latest/fakeroot.html#fakeroot-feature + +When the build process has completed, if you want to check that everything works, you can use the `shell` command +to spawn a new shell within your container and interact with it as though it were a virtual machine. + +.. code-block:: bash + + apptainer shell hpc_container.sif + +For example, you can check the versions of Python and Gams with `python --version` and `gams ?` respectively +inside the shell. Type `exit` to close the container shell, type `exit` to close wsl and then close the terminal. + +Running a Spine Toolbox project on an HPC +----------------------------------------- + +In this section, you need the following: + +- Spine Toolbox project with a Gams Tool (test project available in /execution_tests/gams_on_hpc_tutorial) +- Container file (**hpc_container.sif**) +- GAMS license file (optional for this tutorial; required for real use cases) +- Slurm script + +Preparing files on the HPC +++++++++++++++++++++++++++ + +Upload all required files to your HPC's home directory using SCP, WinSCP or rsync. We will be using +`gams_on_hpc_tutorial` project in this tutorial: + +1. Upload container: + ``$HOME/spinetoolbox/sifs/hpc_container.sif`` + +2. Upload project: + ``$HOME/spinetoolbox/projects/gams_on_hpc_tutorial`` + +3. Upload GAMS license: + ``$HOME/spinetoolbox/licenses/gamslic.txt`` + +4. Create a Slurm script file: + ``$HOME/spinetoolbox/projects/gams_on_hpc_tutorial/run_on_hpc.sh`` with the following content + +.. code-block:: bash + + #!/bin/bash + #SBATCH --job-name=spinetoolbox_on_hpc + #SBATCH --output=out.txt + #SBATCH --error=err.txt + #SBATCH --time=00:30:00 + #SBATCH --cpus-per-task=1 + #SBATCH --mem=4G + + # Load apptainer. Uncomment if apptainer is available as a module. + # module load apptainer + + set -e # Exit on error + + # ---------------------------- + # User configuration + # ---------------------------- + PROJECT_NAME=gams_on_hpc_tutorial + HOME_BASE="$HOME/spinetoolbox" + + # Choose ONE of these (uncomment the appropriate line) + BASE_TMP="$SCRATCH" # Recommended if available + # BASE_TMP="$WORK" # Alternative on some systems + # BASE_TMP="$TMPDIR" # Often set automatically by Slurm + + # Fallback if chosen variable is empty + if [ -z "$BASE_TMP" ]; then + echo "Warning: selected BASE_TMP is not set, falling back to \$HOME/tmp" + BASE_TMP="$HOME/tmp" + fi + + SCRATCH_BASE="$BASE_TMP/spinetoolbox_runs/$SLURM_JOB_ID" + mkdir -p "$SCRATCH_BASE" + + echo "Using temporary directory: $SCRATCH_BASE" + + # ---------------------------- + # Stage data + # ---------------------------- + echo "Copying project to scratch..." + cp -r $HOME_BASE/projects/$PROJECT_NAME $SCRATCH_BASE/ + cp $HOME_BASE/licenses/gamslic.txt $SCRATCH_BASE/ + + cd $SCRATCH_BASE/$PROJECT_NAME + + # ---------------------------- + # Run container + # ---------------------------- + echo "Running Spine Toolbox..." + + apptainer exec \ + --bind $SCRATCH_BASE:$SCRATCH_BASE \ + $HOME_BASE/sifs/hpc_container.sif \ + spinetoolbox --execute-only $PWD/ + + # ---------------------------- + # Copy results back + # ---------------------------- + echo "Copying results back to home..." + cp -r $SCRATCH_BASE/$PROJECT_NAME/* $HOME_BASE/projects/$PROJECT_NAME/ + + echo "Done." + +.. attention:: + + Line endings in Slurm scripts must be Unix style (LF). + +The ``run_on_hpc.sh`` script stages a Spine Toolbox project to a temporary working directory on the HPC system, +runs it inside an Apptainer container, and then copies the results back to the original project location. This +approach ensures efficient use of the HPC filesystem by performing computation on a fast scratch or temporary +storage area while preserving results in the user’s home directory. + +The folder structure on your HPC should look like this now: + +.. code-block:: text + + home/ + └── spinetoolbox/ + ├── sifs/ + │ └── hpc_container.sif + ├── projects/ + │ └── gams_on_hpc_tutorial/ + │ ├── .spinetoolbox/ + │ │ ├── items/ + │ │ │ └── ... + │ │ ├── specifications/ + │ │ │ └── ... + │ │ └── project.json + │ ├── run_on_hpc.sh + │ ├── model.gms + │ └── ... + └── licenses/ + └── gamslic.txt + +When you want to execute another Spine Toolbox project, copy the project under `/home/spinetoolbox/projects/` and add +a separate `run_on_hpc.sh` Slurm script for that project. + +Editing the Slurm script for your HPC ++++++++++++++++++++++++++++++++++++++ +You may need to adjust the Slurm script (``run_on_hpc.sh``) to match your HPC environment: + +1. **Apptainer module** + Check whether Apptainer is available as a module on your system. + If it is, uncomment the following line:: + + # module load apptainer + +2. **Project name** + Update the ``PROJECT_NAME`` variable to match your Spine Toolbox project folder name. + For this tutorial, it should be:: + + PROJECT_NAME=gams_on_hpc_tutorial + +3. **Temporary working directory** + Check your HPC documentation for the recommended working or scratch filesystem. + + - If your system uses ``$SCRATCH``, no changes are needed. + - Otherwise, update the ``BASE_TMP`` setting by commenting or uncommenting the appropriate line (e.g. ``$WORK`` or ``$TMPDIR``). + - If none of these variables are available, you can define your own custom path. + +4. **Slurm job parameters** + Adjust the resource requests and output settings as needed: + + - ``--job-name``: Job name + - ``--time``: Maximum runtime + - ``--cpus-per-task``: Number of CPU cores + - ``--mem``: Memory allocation + - ``--output``: Output log file + - ``--error``: Error log file + +Submit job to Slurm Scheduler ++++++++++++++++++++++++++++++ + +When you are ready to execute the project, cd to home/spinetoolbox/projects/gams_on_hpc_tutorial and run + +.. code-block:: bash + + sbatch run_on_hpc.sh + +The response will be something like + +``` +Submitted batch job 1303767 +``` + +where 1303767 is the job id + +Check status of submitted job ++++++++++++++++++++++++++++++ + +.. code-block:: bash + + squeue -j + +where ** is the id returned by the `sbatch` command. +To check the status of all of your submitted tasks, run + +.. code-block:: bash + + squeue -u $USER + +If this command fails, replace $USER with your user name. When a job disappears from the the list returned by +the `squeue` command, it is finished. + +Check job output files +++++++++++++++++++++++ + +Since `out.txt` and `err.txt` were given in the Slurm script as the values for *--output* and *--error*, you +can find the stdout and stderr of your job in these files. The file `err.txt` is empty if everything is Ok. +To view the files: + +.. code-block:: bash + + cat out.txt + cat err.txt + +Final job status +++++++++++++++++ + +.. code-block:: bash + + sacct -j + +where ```` is the ID returned by the ``sbatch`` command. +This command should return something like: + +.. code-block:: text + + JobID JobName Partition Account AllocCPUS State ExitCode + ------------ ---------- ---------- ---------- ---------- ---------- -------- + 1303767 spinetool+ all ba6401 1 COMPLETED 0:0 + 1303767.batch batch all ba6401 1 COMPLETED 0:0 + 1303767.extern extern all ba6401 1 COMPLETED 0:0 + +Live monitoring ++++++++++++++++ + +.. code-block:: bash + + watch -n 2 squeue -u $USER + +Another option is to use `tail`: + +.. code-block:: bash + + tail -f out.txt + +Again, if $USER is not defined, replace it with your user name. This function tails the job progress and updates +every two seconds. + +Checking the results +++++++++++++++++++++ + +The result files and output from executing the project will be inside the project item folders just like +when executing the project in Spine Toolbox locally. You can check the results on the HPC, or transfer the +project folder back to your local computer, start Spine Toolbox, and open the project there. + +******************************* +HPC's without container support +******************************* + +.. attention:: + + This section is a work in progress + +Verify GAMS installation: + +.. code-block:: bash + + gams ? + +If GAMS is installed correctly, this command prints version and usage information. + +Accessing GAMS on HPC +--------------------- + +Option 1: Using a Module +++++++++++++++++++++++++ + +Many HPC systems provide GAMS via environment modules: + +.. code-block:: bash + + module avail gams + module load gams + +Verify: + +.. code-block:: bash + + which gams + + +Option 2: User Installation ++++++++++++++++++++++++++++ + +If GAMS is not provided: + +1. Download the Linux version from the GAMS website +2. Extract it in your home or project directory +3. Add it to your PATH: + +.. code-block:: bash + + export PATH=$HOME/gams:$PATH + +Ensure that your license file is accessible (e.g., ``gamslice.txt``). + +Common Issues and Troubleshooting +--------------------------------- + +License Errors +++++++++++++++ + +- Ensure license file is accessible on compute nodes +- Check environment variables if needed: + +.. code-block:: bash + + export GAMSLICE=/path/to/gamslice.txt + +File Not Found +++++++++++++++ + +- Verify paths are correct relative to the SLURM working directory +- Use: + +.. code-block:: bash + + echo $PWD + +Job Stuck in Queue +++++++++++++++++++ + +- Cluster is full +- Resource request too large + +Memory Errors ++++++++++++++ + +Increase memory: + +.. code-block:: bash + + #SBATCH --mem=16G + + +Solver Not Found +++++++++++++++++ + +.. code-block:: bash + + module load gurobi + +Check installation: + +.. code-block:: bash + + which gurobi_cl diff --git a/docs/source/examples/hpc_basics.rst b/docs/source/examples/hpc_basics.rst new file mode 100644 index 000000000..37a97e011 --- /dev/null +++ b/docs/source/examples/hpc_basics.rst @@ -0,0 +1,262 @@ +.. _HPC Basics: + +==================================================== +Background on High-Performance Cluster (HPC) Systems +==================================================== + +This section briefly describes how common HPC systems work. The description is intentionally general so it applies to: + +- University HPC clusters +- Company internal clusters +- Cloud-based HPC environments (e.g., AWS, Azure) + +If you are already familiar with HPC systems, you can jump directly to the next section :ref:`Spine Toolbox on HPC`. + +************ +Introduction +************ + +Most HPC systems follow a similar architecture composed of three main components: + +- **Login node**: the entry point to the system where users connect, prepare job scripts, and submit jobs +- **Compute nodes**: the machines where the actual computational work is executed +- **Scheduler (e.g., Slurm)**: the system responsible for allocating resources and dispatching jobs to compute nodes + +.. figure:: ../img/tutorials/hpc_architecture.png + :width: 100% + :align: center + + Conceptual structure of an HPC cluster + +Users typically interact only with the login node. After a job is submitted, the scheduler assigns it to available +compute nodes, where it runs—often in parallel across multiple CPUs or nodes. + +.. warning:: + + Do not run heavy computations on the login node. It is intended only for job preparation and submission. + +***************** +Workflow Overview +***************** + +A typical HPC workflow separates job preparation from execution: + +1. The user prepares input data and job scripts on their local machine +2. The job is submitted to the scheduler via the login node +3. The scheduler places the job in a queue and assigns resources when available +4. The job runs on compute nodes +5. Results are stored on the cluster and can be retrieved back to the local environment + +This separation allows HPC systems to efficiently share resources among many users. + +.. figure:: ../img/tutorials/hpc_workflow.png + :width: 100% + :align: center + + Overview of the HPC workflow + +************* +Job Lifecycle +************* + +When a job is submitted to the scheduler (such as Slurm), it goes through several states: + +- **PENDING**: the job is waiting in the queue for resources to become available +- **RUNNING**: the job is actively executing on compute nodes +- **COMPLETED**: the job has finished successfully (or terminated with an error) + +Understanding these states helps explain why jobs may not start immediately—waiting in the queue is normal and +depends on factors such as resource availability and system load. + +.. figure:: ../img/tutorials/job_lifecycle.png + :width: 100% + :align: center + + Lifecycle of a Slurm job, including submission, queuing, execution, + and completion stages. + + +************************* +Connecting to the Cluster +************************* + +Login +----- + +You need access rights (username/password) for your HPC cluster, which you should request from the HPC administrator. +Once you have the necessary rights, you can make an SSH connection from your terminal to the HPC login node. + +.. code-block:: bash + + ssh username@cluster.address + +On Windows, we recommend using PuTTY as an SSH client. You can install it from the Windows +Store or from the `PuTTY site `_. + +File Transfer +------------- + +You can use scp to transfer files between your local system and the login node. + +.. code-block:: bash + + scp -r my_project/ username@cluster:/home/username/ + +However, in the long run this may become tedious, so it is recommended that you use for example +`WinSCP `_, which makes file transfers quicker by providing an easy to use drag-and-drop UI. + +************************************* +Understanding the Cluster Environment +************************************* + +Working on an HPC cluster differs from working on a local machine. The filesystem is typically distributed and +shared across compute nodes, and different directories are designed for specific purposes. Using them correctly +is essential for performance, data safety, and efficient workflows. + +Common Directory Types +---------------------- + +Running applications on a cluster requires that all compute nodes involved in a job can access the same files. +This is usually achieved through a shared parallel filesystem. While directory names vary between systems, +most clusters provide the following *types* of storage: + +- ``$HOME`` (or home directory) + + Your personal home directory. This is **persistent storage**, meaning files are kept long-term and often backed up. + However, it is typically **not optimized for heavy I/O workloads**, so it should mainly be used for: + + - Source code + - Configuration files + - Small input datasets + - Scripts and job submission files + +- High-performance temporary storage (often called ``$SCRATCH`` or similar) + + A fast storage area intended for **temporary data and intensive I/O operations**. + The exact location and name vary by system. Examples include: + + - ``$SCRATCH`` (if defined) + - ``/scratch`` + - ``/tmp`` + - ``/jobs`` (on some systems) + + Consult your system documentation to find the correct path. + + Use this storage for: + + - Large simulation outputs + - Intermediate data + - Temporary working files + + .. note:: + These locations are usually not backed up and may be cleaned automatically + after a retention period. Always copy important data to persistent storage. + +- Project or shared storage (often called ``$PROJECT`` or ``$WORK``) + + A shared directory intended for **collaborative work within a project or research group**. + Not all systems provide this, but when available it typically offers more space than ``$HOME`` + and longer retention than temporary storage. + + Common uses include: + + - Shared datasets + - Group software installations + - Results that need to be preserved longer-term + + .. note:: + If your system does not provide a dedicated project directory, you may need + to manage shared data manually in agreed-upon locations. + +Best Practices +-------------- + +Because directory names vary between systems, always adapt these guidelines to the paths provided on your cluster: + +- Copy input data from persistent storage (e.g. ``$HOME`` or project space) to temporary high-performance storage + before running jobs. +- Perform all **compute-intensive tasks** using the high-performance temporary storage (e.g. ``$SCRATCH`` or ``/jobs``). +- Regularly clean up unnecessary files from temporary storage. +- Avoid running large jobs directly from your home directory. + +Example Workflow +---------------- + +A typical workflow might look like: + +1. Prepare input files and scripts in your home directory (``$HOME``) +2. Copy necessary data to the system's temporary work directory (e.g. ``$SCRATCH`` or ``/jobs``) +3. Run the job using the scheduler +4. Copy final results back to persistent storage (``$HOME`` or project space) +5. Clean up temporary data + +This approach ensures efficient use of cluster resources while keeping your data safe and organized. Next section +provides an actual example workflow for executing Spine Toolbox projects in an HPC environment. + +Module System +------------- + +Environment modules is a system tool to manage the shell environment. It makes it easier to handle the shell +environment when there are e.g. multiple versions of the same software installed. You can get a list of available +modules with command + +.. code-block:: bash + + module avail + +The needed module can be loaded with the command + +.. code-block:: bash + + module load + +A module can be unloaded with the command + +.. code-block:: bash + + module unload + +You can load any modules you need in a Slurm script with the `module load` command. + +Slurm - Basic Commands +---------------------- + +These basic commands should be available in the HPC login node to help you get started. There are many more +commands available, please also read your HPC documentation if there are special circumstances you should be +aware of when using Slurm. + +sinfo +***** + +This command gives a quick overview of the cluster status. It shows the status of different partitions, time +limits, and available nodes. + +squeue +****** + +This command outputs information of all queues on all partitions. You can filter the output by partition with the +**-p** argument, or by username with the **-u** argument. For example, `squeue -u ` prints the queued +jobs of user . + +sbatch +****** + +This command will submit a job script to the queue. For example, `sbatch job.sh`. Please make sure that the batch +script file uses **Unix (LF)** line endings. + +scancel +******* + +This command cancels a job. It needs a job id as an argument. For example, `scancel 10346`. + +srun +**** + +This command runs a single command on Slurm. It needs the same arguments for resource reservation as `sbatch`. +For example `srun -J jobname --mem=64000 --pty -n 36 -p large36 /bin/bash`. + +scontrol show job +***************** + +This command outputs detailed information about a job. It needs job id as an argument. For example, +`scontrol show job 10345`. diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst index fd6f49457..713ebd7d1 100644 --- a/docs/source/examples/index.rst +++ b/docs/source/examples/index.rst @@ -21,3 +21,10 @@ This module currently contains tutorials for Spine Toolbox and Spine Database Ed first_relationship basic_manage_elements basic_pivot_add_relationships + +.. toctree:: + :maxdepth: 1 + :caption: Running models on a High-Performance Computing (HPC) system + + hpc_basics + gams_on_hpc diff --git a/docs/source/img/tutorials/hpc_architecture.png b/docs/source/img/tutorials/hpc_architecture.png new file mode 100644 index 000000000..6469c2aec Binary files /dev/null and b/docs/source/img/tutorials/hpc_architecture.png differ diff --git a/docs/source/img/tutorials/hpc_workflow.png b/docs/source/img/tutorials/hpc_workflow.png new file mode 100644 index 000000000..6cce763cf Binary files /dev/null and b/docs/source/img/tutorials/hpc_workflow.png differ diff --git a/docs/source/img/tutorials/job_lifecycle.png b/docs/source/img/tutorials/job_lifecycle.png new file mode 100644 index 000000000..f5b9129e9 Binary files /dev/null and b/docs/source/img/tutorials/job_lifecycle.png differ diff --git a/execution_tests/gams_on_hpc_tutorial/.gitignore b/execution_tests/gams_on_hpc_tutorial/.gitignore new file mode 100644 index 000000000..cacf63d26 --- /dev/null +++ b/execution_tests/gams_on_hpc_tutorial/.gitignore @@ -0,0 +1,3 @@ +.spinetoolbox/local/ +.spinetoolbox/items/ +*.bak* diff --git a/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/items/input_data/input.csv b/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/items/input_data/input.csv new file mode 100644 index 000000000..fd96e31e6 --- /dev/null +++ b/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/items/input_data/input.csv @@ -0,0 +1,3 @@ +a,10 +b,20 +c,30 diff --git a/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/project.json b/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/project.json new file mode 100644 index 000000000..e5751d431 --- /dev/null +++ b/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/project.json @@ -0,0 +1,63 @@ +{ + "project": { + "version": 13, + "description": "", + "settings": { + "enable_execute_all": true, + "store_external_paths_as_relative": false + }, + "specifications": { + "Tool": [ + { + "type": "path", + "relative": true, + "path": ".spinetoolbox/specifications/Tool/gams_model.json" + } + ] + }, + "connections": [ + { + "name": "from Input Data to Gams Tool", + "from": [ + "Input Data", + "right" + ], + "to": [ + "Gams Tool", + "left" + ], + "filter_settings": { + "auto_online": true, + "enabled_filter_types": { + "alternative_filter": false, + "scenario_filter": true + } + } + } + ], + "jumps": [] + }, + "items": { + "Input Data": { + "type": "Data Connection", + "description": "", + "x": -417.95092606598814, + "y": -252.78477697003137, + "file_references": [], + "file_patterns": [], + "directory_references": [], + "db_references": [] + }, + "Gams Tool": { + "type": "Tool", + "description": "", + "x": -272.9269902744163, + "y": -285.01231825704724, + "specification": "gams_model", + "execute_in_work": true, + "cmd_line_args": [], + "kill_completed_processes": false, + "log_process_output": false + } + } +} \ No newline at end of file diff --git a/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/specifications/Tool/gams_model.json b/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/specifications/Tool/gams_model.json new file mode 100644 index 000000000..173c03b21 --- /dev/null +++ b/execution_tests/gams_on_hpc_tutorial/.spinetoolbox/specifications/Tool/gams_model.json @@ -0,0 +1,18 @@ +{ + "name": "gams_model", + "tooltype": "gams", + "includes": [ + "model.gms" + ], + "description": "", + "inputfiles": [ + "input.csv" + ], + "inputfiles_opt": [], + "outputfiles": [ + "model.lst", + "output.txt" + ], + "cmdline_args": [], + "includes_main_path": "../../.." +} \ No newline at end of file diff --git a/execution_tests/gams_on_hpc_tutorial/model.gms b/execution_tests/gams_on_hpc_tutorial/model.gms new file mode 100644 index 000000000..f662e40f3 --- /dev/null +++ b/execution_tests/gams_on_hpc_tutorial/model.gms @@ -0,0 +1,35 @@ +* Convert CSV to GDX (VERY SIMPLE, no headers) +$call csv2gdx input.csv id=data index=1 value=2 colCount=2 output=data.gdx + +* Declare symbols +set i; +parameter data(*), value(i); + +* Load data +$gdxin data.gdx +$load data i=Dim1 +$gdxin + +display i, data; + +* Copy values +value(i) = data(i); + +* Compute total +scalar total; +total = sum(i, value(i)); + +display total; + +* Write output file +file fout /output.txt/; +put fout; + +put "Results:" /; +loop(i, + put i.tl, " ", value(i):8:2 /; +); + +put "Total = ", total:8:2 /; + +putclose fout;