adding documentation about client and server cli

Author: telliere

client/job_preparation/utils/cli/cli.py

@@ -8,7 +8,7 @@ def parse_arguments() -> argparse.Namespace:
     Returns:
         ArgumentParser: the ArgumentParser produced
     """
-    parser = argparse.ArgumentParser(description="CLI Optinons")
+    parser = argparse.ArgumentParser(description="CLI Options")
 
     parser.add_argument(
         "--config",
@@ -21,14 +21,14 @@ def parse_arguments() -> argparse.Namespace:
         "--job-name",
         "-J",
         type=str,
-        help="name of job",
+        help="name of the job",
     )
     parser.add_argument(
         "--nodes",
         "-N",
         type=str,
         required=True,
-        help="number of nodes on which to run",
+        help="count of nodes on which to run",
     )
     parser.add_argument(
         "--partition",
@@ -49,7 +49,7 @@ def parse_arguments() -> argparse.Namespace:
         "-A",
         type=str,
         required=True,
-        help="charge job to specified account",
+        help="account to bill the job to",
     )
     parser.add_argument(
         "--nodelist",

docs/README.md

@@ -5,4 +5,8 @@ In-depth documentations are available in this directory.
 ## Architecture
 
 An overview and an in-depth diagram of the architecture is available [here](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/architecture/architecture.md).
-You can also see sequence diagrams for the [container preparation](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/architecture/container_preparation.md), [data preparation](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/architecture/data_preparation.md) and [job preparation (and runtime)](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/architecture/job_preparation.md).
+You can also see sequence diagrams for the [container preparation](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/architecture/container_preparation.md), [data preparation](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/architecture/data_preparation.md) and [job preparation (and runtime)](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/architecture/job_preparation.md).
+
+## CLI
+
+Documentation of the CLI of the server and the client are available under `cli/`. Those documentations are exports of the `python3 code --help` with further explanation if needed.

docs/cli/container_preparation.md

@@ -0,0 +1,37 @@
+# Container preparation
+
+Using the cli directly isn't recommended, the supported way is through docker's entrypoint.
+
+The container preparation cli allows the user to create/encrypt/ship a HPCS ready image based on any OCI image. 
+
+```
+usage: prepare_container.py [-h] --base-oci-image BASE_OCI_IMAGE --sif-path SIF_PATH [--encrypted] [--docker-path DOCKER_PATH]
+
+CLI Options
+
+options:
+  -h, --help            show this help message and exit
+  --base-oci-image BASE_OCI_IMAGE, -b BASE_OCI_IMAGE
+                        Base OCI image
+  --sif-path SIF_PATH, -s SIF_PATH
+                        Path for the final SIF image (please use "$(pwd)" instead of ".")
+  --encrypted, -e       Encrypt final SIF image or not (default : False)
+  --docker-path DOCKER_PATH, -d DOCKER_PATH
+                        Path to the docker socket (default : /var/run/docker.sock)
+```
+
+Examples
+
+```bash
+# Show the help above
+python3 ./client/container_preparation/prepare_container.py --help
+
+# Run the container preparation while specifying every parameters
+python3 ./client/container_preparation/prepare_container.py --base-oci-image talinx/jp2a --sif-path $(pwd) --encrypted --docker-path /var/run/docker.sock
+
+# Run the container preparation while specifying every parameters (shortened version)
+python3 ./client/container_preparation/prepare_container.py -b talinx/jp2a -s $(pwd) -e -d /var/run/docker.sock
+
+# Run the container preparation specifying minimum flags
+python3 ./client/container_preparation/prepare_container.py -b talin/jp2a -s $(pwd)
+```

docs/cli/data_preparation.md

@@ -0,0 +1,31 @@
+# Data preparation
+
+Using the cli directly isn't recommended, the supported way is through docker's entrypoint.
+
+The data preparation cli allows the user to encrypt/ship an encrypted archive based on any source directory. 
+
+```
+usage: prepare_data.py [-h] --input-path INPUT_PATH --output-path OUTPUT_PATH
+
+CLI Options
+
+options:
+  -h, --help            show this help message and exit
+  --input-path INPUT_PATH, -i INPUT_PATH
+                        Path to the input data
+  --output-path OUTPUT_PATH, -o OUTPUT_PATH
+                        Path to the output encrypted data
+```
+
+Examples
+
+```bash
+# Show the help above
+python3 ./client/data_preparation/prepare_data.py --help
+
+# Run the data preparation while specifying every parameters
+python3 ./client/data_preparation/prepare_data.py --input-path $(pwd)/input_data --output-path $(pwd) 
+
+# Run the data preparation while specifying every parameters (shortened version)
+python3 ./client/data_preparation/prepare_data.py -i $(pwd)/input_data -o $(pwd) 
+```

docs/cli/job_preparation.md

@@ -0,0 +1,72 @@
+# Job preparation
+
+Using the cli directly isn't recommended, the supported way is through docker's entrypoint.
+
+The job preparation cli allows the user to run the secure job. It will gather the necessary informations using the info files before connecting to the supercomputer to run a template-generated sbatch file.
+
+```
+usage: prepare_job.py [-h] --config CONFIG [--job-name JOB_NAME] --nodes NODES --partition PARTITION --time TIME --account ACCOUNT --nodelist NODELIST --workdir WORKDIR --application-info APPLICATION_INFO --data-info
+                      DATA_INFO --supplementary-input-scripts SUPPLEMENTARY_INPUT_SCRIPTS --supplementary-output-scripts SUPPLEMENTARY_OUTPUT_SCRIPTS
+                      [--singularity-supplementary-flags SINGULARITY_SUPPLEMENTARY_FLAGS] [--arguments ARGUMENTS] [--follow]
+
+CLI Options
+
+options:
+  -h, --help            show this help message and exit
+  --config CONFIG       Configuration file (INI Format) (default: /tmp/hpcs-client.conf)
+  --job-name JOB_NAME, -J JOB_NAME
+                        name of the job
+  --nodes NODES, -N NODES
+                        count of nodes on which to run
+  --partition PARTITION, -p PARTITION
+                        partition requested
+  --time TIME, -t TIME  time limit
+  --account ACCOUNT, -A ACCOUNT
+                        account to bill the job to
+  --nodelist NODELIST, -w NODELIST
+                        request a specific list of hosts
+  --workdir WORKDIR, -s WORKDIR
+                        directory to work in (LUMI-SD specific -> different from chdir)
+  --application-info APPLICATION_INFO, -ai APPLICATION_INFO
+                        path to the info file for the image to run on supercomputer
+  --data-info DATA_INFO, -di DATA_INFO
+                        path to the info file for the dataset to use on supercomputer
+  --supplementary-input-scripts SUPPLEMENTARY_INPUT_SCRIPTS, -i SUPPLEMENTARY_INPUT_SCRIPTS
+                        path to your input verification scripts directory
+  --supplementary-output-scripts SUPPLEMENTARY_OUTPUT_SCRIPTS, -o SUPPLEMENTARY_OUTPUT_SCRIPTS
+                        path to your output verification scripts directory
+  --singularity-supplementary-flags SINGULARITY_SUPPLEMENTARY_FLAGS, -flags SINGULARITY_SUPPLEMENTARY_FLAGS
+                        supplementary arguments to pass to singularity
+  --arguments ARGUMENTS, -args ARGUMENTS
+                        supplementary arguments to pass to the application
+  --follow, -f          Follow job's output (default : False)
+```
+
+The standard command format would be like this :
+
+```bash
+python3 ./client/job_preparation/prepare_job.py [SLURM OPTIONS] --workdir [WORKDIR] --application-info [PATH TO INFO FILE] --data-info [PATH TO INFO FILE] --supplementary-input-scripts [PATH TO SUPPLEMENTARY SCRIPTS TO RUN BEFORE APPLICATION] --supplementary-output-scripts [PATH TO SUPPLEMENTARY SCRIPTS TO RUN AFTER APPLICATION] --singularity-supplementary-flags [FLAGS TO MANUALLY PASS TO SINGULARITY] --arguments [ARGUMENTS FOR THE FINAL APPLICATION]
+```
+
+Examples
+
+```bash
+python3 ./prepare_job.py -u etellier -N 1 -p standard -t "00:60:00" -A project_462000031 --nodelist nid001791 --workdir /scratch/project_462000031/etellier -ai /pfs/lustrep4/scratch/project_462000031/etellier/encrypted_prepared_jp2a.sif.info.yaml  -di /pfs/lustrep4/scratch/project_462000031/etellier/encrypted_jp2a_input.tgz.info.yaml -args "\" /sd-container/encrypted/jp2a_input/image.png --width=100 --output=/sd-container/output/result \"" -i /pfs/lustrep4/scratch/project_462000031/etellier/verification_scripts -o /pfs/lustrep4/scratch/project_462000031/etellier/verification_scripts -flags "--env TERM=xterm-256color" -f
+```
+
+Will create a slurm job with the following configuration :
+- 1 node (nid001791)
+- On partition "standard"
+- As etellier
+- for an hour
+
+Also : 
+- HPCS will run the job using `/scratch/project_462000031/etellier` as its workdir
+- The application will be handled using info file at `/pfs/lustrep4/scratch/project_462000031/etellier/encrypted_prepared_jp2a.sif.info.yaml`
+- The application will be handled using info file at `/pfs/lustrep4/scratch/project_462000031/etellier/encrypted_jp2a_input.tgz.info.yaml`
+- "`--env TERM=xterm-256color`" will be passed to `singularity run` when creating the final container
+- The scripts available under the `/pfs/lustrep4/scratch/project_462000031/etellier/verification_scripts` directory will be mounted in the final container in order to run them before and after the container's application
+- `\" /sd-container/encrypted/jp2a_input/image.png --width=100 --output=/sd-container/output/result \"` will be passed to the `singularity run` command, i.e : `singularity run prepared_jp2a.sif -- \" /sd-container/encrypted/jp2a_input/image.png --width=100 --output=/sd-container/output/result \"`
+- The client will follow the output of the job, because of the `-f` flag
+
+For more information about the server configuration, see the [associated documentation](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/configuration/hpcs-client.md).

docs/cli/server.md

@@ -0,0 +1,28 @@
+# Server
+
+Using the cli directly isn't recommended, the supported way is through docker's entrypoint.
+
+The server cli allows the user to invoke the server, passing the hpcs server configuration.
+
+```
+usage: app.py [-h] [--config CONFIG]
+
+CLI Options
+
+options:
+  -h, --help            show this help message and exit
+  --config CONFIG, -c CONFIG
+                        Configuration file (INI Format) (default: /tmp/hpcs-server.conf)
+```
+
+Examples
+
+```bash
+# Show the help above
+python3 ./server/app.py --help
+
+# Run server, using ~/.config/hpcs-server.conf config file
+python3 ./server/app.py --config ~/.config/hpcs-server.conf
+```
+
+For more information about the server configuration, see the [associated documentation](https://github.com/CSCfi/HPCS/tree/doc/readme_and_sequence_diagrams/docs/configuration/hpcs-server.md).

server/tools/cli/cli.py

@@ -8,7 +8,7 @@ def parse_arguments():
     Returns:
         ArgumentParser: the ArgumentParser produced
     """
-    parser = argparse.ArgumentParser(description="CLI Optinons")
+    parser = argparse.ArgumentParser(description="CLI Options")
 
     parser.add_argument(
         "--config",

utils/spawn_agent.py

@@ -9,7 +9,7 @@ def parse_arguments():
     Returns:
         ArgumentParser: the ArgumentParser produced
     """
-    parser = argparse.ArgumentParser(description="CLI Optinons")
+    parser = argparse.ArgumentParser(description="CLI Options")
 
     parser.add_argument(
         "--config",