adding docs directory, and setting up modules

Author: telliere

client/__init__.py

@@ -1,6 +1,6 @@
 from dockerfile_parse import DockerfileParser
 import os, sys
-from tools.docker.docker_utils import build_build_env, check_build_env_exists
+from client.container_preparation.tools.docker.docker_utils import build_build_env, check_build_env_exists
 
 sys.path.append(os.path.expanduser("../../../"))  # For cli usage
 sys.path.append(os.path.expanduser("../../"))  # For inside-container usage

client/container_preparation/__init__.py

@@ -1,6 +1,6 @@
-from tools.docker.docker_utils import check_docker_socket_access
-from lib.image_build import build_prepared_image, create_sif_image
-from tools.cli.cli import parse_arguments
+from client.container_preparation.tools.docker.docker_utils import check_docker_socket_access
+from client.container_preparation.lib.image_build import build_prepared_image, create_sif_image
+from client.container_preparation.tools.cli.cli import parse_arguments
 from docker import DockerClient
 
 

client/container_preparation/lib/__init__.py

@@ -1,6 +1,5 @@
 import argparse
 
-
 # Parse arguments from the cli
 def parse_arguments() -> argparse.ArgumentParser:
     """Parse arguments from the cli

client/container_preparation/lib/image_build.py

@@ -1,5 +1,4 @@
-from tools.cli.cli import parse_arguments
-from tools.utils import check_arguments
+from client.data_preparation.tools.cli.cli import parse_arguments, check_arguments
 from pyrage import x25519, encrypt
 from os import path, remove
 import tarfile

client/container_preparation/prepare_container.py

@@ -1,5 +1,5 @@
 import argparse
-
+from os.path import isfile, isdir
 
 # Parse arguments from the cli
 def parse_arguments() -> argparse.ArgumentParser:
@@ -22,3 +22,28 @@ def parse_arguments() -> argparse.ArgumentParser:
     )
 
     return parser.parse_args()
+
+def check_arguments(input_path: str, output_path: str) -> bool:
+    """Check cli arguments for data preparation and check wether or not input is a file
+
+    Args:
+        input_path (str): Path to the input file or directory
+        output_path (str): Path to the output directory
+
+    Returns:
+        bool: Wether or not the input is a file
+    """
+
+    # Check input path exists, store wether it's a file or a directory
+    try:
+        input_is_file = isfile(input_path)
+    except FileNotFoundError:
+        print(f"File {input_path} doesn't exist.")
+
+    # Check output path exists, and it's a directory
+    try:
+        isdir(input_path)
+    except FileNotFoundError:
+        print(f"Directory {output_path} doesn't exist.")
+
+    return input_is_file

client/container_preparation/tools/__init__.py

@@ -1,10 +1,6 @@
-from utils.cli.cli import parse_arguments, check_arguments
-from lib.sbatch_generation import sbatch_from_template
-from lib.info_file import get_info_from_infofile
-import sys, os
-
-sys.path.append(os.path.expanduser("../../../"))  # For cli usage
-sys.path.append(os.path.expanduser("../../"))  # For inside-container usage
+from  client.job_preparation.utils.cli.cli import parse_arguments, check_arguments
+from client.job_preparation.lib.sbatch_generation import sbatch_from_template
+from client.job_preparation.lib.info_file import get_info_from_infofile
 from utils.ssh_utils import ssh_connect, ssh_copy_file, ssh_run_command
 from utils.conf.client.conf import parse_configuration
 from time import sleep

client/container_preparation/tools/cli/__init__.py

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

client/container_preparation/tools/cli/cli.py

@@ -0,0 +1,31 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+import sys, os
+sys.path.insert(0, os.path.abspath('..'))
+
+project = 'HPCS'
+copyright = '2024, Martin Matthiesen, Emilien Tellier'
+author = 'Martin Matthiesen, Emilien Tellier'
+release = '0.1.0'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ['sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx.ext.autodoc']
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'alabaster'
+html_static_path = ['_static']

client/container_preparation/tools/docker/__init__.py

@@ -0,0 +1,22 @@
+.. HPCS documentation master file, created by
+   sphinx-quickstart on Tue Mar 19 13:07:16 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to HPCS's documentation!
+================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+   
+   modules
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

client/data_preparation/__init__.py

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

client/data_preparation/lib/__init__.py

@@ -1,17 +1,18 @@
-import hashlib, sys, os
+import hashlib
 from quart import Quart, jsonify, request
-from lib.spire_interactions import (
+from pyspiffe.spiffe_id.spiffe_id import SpiffeId
+from pyspiffe.spiffe_id.spiffe_id import SpiffeId
+
+from server.lib.spire_interactions import (
     token_generate,
     entry_create,
     get_server_identity_JWT,
     validate_client_JWT_SVID,
 )
-from lib  import spire_interactions
-from tools.docker_utils import get_build_env_image_digests
-from pyspiffe.spiffe_id.spiffe_id import SpiffeId
-
-from tools.config.config import parse_configuration
-from tools.cli.cli import parse_arguments
+from server.lib  import spire_interactions
+from server.tools.docker_utils import get_build_env_image_digests
+from server.tools.config.config import parse_configuration
+from server.tools.cli.cli import parse_arguments
 from utils.vault.vault_utils import (
     vault_login,
     write_client_policy,
@@ -22,22 +23,24 @@
 
 app = Quart(__name__)
 
-options = parse_arguments()
-configuration = parse_configuration(options.config)
+if __name__ == "__main__":
+    
+    options = parse_arguments()
+    configuration = parse_configuration(options.config)
 
-if configuration['spire-server'].get('spire-server-bin') :
-    spire_interactions.spire_server_bin = configuration['spire-server']['spire-server-bin']
+    if configuration['spire-server'].get('spire-server-bin') :
+        spire_interactions.spire_server_bin = configuration['spire-server']['spire-server-bin']
 
-if configuration['spire-server'].get('pre-command') :
-    spire_interactions.pre_command = configuration['spire-server']['pre-command']
-    if configuration['spire-server']['pre-command'] == "\"\"":
-        spire_interactions.pre_command = ""
-    
-# Defining the trust domain (SPIRE Trust Domain)
-trust_domain = configuration['spire-server']['trust-domain']
+    if configuration['spire-server'].get('pre-command') :
+        spire_interactions.pre_command = configuration['spire-server']['pre-command']
+        if configuration['spire-server']['pre-command'] == "\"\"":
+            spire_interactions.pre_command = ""
+        
+    # Defining the trust domain (SPIRE Trust Domain)
+    trust_domain = configuration['spire-server']['trust-domain']
 
-# Perform vault login, to be able to run later operations against vault
-hvac_client = vault_login(configuration['vault']['url'], get_server_identity_JWT(), configuration['vault']['server-role'])
+    # Perform vault login, to be able to run later operations against vault
+    hvac_client = vault_login(configuration['vault']['url'], get_server_identity_JWT(), configuration['vault']['server-role'])
 
 
 # Dummy endpoint that handles the registration of compute nodes.

client/data_preparation/prepare_data.py

@@ -67,9 +67,11 @@ def entry_create(
     return subprocess.run(command, capture_output=True)
 
 
-def get_server_identity_JWT() -> JwtSvid:
+def get_server_identity_JWT(JwtSource : default_jwt_source.DefaultJwtSource) -> JwtSvid:
     """Get jwt SVID of the server
 
+    Args:
+        JwtSource (default_jwt_source.DefaultJwtSource): connection to the spire agent workload api
     Returns:
         JwtSvid: the SVID
     """
@@ -82,28 +84,32 @@ def get_server_identity_JWT() -> JwtSvid:
     return SVID
 
 
-def validate_JWT_SVID(JwtSVID: str) -> JwtSvid:
+def validate_JWT_SVID(JwtSource : default_jwt_source.DefaultJwtSource, JwtSVID: str) -> JwtSvid:
     """Validate a provided JWT SVID against spire-agent
 
+    Args:
+        JwtSource (default_jwt_source.DefaultJwtSource): connection to the spire agent workload api
+        
     Returns:
         JwtSvid: parsed and validated SVID
     """
-    return jwt_workload_api._workload_api_client.validate_jwt_svid(
+    return JwtSource._workload_api_client.validate_jwt_svid(
         JwtSVID, audience="TESTING"
     )
 
 
-def validate_client_JWT_SVID(JwtSVID: str, client_id: str) -> bool:
+def validate_client_JWT_SVID(JwtSource : default_jwt_source.DefaultJwtSource, JwtSVID: str, client_id: str) -> bool:
     """Validate a client JWT, same as `validate_JWT_SVID` plus client_id verification
 
     Args:
+        JwtSource (default_jwt_source.DefaultJwtSource): connection to the spire agent workload api
         JwtSVID (str): the client's JWT SVID
         client_id (str): the client's id
 
     Returns:
         bool: JWT SVID is valide and matches client id or not
     """
-    SVID = validate_JWT_SVID(JwtSVID)
+    SVID = validate_JWT_SVID(JwtSource, JwtSVID)
     if str(SVID.spiffe_id).split("/")[-2] != client_id:
         return False
     return True

client/data_preparation/tools/__init__.py

@@ -1,16 +1,19 @@
 import argparse
+import requests
+import yaml
+
 from re import search
 from os import path
+
 from pyspiffe.workloadapi import default_jwt_source, default_workload_api_client
 from pyspiffe.spiffe_id import spiffe_id
 from pyspiffe.exceptions import SpiffeIdError
 from pyspiffe.svid.jwt_svid import JwtSvid
-import requests
-from vault.vault_utils import vault_login, write_secret
-import yaml
 from hashlib import sha512
-from ssh_utils import ssh_connect, ssh_copy_file
-from conf.client.conf import parse_configuration
+
+from utils.vault.vault_utils import vault_login, write_secret
+from utils.ssh_utils import ssh_connect, ssh_copy_file
+from utils.conf.client.conf import parse_configuration
 
 # Provide client_id from cli$
 # Same for trust domain

client/data_preparation/tools/cli/__init__.py

@@ -1,5 +1,5 @@
 import platform, argparse, subprocess, requests
-from conf.client.conf import parse_configuration
+from utils.conf.client.conf import parse_configuration
 
 
 # Parse arguments from the cli