adapting job preparation to config file usage

Author: telliere

client/container_preparation/input_logic/run.sh

@@ -7,10 +7,10 @@ PATH="$PATH:/sd-container/tools/input_logic/"
 echo "[SD-Container][Input-Logic] : Getting data decryption key from vault"
 
 # Get token via vault login. The data_login environment variable need to be exported from calling script
-data_token=$(curl -s --request POST --data "$data_login" http://${vault}/v1/auth/jwt/login | jq '.auth.client_token' -r)  || exit 1
+data_token=$(curl -s --request POST --data "$data_login" $vault/v1/auth/jwt/login | jq '.auth.client_token' -r)  || exit 1
 
 # Use the token to access the key. The data_path environment variable needs to be exported from calling script
-data_key=$(curl -s -H "X-Vault-Token: $data_token" http://${vault}/v1/kv/data/${data_path} | jq '.data.data.key' -r)    || exit 1
+data_key=$(curl -s -H "X-Vault-Token: $data_token" $vault/v1/kv/data/${data_path} | jq '.data.data.key' -r)    || exit 1
 
 # Write the key in an encrypted volume
 echo "$data_key" > /sd-container/encrypted/decryption_key

client/job_preparation/lib/sbatch_generation.py

@@ -44,7 +44,8 @@ def boostrap_from_template(options: argparse.Namespace, template_path: str) -> s
         sbatch = sbatch.replace("ACCOUNT", options.account)
         sbatch = sbatch.replace("NODELIST", options.nodelist)
         sbatch = sbatch.replace("WORKDIR", options.workdir)
-        sbatch = sbatch.replace("TRUST_DOMAIN", "lumi-sd-dev")
+        sbatch = sbatch.replace("TRUST_DOMAIN", options.trust_domain)
+        sbatch = sbatch.replace("VAULT_ADDRESS", options.vault_address)
 
         # Dataset info
         sbatch = sbatch.replace("DATA_PATH", options.data_path_at_rest)

client/job_preparation/prepare_job.py

@@ -6,12 +6,24 @@
 sys.path.append(os.path.expanduser("../../../"))  # For cli usage
 sys.path.append(os.path.expanduser("../../"))  # For inside-container usage
 from utils.ssh_utils import ssh_connect, ssh_copy_file, ssh_run_command
+from utils.conf.client.conf import parse_configuration
 from time import sleep
 from pyrage import x25519
 
 if __name__ == "__main__":
     # Parse arguments
-    options = check_arguments(parse_arguments())
+    options = parse_arguments()
+
+    # Parse configuration
+    configuration = parse_configuration(options.config)
+    
+    # Parse configuration as options
+    options.username = configuration['supercomputer']['username']
+    options.trust_domain = configuration['spire-server']['trust-domain']
+    options.vault_address = configuration['vault']['url']
+    
+    # Check arguments
+    options = check_arguments(options)
 
     # Connect via SSH to supercomputer
     ssh_client = ssh_connect(options.username)
@@ -39,6 +51,9 @@
 
     # Copy SBATCH to supercomputer
     ssh_copy_file(ssh_client, sbatch_path, f"~/")
+    
+    # Copy config file to supercomputer
+    ssh_copy_file(ssh_client, options.config, f"~/.config/hpcs.conf")
 
     # Create public encryption key for output data
     ident = x25519.Identity.generate()

client/job_preparation/utils/cli/cli.py

@@ -11,11 +11,11 @@ def parse_arguments() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="CLI Optinons")
 
     parser.add_argument(
-        "--username",
-        "-u",
-        required=True,
+        "--config",
         type=str,
-        help="username on supercomputer",
+        required=True,
+        default="/tmp/hpcs-client.conf",
+        help="Configuration file (INI Format) (default: /tmp/hpcs-client.conf)",
     )
     parser.add_argument(
         "--job-name",

client/job_preparation/utils/sbatch.template

@@ -61,8 +61,8 @@ mkdir -p ${WORKING_DIRECTORY}
 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Running agent registration"
 
 # Spawn spire-agent
-cd ~/LUMI-secure-processing           || exit 1
-python3 ./utils/spawn_agent.py -cn > $WORKING_DIRECTORY/agent.log 2> $WORKING_DIRECTORY/agent.log &
+cd ~/HPCS           || exit 1
+python3 ./utils/spawn_agent.py --config ~/.config/hpcs-client.conf -cn > $WORKING_DIRECTORY/agent.log 2> $WORKING_DIRECTORY/agent.log &
 spire_agent_pid=$!
 
 # Wait until agent runs properly
@@ -85,18 +85,18 @@ echo "Logging in to the vault ..."
 
 # Log in to the vault using SVID, access role
 echo "{\"role\": \"APPLICATION_ACCESS_ROLE\", \"jwt\" : \"$svid\"}" > /tmp/login
-application_token=$(curl -s --request POST --data @/tmp/login http://${vault}/v1/auth/jwt/login | jq '.auth.client_token' -r)  || cleanup $spire_agent_pid 1
+application_token=$(curl -s --request POST --data @/tmp/login $vault/v1/auth/jwt/login | jq '.auth.client_token' -r)  || cleanup $spire_agent_pid 1
 
 echo "Getting container decryption key ..."
 
 # Use provided vault token (from login) to access secrets
-data_key=$(curl -s -H "X-Vault-Token: $application_token" http://${vault}/v1/kv/data/APPLICATION_SECRET_PATH | jq '.data.data.key' -r)    || cleanup $spire_agent_pid 1
+data_key=$(curl -s -H "X-Vault-Token: $application_token" $vault/v1/kv/data/APPLICATION_SECRET_PATH | jq '.data.data.key' -r)    || cleanup $spire_agent_pid 1
 echo "$data_key" > /tmp/container_key
 
 echo "Decrypting container image ..."
 
 # Decrypt the container image
-~/LUMI-secure-processing/client/container_preparation/input_logic/age --decrypt -i  /tmp/container_key -o $WORKING_DIRECTORY/app.sif APPLICATION_PATH || exit 1
+~/HPCS/client/container_preparation/input_logic/age --decrypt -i  /tmp/container_key -o $WORKING_DIRECTORY/app.sif APPLICATION_PATH || exit 1
 
 echo -e "${YELLOW}[LUMI-SD]${NC}${BLUE}[Job]${NC} Creating encrypted volumes"
 

utils/conf/client/conf.py

@@ -5,6 +5,9 @@ def parse_configuration(path : str):
     config = ConfigParser()
     config.read(path)
 
+    if not 'supercomputer' in config:
+        raise NoSectionError("supercomputer section missing in configuration file, aborting")
+    
     if not 'spire-server' in config:
         raise NoSectionError("hpcs-server section missing in configuration file, aborting")
 
@@ -14,6 +17,9 @@ def parse_configuration(path : str):
     if not 'vault' in config:
         raise NoSectionError("vault section missing in configuration file, aborting")
 
+    if not 'address' in config['supercomputer'] or not 'username' in config['supercomputer']:
+        raise NoOptionError("'spire-server' section is incomplete in configuration file, aborting")
+    
     if not 'address' in config['spire-server'] or not 'port' in config['spire-server'] or not 'trust-domain' in config['spire-server']:
         raise NoOptionError("'spire-server' section is incomplete in configuration file, aborting")
 

utils/ship_a_key.py

@@ -30,7 +30,8 @@ def parse_arguments() -> argparse.ArgumentParser:
     parser.add_argument(
         "--config",
         required=True,
-        help="Path to the client configuration file",
+        default="/tmp/hpcs-client.conf",
+        help="Configuration file (INI Format) (default: /tmp/hpcs-client.conf)",
     )
     parser.add_argument(
         "--users",

utils/spawn_agent.py

@@ -14,7 +14,8 @@ def parse_arguments():
     parser.add_argument(
         "--config",
         required=True,
-        help="Path to the client configuration file",
+        default="/tmp/hpcs-client.conf",
+        help="Configuration file (INI Format) (default: /tmp/hpcs-client.conf)",
     )
     parser.add_argument(
         "--socketpath",