diff --git a/.github/workflows/sbt_test.yml b/.github/workflows/sbt_test.yml index cadbdde36..b19ca35cf 100644 --- a/.github/workflows/sbt_test.yml +++ b/.github/workflows/sbt_test.yml @@ -50,14 +50,34 @@ jobs: - name: Set up sbt uses: sbt/setup-sbt@v1 - - name: Set up Scala + - name: Set up Scala (via Coursier) + uses: coursier/setup-action@v2 + with: + apps: scala:3.6.2 bloop + + - name: Warm up scala run: | - if [[ "${{ matrix.config.os }}" =~ ^macos.*$ ]]; then - brew install scala - else - sudo apt-get update - sudo apt-get install -y scala - fi + # start the bloop server, it'll download a lot of dependencies, so do it early + # also we need to start it before running any scala code, otherwise it will often timeout and cause the whole job to fail + bloop server & + for i in $(seq 1 30); do + timeout 5 bloop about 2>/dev/null && echo "Bloop is ready!" && break + echo "Waiting for Bloop... ($i/30)" + sleep 2 + done + + scala -version + scala -e 'println("Hello, Scala!")' + + - name: Set up .NET SDK + if: ${{ runner.os == 'Linux' }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '9.0.x' + + - name: Set up dotnet-script + if: ${{ runner.os == 'Linux' }} + run: dotnet tool install -g dotnet-script - name: Set up Python uses: actions/setup-python@v6 @@ -91,7 +111,7 @@ jobs: - name: Upload check results on fail if: failure() - uses: actions/upload-artifact@master + uses: actions/upload-artifact@v6 with: name: ${{ matrix.config.name }}_results path: check diff --git a/CHANGELOGS/CHANGELOG_0.10.md b/CHANGELOGS/CHANGELOG_0.10.md index 64e1884a6..813cb2d7e 100644 --- a/CHANGELOGS/CHANGELOG_0.10.md +++ b/CHANGELOGS/CHANGELOG_0.10.md @@ -10,6 +10,11 @@ TODO add summary This allows for running a component without having to build it first. Example: `viash run vsh://toolbox@v0.1.0/yq -- --input input.yaml --output output.yaml`. +* `Parameter passing`: Add support for unsetting argument values and computational requirements at runtime (PR #762, fixes #375). + * Pass the literal `UNDEFINED` (unquoted) to set a single-value argument to undefined/null: `./my_component --arg UNDEFINED` + * Pass `UNDEFINED_ITEM` as a value in a multi-value argument to represent a missing item: `./my_component --args "value1;UNDEFINED_ITEM;value3"` + * Unset computational requirements with `---cpus UNDEFINED` or `---memory UNDEFINED` + * Quote the values (`'"UNDEFINED"'` or `"'UNDEFINED'"`) to pass the literal string `UNDEFINED` instead of null. ## BREAKING CHANGES @@ -19,10 +24,25 @@ TODO add summary * Remove deprecated functionality `functionality` and `platforms` (PR #832). +* `Bash scripts`: Arguments with `multiple: true` are now stored as bash arrays instead of semicolon-separated strings (PR #762). + Update scripts to use array syntax: `for item in "${par_inputs[@]}"; do ...` instead of IFS splitting. + ## BUG FIXES * `NextflowRunner`: Automatically convert integers to doubles when argument type is `double` (port of PR #824, PR #825). +* `Parameter passing`: Fix handling of special characters in argument values (PR #762, fixes #619, #705, #763, #821, #840). + * Backticks in argument values no longer cause command substitution + * Backslash-quote sequences (`\'`) no longer break Python syntax + * Dollar signs, newlines, and other special characters are properly preserved + ## MINOR FIXES * `Executable`: Add more info to the --help (PR #802). + +## INTERNAL CHANGES + +* `Parameter passing`: Switch from code injection to JSON-based parameter passing (PR #762). + Instead of injecting argument values directly into script code, values are now stored in a JSON file + (`params.json`) and parsed at runtime using language-specific JSON parsers. This approach is more + robust, easier to debug, and handles special characters (backticks, quotes, newlines) correctly. diff --git a/Makefile.in b/Makefile.in index 6a279961c..c1b7415a4 100644 --- a/Makefile.in +++ b/Makefile.in @@ -10,7 +10,11 @@ $(obj): $(shell find src/ -name "*.scala") cat src/stub.sh target/viash.jar > $(obj) chmod +x $(obj) @echo "==================================> $(obj)" - + +touch: $(obj) + @echo "Touching src/main/scala/io/viash/Main.scala to trigger rebuild" + touch src/main/scala/io/viash/Main.scala + tools: $(obj) $(shell find src/viash) bin/viash ns build -s src/viash -t bin --flatten -c ".version := '$(VERSION)'" @echo "==================================> tools" diff --git a/docs/reference/viash_code_block/index.qmd b/docs/reference/viash_code_block/index.qmd index 752ba4c68..c0c923947 100644 --- a/docs/reference/viash_code_block/index.qmd +++ b/docs/reference/viash_code_block/index.qmd @@ -95,6 +95,28 @@ The `par` object (or `par_` environment variables in Bash) will contain argument Try adding more [arguments]({{< var reference.arguments >}}) with different types to see what effect this has on the resulting placeholder. ::: +### Special values: undefined and missing items + +When calling a Viash component, you can use special values to explicitly set arguments to undefined or represent missing items in multi-value arguments: + +* **Unsetting a single-value argument**: Pass the literal `UNDEFINED` (unquoted) to set a single-value argument to undefined/null: + ```bash + ./my_component --arg UNDEFINED + ``` + In the script, `par["arg"]` will be `None` (Python), `NULL` (R), `null` (JavaScript), or unset (Bash). + +* **Missing items in multi-value arguments**: When passing multiple values via semicolon-separated syntax, use `UNDEFINED_ITEM` to represent a missing element: + ```bash + ./my_component --values "item1;UNDEFINED_ITEM;item3" + ``` + In the script, `par["values"]` will be `["item1", None, "item3"]` (Python) or equivalent. + +* **Passing the literal string "UNDEFINED"**: Quote the value to pass it as a literal string: + ```bash + ./my_component --arg '"UNDEFINED"' # par["arg"] = "UNDEFINED" + ./my_component --arg "'UNDEFINED'" # par["arg"] = "UNDEFINED" + ``` + ## Meta variables in `meta` Meta-variables offer information on the runtime environment which you can use from within your script. @@ -215,6 +237,9 @@ viash test config.vsh.yaml --cpus 10 viash build config.vsh.yaml -o output output/my_executable ---cpus 10 # ↑ notice the triple dash + +# to unset a default cpu value, pass UNDEFINED +output/my_executable ---cpus UNDEFINED ``` ### `config` (string) @@ -267,6 +292,9 @@ viash test config.vsh.yaml --memory 2GB viash build config.vsh.yaml -o output output/my_executable ---memory 2GB # ↑ notice the triple dash + +# to unset a default memory value, pass UNDEFINED +output/my_executable ---memory UNDEFINED ``` ### `resources_dir` (string) diff --git a/src/main/resources/io/viash/helpers/bashutils/ViashCleanupRegistry.sh b/src/main/resources/io/viash/helpers/bashutils/ViashCleanupRegistry.sh new file mode 100644 index 000000000..1df5ac07f --- /dev/null +++ b/src/main/resources/io/viash/helpers/bashutils/ViashCleanupRegistry.sh @@ -0,0 +1,30 @@ +# Cleanup handler registry +# Allows multiple cleanup functions to be registered and called on exit + +# Array to store registered cleanup function names +VIASH_CLEANUP_HANDLERS=() + +# ViashRegisterCleanup: Register a cleanup function to be called on exit +# $1: Name of the function to call during cleanup +# usage: ViashRegisterCleanup my_cleanup_function +function ViashRegisterCleanup { + local handler="$1" + VIASH_CLEANUP_HANDLERS+=("$handler") +} + +# ViashRunCleanupHandlers: Run all registered cleanup handlers in reverse order +# This function is meant to be used as the EXIT trap handler +function ViashRunCleanupHandlers { + # Run handlers in reverse order (LIFO - last registered runs first) + local i + for (( i=${#VIASH_CLEANUP_HANDLERS[@]}-1 ; i>=0 ; i-- )); do + local handler="${VIASH_CLEANUP_HANDLERS[$i]}" + if type "$handler" &>/dev/null; then + ViashDebug "Running cleanup handler: $handler" + "$handler" + fi + done +} + +# Set up the master EXIT trap that runs all registered handlers +trap ViashRunCleanupHandlers EXIT diff --git a/src/main/resources/io/viash/helpers/bashutils/ViashDockerFuns.sh b/src/main/resources/io/viash/helpers/bashutils/ViashDockerFuns.sh index dbad31070..a9f65f298 100644 --- a/src/main/resources/io/viash/helpers/bashutils/ViashDockerFuns.sh +++ b/src/main/resources/io/viash/helpers/bashutils/ViashDockerFuns.sh @@ -189,7 +189,7 @@ function ViashDockerSetup { # $1 : image identifier with format `[registry/]image[:tag]` # $@ : commands to verify being present # examples: -# ViashDockerCheckCommands bash:4.0 bash ps foo +# ViashDockerCheckCommands bash:3.2 bash ps foo function ViashDockerCheckCommands { local image_id="$1" shift 1 @@ -220,10 +220,13 @@ function ViashDockerBuild { # create temporary directory to store dockerfile & optional resources in local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") local dockerfile="$tmpdir/Dockerfile" - function clean_up { - rm -rf "$tmpdir" + + # Use a unique cleanup function name to avoid conflicts + VIASH_DOCKER_BUILD_TMPDIR="$tmpdir" + function ViashDockerBuildCleanup { + rm -rf "$VIASH_DOCKER_BUILD_TMPDIR" } - trap clean_up EXIT + ViashRegisterCleanup ViashDockerBuildCleanup # store dockerfile and resources ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" diff --git a/src/main/resources/io/viash/helpers/bashutils/ViashParseArgumentValue.sh b/src/main/resources/io/viash/helpers/bashutils/ViashParseArgumentValue.sh new file mode 100644 index 000000000..924774eb3 --- /dev/null +++ b/src/main/resources/io/viash/helpers/bashutils/ViashParseArgumentValue.sh @@ -0,0 +1,191 @@ +# ViashParseArgumentValue: Parse the value of an argument +# +# This script is used by to parse the value of an argument and set +# the corresponding environment variable. +# +# If the argument is multiple: false: +# +# * If the variable is already set, an error is thrown. +# * If the value is equal to UNDEFINED, it is replaced with +# @@VIASH_UNDEFINED@@. +# * If the value is quoted, the quotes are removed. +# +# If the argument is multiple: true: +# +# * If the value is equal to UNDEFINED, it is replaced with +# @@VIASH_UNDEFINED@@. +# * If the value is quoted, the quotes are removed. +# * If the value is a list of values, the values are split by semicolons. +# * If the value contains escaped characters '"\;, they are unescaped. +# +# Arguments: +# $1: The name of an argument (used for error messages) +# $2: The name of the environment variable to set +# $3: Whether the argument can be passed multiple times (true/false) +# $4: The value of the argument +# return: None, but sets the environment variable +# +# Examples: +# +# ViashParseArgumentValue "--input" "par_input" "true" "'UNDEFINED'" "false" +# +# See ViashParseArgumentValue.test.sh for additional examples. +# +# Note: This function is written to be compatible with bash 3.2 (macOS default) +# and avoids bash 4+ features like declare -g, local -n, ${var@Q}, and readarray. +function ViashParseArgumentValue { + local flag="$1" + local env_name="$2" + local multiple="$3" + local value="$4" + + if [ $# -lt 4 ]; then + ViashError "Not enough arguments passed to ${flag}. Use '--help' to get more information on the parameters." + exit 1 + fi + + if [ "$multiple" == "false" ]; then + # check whether the variable is already set (bash 3.2 compatible) + eval "local is_set=\${${env_name}+x}" + if [ ! -z "$is_set" ]; then + eval "local prev_value=\"\${$env_name}\"" + ViashError "Pass only one argument to argument '${flag}'. Found: '${prev_value}' & '${value}'" + exit 1 + fi + + value=$(ViashParseSingleString "$value") + + # set the variable globally (bash 3.2 compatible - using eval instead of declare -g) + # use printf %q to safely escape the value for eval to prevent backtick/command substitution + local escaped_value + escaped_value=$(printf '%q' "$value") + eval "$env_name=$escaped_value" + else + # Get existing array values (bash 3.2 compatible) + eval "local prev_values=(\"\${${env_name}[@]}\")" + + # Parse new values into array (bash 3.2 compatible - using while loop instead of readarray) + local new_values=() + while IFS= read -r line || [ -n "$line" ]; do + new_values+=("$line") + done < <(ViashParseMultipleStringAsArray "$value") + + local combined_values=( "${prev_values[@]}" "${new_values[@]}" ) + + # if length is larger than 1 and some element is @@VIASH_UNDEFINED@@, throw error + if [ ${#combined_values[@]} -gt 1 ]; then + for element in "${combined_values[@]}"; do + if [ "$element" == "@@VIASH_UNDEFINED@@" ]; then + # bash 3.2 compatible quoting (using printf %q instead of ${var@Q}) + local combined_quoted=$(printf '%q ' "${combined_values[@]}") + ViashError "Argument '${flag}': If argument value 'UNDEFINED' is passed, no other values should be provided.\nFound: ${combined_quoted}" + exit 1 + fi + done + fi + + # Set the global array (bash 3.2 compatible - using eval instead of declare -g -a) + eval "$env_name=(\"\${combined_values[@]}\")" + fi +} + +function ViashParseSingleString() { + local value="$1" + + # if value is equal to UNDEFINED, replace with @@VIASH_UNDEFINED@@ + if [ "$value" == "UNDEFINED" ]; then + value="@@VIASH_UNDEFINED@@" + fi + + # if value is quoted, remove the quotes + if [[ "$value" =~ ^\".*\"$ ]]; then + value="${value:1:${#value}-2}" + elif [[ "$value" =~ ^\'.*\'$ ]]; then + value="${value:1:${#value}-2}" + fi + + echo "$value" +} + +function ViashParseMultipleStringAsArray() { + local value="$1" + + # if value is equal to UNDEFINED, replace with @@VIASH_UNDEFINED@@ + if [ "$value" == "UNDEFINED" ]; then + echo "@@VIASH_UNDEFINED@@" + return + fi + + # if value is empty, return nothing (results in 0-length array) + if [ -z "$value" ]; then + return + fi + + # Parse semicolon-separated values with proper quote handling + # This is a bash-native implementation that doesn't rely on gawk's FPAT + # (which isn't available in BSD awk on macOS or BusyBox awk) + local i=0 + local len=${#value} + local in_double_quote=false + local in_single_quote=false + local escape_next=false + local current_field="" + local field_was_quoted=false # Track if field started with a quote + local char + + while [ $i -lt $len ]; do + char="${value:$i:1}" + + if $escape_next; then + # Previous char was backslash - add escaped char to field + current_field+="$char" + escape_next=false + elif [ "$char" = "\\" ]; then + # Backslash - escape next character + escape_next=true + elif [ "$char" = '"' ] && ! $in_single_quote; then + # Toggle double quote state (unless we're in single quotes) + if $in_double_quote; then + in_double_quote=false + else + in_double_quote=true + # Mark field as quoted if this is the first char + if [ -z "$current_field" ]; then + field_was_quoted=true + fi + fi + elif [ "$char" = "'" ] && ! $in_double_quote; then + # Toggle single quote state (unless we're in double quotes) + if $in_single_quote; then + in_single_quote=false + else + in_single_quote=true + # Mark field as quoted if this is the first char + if [ -z "$current_field" ]; then + field_was_quoted=true + fi + fi + elif [ "$char" = ";" ] && ! $in_double_quote && ! $in_single_quote; then + # Semicolon outside quotes - end of field + # Handle UNDEFINED_ITEM replacement (only if not quoted) + if [ "$current_field" == "UNDEFINED_ITEM" ] && ! $field_was_quoted; then + current_field="@@VIASH_UNDEFINED_ITEM@@" + fi + echo "$current_field" + current_field="" + field_was_quoted=false + else + # Regular character - add to field + current_field+="$char" + fi + + i=$((i + 1)) + done + + # Don't forget the last field (after the last semicolon or if no semicolon) + # Handle UNDEFINED_ITEM replacement (only if not quoted) + if [ "$current_field" == "UNDEFINED_ITEM" ] && ! $field_was_quoted; then + current_field="@@VIASH_UNDEFINED_ITEM@@" + fi + echo "$current_field" +} diff --git a/src/main/resources/io/viash/helpers/bashutils/ViashRenderJson.sh b/src/main/resources/io/viash/helpers/bashutils/ViashRenderJson.sh new file mode 100644 index 000000000..1b96010d7 --- /dev/null +++ b/src/main/resources/io/viash/helpers/bashutils/ViashRenderJson.sh @@ -0,0 +1,114 @@ +# ViashRenderJsonKeyValue: renders a key-value pair in JSON format +# +# Arguments: +# $1: The key +# $2: The type of the value (string, boolean, boolean_true, boolean_false, file, double, integer) +# $3: Whether the value can be passed multiple times (true/false) +# $4+: The value(s) of the key +# return: prints the key-value pair in JSON format +# +# Examples: +# +# ViashRenderJsonKeyValue "input" "string" "false" "file.txt" +# ViashRenderJsonKeyValue "input" "string" "true" "file1.txt" "file2.txt" +function ViashRenderJsonKeyValue { + local key="$1" + local type="$2" + local multiple="$3" + shift 3 + + local out=" \"${key}\":" + + # Handle null case + if [ $# -eq 1 ] && [ "$1" == "@@VIASH_UNDEFINED@@" ]; then + out+=" null" + echo "$out" + return + fi + + # Handle multiple values (array) + if [ "$multiple" == "true" ]; then + out+=" [" + local first=true + while [ $# -gt 0 ]; do + if [ "$first" == "false" ]; then + out+="," + fi + first=false + + if [ "$1" == "@@VIASH_UNDEFINED_ITEM@@" ]; then + out+=" null" + elif [ "$type" == "string" ] || [ "$type" == "file" ]; then + out+=" $(ViashRenderJsonQuotedValue "$key" "$1")" + elif [ "$type" == "boolean" ] || [ "$type" == "boolean_true" ] || [ "$type" == "boolean_false" ]; then + out+=" $(ViashRenderJsonBooleanValue "$key" "$1")" + else + out+=" $(ViashRenderJsonUnquotedValue "$key" "$1")" + fi + shift + done + out+=" ]" + else + # Handle single value + out+=" " + if [ "$1" == "@@VIASH_UNDEFINED_ITEM@@" ]; then + out+="null" + elif [ "$type" == "string" ] || [ "$type" == "file" ]; then + out+="$(ViashRenderJsonQuotedValue "$key" "$1")" + elif [ "$type" == "boolean" ] || [ "$type" == "boolean_true" ] || [ "$type" == "boolean_false" ]; then + out+="$(ViashRenderJsonBooleanValue "$key" "$1")" + else + out+="$(ViashRenderJsonUnquotedValue "$key" "$1")" + fi + fi + + echo "$out" +} + +function ViashRenderJsonQuotedValue { + local key="$1" + local value="$2" + + # Handle empty string case + if [ -z "$value" ]; then + echo '""' + return + fi + + # escape backslashes, quotes, and newlines for JSON + # Note: Uses awk instead of sed for newline replacement for BSD/macOS compatibility + echo "$value" | \ + sed 's#\\#\\\\#g' | \ + sed 's#"#\\"#g' | \ + awk 'BEGIN{ORS="\\n"} {print}' | \ + sed 's#\\n$##' | \ + sed 's#^#"#g;s#$#"#g' +} + +function ViashRenderJsonBooleanValue { + local key="$1" + local value="$2" + # convert to lowercase + value=$(echo "$value" | tr '[:upper:]' '[:lower:]') + if [[ "$value" == "true" || "$value" == "yes" ]]; then + echo "true" + elif [[ "$value" == "false" || "$value" == "no" ]]; then + echo "false" + else + echo "Argument '$key' has to be a boolean, but got '$value'. Use '--help' to get more information on the parameters." >&2 + exit 1 + fi +} + +function ViashRenderJsonUnquotedValue { + local key="$1" + local value="$2" + + # Validate that the value is a valid number (integer, decimal, or scientific notation) + if ! [[ "$value" =~ ^-?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?$ ]]; then + echo "Argument '$key' has to be a number, but got '$value'. Use '--help' to get more information on the parameters." >&2 + exit 1 + fi + + echo "$value" +} diff --git a/src/main/resources/io/viash/languages/bash/ViashParseJson.sh b/src/main/resources/io/viash/languages/bash/ViashParseJson.sh new file mode 100644 index 000000000..51b3df4d2 --- /dev/null +++ b/src/main/resources/io/viash/languages/bash/ViashParseJson.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# ViashParseJsonBash: Parse JSON parameters into Bash variables using jq +# +# Reads JSON from stdin and sets variables for each key-value pair. +# +# Structure: +# - Top-level keys mapping to objects are treated as sections, +# with variables named "section_key" (e.g., par_input, meta_name). +# - Top-level scalar/array keys are set directly. +# - Arrays become Bash arrays. +# - Deep nesting (depth 3+) is stored as JSON strings. +# - Null values leave the variable unset. +# +# Requires: jq (https://jqlang.github.io/jq/) +# +# Usage: +# ViashParseJsonBash < json_file +# ViashParseJsonBash <<< "$json_content" + +function ViashParseJsonBash { + local _viash_json + _viash_json="$(cat)" + + if [ -z "$_viash_json" ]; then + return 0 + fi + + # Verify jq is available + if ! command -v jq &>/dev/null; then + echo "ViashParseJsonBash: jq is required but not found. Install jq or set 'use_jq: false' in your bash_script resource." >&2 + return 1 + fi + + local _viash_assignments + _viash_assignments="$(printf '%s' "$_viash_json" | jq -r ' + # Generate a bash assignment for a variable. + # For arrays: varname=(elem1 elem2 ...) + # For scalars: varname=value + # For null: no output (variable left unset) + def to_bash_assignment(varname): + if type == "null" then + empty + elif type == "array" then + varname + "=(" + ([.[] | + if type == "null" then "\"null\"" + elif type == "boolean" or type == "number" then tostring | @sh + elif type == "string" then @sh + elif type == "object" or type == "array" then tojson | @sh + else tojson | @sh + end + ] | join(" ")) + ")" + elif type == "boolean" or type == "number" then + varname + "=" + tostring + elif type == "string" then + varname + "=" + @sh + elif type == "object" then + varname + "=" + (tojson | @sh) + else + varname + "=" + (tojson | @sh) + end; + + to_entries[] | + if .value | type == "object" then + # Section: iterate keys and produce section_key assignments + .key as $section | + .value | to_entries[] | + .key as $key | + .value | to_bash_assignment("\($section)_\($key)") + else + # Direct top-level key + .key as $key | + .value | to_bash_assignment($key) + end + ')" || { + echo "ViashParseJsonBash: jq failed to parse JSON input" >&2 + return 1 + } + + if [ -n "$_viash_assignments" ]; then + eval "$_viash_assignments" + fi +} diff --git a/src/main/resources/io/viash/languages/bash/ViashParseJsonCompatibility.sh b/src/main/resources/io/viash/languages/bash/ViashParseJsonCompatibility.sh new file mode 100644 index 000000000..7bc811e32 --- /dev/null +++ b/src/main/resources/io/viash/languages/bash/ViashParseJsonCompatibility.sh @@ -0,0 +1,560 @@ +#!/usr/bin/env bash + +# ViashParseJsonBash: Parse JSON parameters into Bash variables +# +# Recursive descent JSON parser (similar to the Scala ViashJsonParser). +# Reads JSON from stdin and sets variables for each key-value pair. +# +# Structure: +# - Top-level keys mapping to objects are treated as sections, +# with variables named "section_key" (e.g., par_input, meta_name). +# - Top-level scalar/array keys are set directly. +# - Arrays become Bash arrays. +# - Deep nesting (depth 3+) is stored as JSON strings. +# +# Usage: +# ViashParseJsonBash < json_file +# ViashParseJsonBash <<< "$json_content" +# +# Note: Compatible with bash 3.2 (macOS default). +# Avoids bash 4+ features like declare -g, local -n, and readarray. + +# -- Parser state (globals for bash 3.2 compatibility) -- +_viash_json="" # Full JSON input string +_viash_chars=() # Characters array for O(1) indexing +_viash_len=0 # Length of input +_viash_pos=0 # Current parse position +_viash_result="" # Result of last parse operation + +# ViashParseJsonBash: entry point -- reads stdin and parses top-level object +function ViashParseJsonBash { + _viash_json="$(cat)" + _viash_pos=0 + _viash_len=${#_viash_json} + _viash_result="" + + # Pre-split into character array for O(1) access. + # Using fold -w1 to split efficiently + _viash_chars=() + if [ $_viash_len -gt 0 ]; then + local IFS=$'\n' + # fold -w1 splits each character onto its own line. + # We need to preserve empty lines (which represent newlines in the input) + # by using a read loop instead of command substitution word splitting. + while IFS= read -r _viash_char || [ -n "$_viash_char" ]; do + _viash_chars+=("$_viash_char") + done < <(printf '%s' "$_viash_json" | fold -w1) + fi + + _viash_skip_whitespace + _viash_parse_toplevel_object +} + +# Helper: extract a substring from the character array. +# Usage: _viash_substr start length +# Sets _viash_result to the extracted substring. +function _viash_substr { + local start=$1 len=$2 + local end=$((start + len)) + local out="" + local i + for ((i=start; i&2 + exit 1 + fi + _viash_result="${_viash_chars[$_viash_pos]}" +} + +# Consume an expected character, or exit with error. +function _viash_consume { + local expected="$1" + _viash_skip_whitespace + if [ $_viash_pos -ge $_viash_len ]; then + echo "ViashParseJsonBash: Expected '$expected' but reached end of JSON" >&2 + exit 1 + fi + local actual="${_viash_chars[$_viash_pos]}" + if [ "$actual" != "$expected" ]; then + echo "ViashParseJsonBash: Expected '$expected' at position $_viash_pos, got '$actual'" >&2 + exit 1 + fi + ((_viash_pos++)) || true +} + +# -- Core parse functions -- + +# Parse a JSON string. Sets _viash_result to the unescaped string content. +function _viash_parse_string { + _viash_consume '"' + local result="" + while [ $_viash_pos -lt $_viash_len ]; do + local char="${_viash_chars[$_viash_pos]}" + if [ "$char" = '"' ]; then + ((_viash_pos++)) || true + _viash_result="$result" + return + elif [ "$char" = '\' ]; then + ((_viash_pos++)) || true + if [ $_viash_pos -ge $_viash_len ]; then + echo "ViashParseJsonBash: Unterminated string escape at position $_viash_pos" >&2 + exit 1 + fi + local esc="${_viash_chars[$_viash_pos]}" + case "$esc" in + # Note: \n and \t are kept as literal two-character sequences (\n, \t) + # to match Viash YAML parser behavior and test expectations + 'n') result+='\n' ;; + 't') result+='\t' ;; + 'r') result+=$'\r' ;; + 'b') result+=$'\b' ;; + 'f') result+=$'\f' ;; + '\\') result+='\' ;; + '"') result+='"' ;; + '/') result+='/' ;; + 'u') + # Unicode escape \uXXXX + _viash_substr $((_viash_pos+1)) 4 + local hex="$_viash_result" + if [ ${#hex} -lt 4 ]; then + echo "ViashParseJsonBash: Invalid unicode escape at position $_viash_pos" >&2 + exit 1 + fi + result+="$(printf "\\$(printf '%03o' "0x$hex")")" + _viash_pos=$((_viash_pos + 4)) + ;; + *) + # Unknown escape - keep as-is + result+="$esc" ;; + esac + else + result+="$char" + fi + ((_viash_pos++)) || true + done + echo "ViashParseJsonBash: Unterminated string starting near position $_viash_pos" >&2 + exit 1 +} + +# Parse a JSON number. Sets _viash_result to the number string. +function _viash_parse_number { + local start=$_viash_pos + # Optional minus + if [ "${_viash_chars[$_viash_pos]}" = '-' ]; then + ((_viash_pos++)) || true + fi + # Integer digits + while [ $_viash_pos -lt $_viash_len ] && [[ "${_viash_chars[$_viash_pos]}" =~ [0-9] ]]; do + ((_viash_pos++)) || true + done + # Decimal part + if [ $_viash_pos -lt $_viash_len ] && [ "${_viash_chars[$_viash_pos]}" = '.' ]; then + ((_viash_pos++)) || true + while [ $_viash_pos -lt $_viash_len ] && [[ "${_viash_chars[$_viash_pos]}" =~ [0-9] ]]; do + ((_viash_pos++)) || true + done + fi + # Exponent part + if [ $_viash_pos -lt $_viash_len ]; then + local ec="${_viash_chars[$_viash_pos]}" + if [ "$ec" = 'e' ] || [ "$ec" = 'E' ]; then + ((_viash_pos++)) || true + if [ $_viash_pos -lt $_viash_len ]; then + local sign="${_viash_chars[$_viash_pos]}" + if [ "$sign" = '+' ] || [ "$sign" = '-' ]; then + ((_viash_pos++)) || true + fi + fi + while [ $_viash_pos -lt $_viash_len ] && [[ "${_viash_chars[$_viash_pos]}" =~ [0-9] ]]; do + ((_viash_pos++)) || true + done + fi + fi + _viash_substr $start $((_viash_pos - start)) +} + +# Parse a JSON boolean (true/false). Sets _viash_result. +function _viash_parse_boolean { + _viash_substr $_viash_pos 5 + local word="$_viash_result" + if [ "${word:0:4}" = "true" ]; then + _viash_pos=$((_viash_pos + 4)) + _viash_result="true" + elif [ "$word" = "false" ]; then + _viash_pos=$((_viash_pos + 5)) + _viash_result="false" + else + echo "ViashParseJsonBash: Invalid boolean at position $_viash_pos" >&2 + exit 1 + fi +} + +# Parse a JSON null. Sets _viash_result to empty string. +function _viash_parse_null { + _viash_substr $_viash_pos 4 + if [ "$_viash_result" = "null" ]; then + _viash_pos=$((_viash_pos + 4)) + _viash_result="" + else + echo "ViashParseJsonBash: Invalid null at position $_viash_pos" >&2 + exit 1 + fi +} + +# Skip over any JSON value without storing it (for values we don't need). +# Correctly handles nested structures. +function _viash_skip_value { + _viash_peek + case "$_viash_result" in + '"') _viash_parse_string ;; + '{') _viash_skip_object ;; + '[') _viash_skip_array ;; + 't'|'f') _viash_parse_boolean ;; + 'n') _viash_parse_null ;; + '-'|[0-9]) _viash_parse_number ;; + *) + echo "ViashParseJsonBash: Unexpected character '$_viash_result' at position $_viash_pos" >&2 + exit 1 + ;; + esac +} + +function _viash_skip_object { + _viash_consume '{' + _viash_peek + if [ "$_viash_result" = '}' ]; then + _viash_consume '}' + return + fi + while true; do + _viash_parse_string # key + _viash_consume ':' + _viash_skip_value # value + _viash_peek + if [ "$_viash_result" = ',' ]; then + _viash_consume ',' + else + break + fi + done + _viash_consume '}' +} + +function _viash_skip_array { + _viash_consume '[' + _viash_peek + if [ "$_viash_result" = ']' ]; then + _viash_consume ']' + return + fi + while true; do + _viash_skip_value + _viash_peek + if [ "$_viash_result" = ',' ]; then + _viash_consume ',' + else + break + fi + done + _viash_consume ']' +} + +# -- Serialization functions (for deep nesting stored as JSON strings) -- + +# Serialize the JSON value at the current position back into a JSON string. +# Sets _viash_result to the JSON fragment. +function _viash_serialize_value { + _viash_peek + case "$_viash_result" in + '"') _viash_serialize_string ;; + '{') _viash_serialize_object ;; + '[') _viash_serialize_array ;; + 't'|'f') + _viash_parse_boolean + ;; + 'n') + _viash_parse_null + _viash_result="null" + ;; + '-'|[0-9]) + _viash_parse_number + ;; + *) + echo "ViashParseJsonBash: Unexpected character '$_viash_result' at position $_viash_pos" >&2 + exit 1 + ;; + esac +} + +# Serialize a JSON string (keeps it in JSON-encoded form with quotes). +function _viash_serialize_string { + local start=$_viash_pos + _viash_consume '"' + while [ $_viash_pos -lt $_viash_len ]; do + local char="${_viash_chars[$_viash_pos]}" + if [ "$char" = '"' ]; then + ((_viash_pos++)) || true + _viash_substr $start $((_viash_pos - start)) + return + elif [ "$char" = '\' ]; then + # Skip escape sequence + ((_viash_pos++)) || true + fi + ((_viash_pos++)) || true + done + echo "ViashParseJsonBash: Unterminated string at position $start" >&2 + exit 1 +} + +function _viash_serialize_object { + local out="{" + _viash_consume '{' + _viash_peek + if [ "$_viash_result" = '}' ]; then + _viash_consume '}' + _viash_result="{}" + return + fi + local first=true + while true; do + if [ "$first" = "false" ]; then + out+="," + fi + first=false + _viash_serialize_string + out+="$_viash_result" + _viash_consume ':' + out+=":" + _viash_serialize_value + out+="$_viash_result" + _viash_peek + if [ "$_viash_result" = ',' ]; then + _viash_consume ',' + else + break + fi + done + _viash_consume '}' + out+="}" + _viash_result="$out" +} + +function _viash_serialize_array { + local out="[" + _viash_consume '[' + _viash_peek + if [ "$_viash_result" = ']' ]; then + _viash_consume ']' + _viash_result="[]" + return + fi + local first=true + while true; do + if [ "$first" = "false" ]; then + out+="," + fi + first=false + _viash_serialize_value + out+="$_viash_result" + _viash_peek + if [ "$_viash_result" = ',' ]; then + _viash_consume ',' + else + break + fi + done + _viash_consume ']' + out+="]" + _viash_result="$out" +} + +# -- Top-level parsing: maps JSON structure to Bash variables -- + +# Parse the root object. Handles two patterns: +# 1. Key -> object: treated as a section (variables named section_key) +# 2. Key -> scalar/array: set directly as variable +function _viash_parse_toplevel_object { + _viash_consume '{' + _viash_peek + if [ "$_viash_result" = '}' ]; then + _viash_consume '}' + return + fi + + while true; do + _viash_parse_string + local key="$_viash_result" + _viash_consume ':' + + _viash_peek + case "$_viash_result" in + '{') + _viash_parse_section_object "$key" + ;; + '[') + _viash_parse_and_assign_array "$key" + ;; + *) + _viash_parse_and_assign_scalar "$key" + ;; + esac + + _viash_peek + if [ "$_viash_result" = ',' ]; then + _viash_consume ',' + else + break + fi + done + _viash_consume '}' +} + +# Parse a section object (depth 2). Each key becomes a variable "section_key". +function _viash_parse_section_object { + local section="$1" + _viash_consume '{' + _viash_peek + if [ "$_viash_result" = '}' ]; then + _viash_consume '}' + return + fi + + while true; do + _viash_parse_string + local key="$_viash_result" + local var_name="${section}_${key}" + _viash_consume ':' + + _viash_peek + case "$_viash_result" in + '{') + # Depth 3+ object -> serialize and store as JSON string + _viash_serialize_object + printf -v _viash_escaped '%q' "$_viash_result" + eval "${var_name}=$_viash_escaped" + ;; + '[') + _viash_parse_and_assign_array "$var_name" + ;; + *) + _viash_parse_and_assign_scalar "$var_name" + ;; + esac + + _viash_peek + if [ "$_viash_result" = ',' ]; then + _viash_consume ',' + else + break + fi + done + _viash_consume '}' +} + +# Parse a scalar value and assign it to the given variable name. +# Null values leave the variable unset. +function _viash_parse_and_assign_scalar { + local var_name="$1" + _viash_peek + case "$_viash_result" in + '"') + _viash_parse_string + printf -v _viash_escaped '%q' "$_viash_result" + eval "${var_name}=$_viash_escaped" + ;; + 't'|'f') + _viash_parse_boolean + eval "${var_name}=$_viash_result" + ;; + 'n') + _viash_parse_null + # Leave variable unset for null + ;; + '-'|[0-9]) + _viash_parse_number + eval "${var_name}=$_viash_result" + ;; + *) + echo "ViashParseJsonBash: Unexpected value '$_viash_result' at position $_viash_pos" >&2 + exit 1 + ;; + esac +} + +# Parse a JSON array and assign it to the given variable name as a Bash array. +function _viash_parse_and_assign_array { + local var_name="$1" + _viash_consume '[' + _viash_peek + if [ "$_viash_result" = ']' ]; then + _viash_consume ']' + eval "${var_name}=()" + return + fi + + local items=() + while true; do + _viash_peek + case "$_viash_result" in + '"') + _viash_parse_string + items+=("$_viash_result") + ;; + 't'|'f') + _viash_parse_boolean + items+=("$_viash_result") + ;; + 'n') + _viash_parse_null + items+=("null") + ;; + '-'|[0-9]) + _viash_parse_number + items+=("$_viash_result") + ;; + '{') + _viash_serialize_object + items+=("$_viash_result") + ;; + '[') + _viash_serialize_array + items+=("$_viash_result") + ;; + *) + echo "ViashParseJsonBash: Unexpected array element '$_viash_result' at position $_viash_pos" >&2 + exit 1 + ;; + esac + + _viash_peek + if [ "$_viash_result" = ',' ]; then + _viash_consume ',' + else + break + fi + done + _viash_consume ']' + + # Assign array (bash 3.2 compatible) + eval "${var_name}=($(printf '%q ' "${items[@]}"))" +} diff --git a/src/main/resources/io/viash/languages/csharp/ViashParseJson.csx b/src/main/resources/io/viash/languages/csharp/ViashParseJson.csx new file mode 100644 index 000000000..d7161f1ee --- /dev/null +++ b/src/main/resources/io/viash/languages/csharp/ViashParseJson.csx @@ -0,0 +1,86 @@ +/* + * Parse JSON parameters file into a C# Dictionary + */ + +using System; +using System.IO; +using System.Collections.Generic; +using System.Text.Json; + +public static class ViashJsonParser +{ + /// + /// Parse JSON parameters file into a C# Dictionary. + /// + /// Path to the JSON file. If null, reads from $VIASH_WORK_PARAMS environment variable. + /// Dictionary containing the parsed JSON data + public static Dictionary ParseJson(string jsonPath = null) + { + if (jsonPath == null) + { + jsonPath = Environment.GetEnvironmentVariable("VIASH_WORK_PARAMS"); + if (string.IsNullOrEmpty(jsonPath)) + { + throw new InvalidOperationException("VIASH_WORK_PARAMS environment variable not set"); + } + } + + if (!File.Exists(jsonPath)) + { + throw new FileNotFoundException($"Parameters file not found: {jsonPath}"); + } + + try + { + string jsonText = File.ReadAllText(jsonPath); + using var document = JsonDocument.Parse(jsonText); + return ConvertJsonElement(document.RootElement) as Dictionary; + } + catch (JsonException ex) + { + throw new InvalidOperationException($"Error parsing JSON file: {ex.Message}", ex); + } + } + + private static object ConvertJsonElement(JsonElement element) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + var dict = new Dictionary(); + foreach (var prop in element.EnumerateObject()) + { + dict[prop.Name] = ConvertJsonElement(prop.Value); + } + return dict; + + case JsonValueKind.Array: + var list = new List(); + foreach (var item in element.EnumerateArray()) + { + list.Add(ConvertJsonElement(item)); + } + return list; + + case JsonValueKind.String: + return element.GetString(); + + case JsonValueKind.Number: + if (element.TryGetInt32(out int intValue)) + return intValue; + return element.GetDouble(); + + case JsonValueKind.True: + return true; + + case JsonValueKind.False: + return false; + + case JsonValueKind.Null: + return null; + + default: + return element.ToString(); + } + } +} diff --git a/src/main/resources/io/viash/languages/javascript/ViashParseJson.js b/src/main/resources/io/viash/languages/javascript/ViashParseJson.js new file mode 100644 index 000000000..f5f84d897 --- /dev/null +++ b/src/main/resources/io/viash/languages/javascript/ViashParseJson.js @@ -0,0 +1,29 @@ +const _viashFs = require('fs'); + +/** + * Parse JSON parameters file into a JavaScript object. + * + * @param {string|null} jsonPath - Path to the JSON file. If null, reads from $VIASH_WORK_PARAMS environment variable. + * @returns {Object} Parsed JSON data. + */ +function viashParseJson(jsonPath = null) { + if (jsonPath === null) { + jsonPath = process.env.VIASH_WORK_PARAMS; + if (!jsonPath) { + throw new Error("VIASH_WORK_PARAMS environment variable not set"); + } + } + + if (!_viashFs.existsSync(jsonPath)) { + throw new Error(`Parameters file not found: ${jsonPath}`); + } + + try { + const jsonText = _viashFs.readFileSync(jsonPath, 'utf8'); + return JSON.parse(jsonText); + } catch (error) { + throw new Error(`Error parsing JSON file: ${error.message}`); + } +} + +module.exports = { viashParseJson }; diff --git a/src/main/resources/io/viash/languages/python/ViashParseJson.py b/src/main/resources/io/viash/languages/python/ViashParseJson.py new file mode 100644 index 000000000..0b18b16c0 --- /dev/null +++ b/src/main/resources/io/viash/languages/python/ViashParseJson.py @@ -0,0 +1,26 @@ +import json +import sys + +def viash_parse_json(json_path=None): + """ + Parse JSON parameters file into a Python dictionary. + + Args: + json_path: Path to the JSON file. If None, reads from $VIASH_WORK_PARAMS environment variable. + + Returns: + Dictionary containing the parsed JSON data. + """ + if json_path is None: + import os + json_path = os.environ.get('VIASH_WORK_PARAMS') + if json_path is None: + raise ValueError("VIASH_WORK_PARAMS environment variable not set") + + try: + with open(json_path, 'r') as f: + return json.load(f) + except FileNotFoundError: + raise FileNotFoundError(f"Parameters file not found: {json_path}") + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON in parameters file: {e}") diff --git a/src/main/resources/io/viash/languages/r/ViashParseJson.R b/src/main/resources/io/viash/languages/r/ViashParseJson.R new file mode 100644 index 000000000..9cef556e1 --- /dev/null +++ b/src/main/resources/io/viash/languages/r/ViashParseJson.R @@ -0,0 +1,36 @@ +#' Parse JSON parameters file into an R list using jsonlite +#' +#' Uses the jsonlite package for JSON parsing. Requires jsonlite to be installed. +#' +#' @param json_path Path to the JSON file. If NULL, reads from +#' $VIASH_WORK_PARAMS environment variable. +#' @return Named list containing the parsed JSON data. +viash_parse_json <- function(json_path = NULL) { + if (is.null(json_path)) { + json_path <- Sys.getenv("VIASH_WORK_PARAMS") + if (json_path == "") { + stop("VIASH_WORK_PARAMS environment variable not set") + } + } + + if (!file.exists(json_path)) { + stop(paste0("Parameters file not found: ", json_path)) + } + + if (!requireNamespace("jsonlite", quietly = TRUE)) { + stop( + "The 'jsonlite' R package is required but not installed. ", + "Please install it with: install.packages('jsonlite')" + ) + } + + tryCatch({ + json_text <- readLines(json_path, warn = FALSE) + json_text <- paste(json_text, collapse = "\n") + # Use bigint_as_char = TRUE to preserve large integers as strings + # This prevents precision loss for values > 2^53 + jsonlite::parse_json(json_text, simplifyVector = TRUE, bigint_as_char = TRUE) + }, error = function(e) { + stop(paste0("Error parsing JSON file: ", e$message)) + }) +} diff --git a/src/main/resources/io/viash/languages/r/ViashParseJsonHybrid.R b/src/main/resources/io/viash/languages/r/ViashParseJsonHybrid.R new file mode 100644 index 000000000..5e358636c --- /dev/null +++ b/src/main/resources/io/viash/languages/r/ViashParseJsonHybrid.R @@ -0,0 +1,283 @@ +#' Parse JSON parameters file into an R list +#' +#' Tries to use jsonlite if available, falls back to the built-in +#' recursive descent parser. +#' +#' @param json_path Path to the JSON file. If NULL, reads from +#' $VIASH_WORK_PARAMS environment variable. +#' @return Named list containing the parsed JSON data. +viash_parse_json <- function(json_path = NULL) { + if (is.null(json_path)) { + json_path <- Sys.getenv("VIASH_WORK_PARAMS") + if (json_path == "") { + stop("VIASH_WORK_PARAMS environment variable not set") + } + } + + if (!file.exists(json_path)) { + stop(paste0("Parameters file not found: ", json_path)) + } + + if (requireNamespace("jsonlite", quietly = TRUE)) { + tryCatch({ + json_text <- readLines(json_path, warn = FALSE) + json_text <- paste(json_text, collapse = "\n") + # Use bigint_as_char = TRUE to preserve large integers as strings + # This prevents precision loss for values > 2^53 + jsonlite::parse_json(json_text, simplifyVector = TRUE, bigint_as_char = TRUE) + }, error = function(e) { + stop(paste0("Error parsing JSON file: ", e$message)) + }) + } else { + json_text <- paste(readLines(json_path, warn = FALSE), collapse = "\n") + .viash_json_parse(json_text) + } +} + +# Recursive descent JSON parser. +# Pre-splits input into a character vector for O(1) indexing. +# Returns nested R lists with vector simplification for homogeneous arrays. +.viash_json_parse <- function(json) { + chars <- strsplit(json, "")[[1]] + len <- length(chars) + pos <- 1L + + # Lookup table for single-character JSON escape sequences + escape_map <- list( + "n" = "\n", "t" = "\t", "r" = "\r", + "b" = "\b", "f" = "\f", + "\\" = "\\", "\"" = "\"", "/" = "/" + ) + + skip_ws <- function() { + while (pos <= len) { + ch <- chars[pos] + if (ch == " " || ch == "\t" || ch == "\n" || ch == "\r") { + pos <<- pos + 1L + } else { + return(invisible(NULL)) + } + } + } + + peek <- function() { + skip_ws() + if (pos > len) stop("viash_parse_json: Unexpected end of JSON") + chars[pos] + } + + consume <- function(expected) { + skip_ws() + if (pos > len) { + stop(paste0("viash_parse_json: Expected '", expected, + "' but reached end of JSON")) + } + if (chars[pos] != expected) { + stop(paste0("viash_parse_json: Expected '", expected, + "' at position ", pos, ", got '", chars[pos], "'")) + } + pos <<- pos + 1L + } + + parse_value <- function() { + ch <- peek() + if (ch == "\"") return(parse_string()) + if (ch == "{") return(parse_object()) + if (ch == "[") return(parse_array()) + if (ch == "t" || ch == "f") return(parse_boolean()) + if (ch == "n") return(parse_null()) + if (ch == "-" || (ch >= "0" && ch <= "9")) return(parse_number()) + stop(paste0("viash_parse_json: Unexpected character '", ch, + "' at position ", pos)) + } + + parse_string <- function() { + consume("\"") + buf <- character(min(len - pos + 1L, 1024L)) + buf_len <- 0L + + while (pos <= len) { + ch <- chars[pos] + if (ch == "\"") { + pos <<- pos + 1L + if (buf_len == 0L) return("") + return(paste0(buf[seq_len(buf_len)], collapse = "")) + } + buf_len <- buf_len + 1L + # Grow buffer if needed + if (buf_len > length(buf)) { + buf <- c(buf, character(length(buf))) + } + if (ch == "\\") { + pos <<- pos + 1L + if (pos > len) stop("viash_parse_json: Unterminated string escape") + esc <- chars[pos] + mapped <- escape_map[[esc]] + if (!is.null(mapped)) { + buf[buf_len] <- mapped + } else if (esc == "u") { + # Unicode escape \uXXXX + if (pos + 4L > len) { + stop("viash_parse_json: Invalid unicode escape") + } + hex <- paste0(chars[(pos + 1L):(pos + 4L)], collapse = "") + buf[buf_len] <- intToUtf8(strtoi(hex, 16L)) + pos <<- pos + 4L + } else { + # Unknown escape: keep escaped character as-is + buf[buf_len] <- esc + } + } else { + buf[buf_len] <- ch + } + pos <<- pos + 1L + } + stop("viash_parse_json: Unterminated string") + } + + parse_number <- function() { + start <- pos + # Optional minus + if (chars[pos] == "-") pos <<- pos + 1L + # Integer digits + while (pos <= len && chars[pos] >= "0" && chars[pos] <= "9") { + pos <<- pos + 1L + } + # Fractional part + has_decimal <- FALSE + if (pos <= len && chars[pos] == ".") { + has_decimal <- TRUE + pos <<- pos + 1L + while (pos <= len && chars[pos] >= "0" && chars[pos] <= "9") { + pos <<- pos + 1L + } + } + # Exponent part + has_exp <- FALSE + if (pos <= len && (chars[pos] == "e" || chars[pos] == "E")) { + has_exp <- TRUE + pos <<- pos + 1L + if (pos <= len && (chars[pos] == "+" || chars[pos] == "-")) { + pos <<- pos + 1L + } + while (pos <= len && chars[pos] >= "0" && chars[pos] <= "9") { + pos <<- pos + 1L + } + } + + num_str <- paste0(chars[start:(pos - 1L)], collapse = "") + if (has_decimal || has_exp) { + return(as.double(num_str)) + } + # Integer: try R's 32-bit integer first + n <- suppressWarnings(as.integer(num_str)) + if (!is.na(n)) return(n) + # Larger integer: use double if within exact precision range + digits <- nchar(gsub("-", "", num_str)) + if (digits <= 15L) return(as.double(num_str)) + # Very large integer: keep as character for bit64 handling + num_str + } + + parse_boolean <- function() { + remaining <- len - pos + 1L + if (remaining >= 4L && + paste0(chars[pos:(pos + 3L)], collapse = "") == "true") { + pos <<- pos + 4L + return(TRUE) + } + if (remaining >= 5L && + paste0(chars[pos:(pos + 4L)], collapse = "") == "false") { + pos <<- pos + 5L + return(FALSE) + } + stop(paste0("viash_parse_json: Invalid boolean at position ", pos)) + } + + parse_null <- function() { + if (len - pos + 1L >= 4L && + paste0(chars[pos:(pos + 3L)], collapse = "") == "null") { + pos <<- pos + 4L + return(NULL) + } + stop(paste0("viash_parse_json: Invalid null at position ", pos)) + } + + parse_array <- function() { + consume("[") + if (peek() == "]") { + consume("]") + return(list()) + } + + items <- list() + items[1L] <- list(parse_value()) + while (peek() == ",") { + consume(",") + items[length(items) + 1L] <- list(parse_value()) + } + consume("]") + + # Simplify homogeneous scalar arrays to vectors (like jsonlite simplifyVector) + .viash_simplify_array(items) + } + + parse_object <- function() { + consume("{") + if (peek() == "}") { + consume("}") + return(structure(list(), names = character(0))) + } + + keys <- character(0) + vals <- list() + key <- parse_string() + consume(":") + keys <- c(keys, key) + vals[length(vals) + 1L] <- list(parse_value()) + while (peek() == ",") { + consume(",") + key <- parse_string() + consume(":") + keys <- c(keys, key) + vals[length(vals) + 1L] <- list(parse_value()) + } + consume("}") + + names(vals) <- keys + vals + } + + parse_value() +} + +# Simplify a list to a vector if all elements are the same scalar type. +.viash_simplify_array <- function(items) { + n <- length(items) + if (n == 0L) return(list()) + + # NULL elements prevent simplification + has_null <- vapply(items, is.null, logical(1)) + if (any(has_null)) return(items) + + # Only simplify scalar (length-1, non-list) elements + is_scalar <- vapply(items, function(x) { + length(x) == 1L && !is.list(x) + }, logical(1)) + if (!all(is_scalar)) return(items) + + types <- vapply(items, function(x) class(x)[[1L]], character(1)) + unique_types <- unique(types) + + if (length(unique_types) == 1L) { + # All same type: collapse to vector + return(unlist(items)) + } + if (all(unique_types %in% c("integer", "numeric"))) { + # Mix of integer and numeric: promote to numeric + return(as.numeric(unlist(items))) + } + + # Mixed types: keep as list + items +} diff --git a/src/main/resources/io/viash/languages/scala/ViashParseJson.scala b/src/main/resources/io/viash/languages/scala/ViashParseJson.scala new file mode 100644 index 000000000..1ac8acf7f --- /dev/null +++ b/src/main/resources/io/viash/languages/scala/ViashParseJson.scala @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import scala.io.Source +import java.io.File + +object ViashJsonParser { + + def parseJson(path: Option[String] = None): Map[String, Any] = { + val jsonPath = path.getOrElse { + sys.env.getOrElse("VIASH_WORK_PARAMS", + throw new RuntimeException("VIASH_WORK_PARAMS environment variable not set")) + } + + val file = new File(jsonPath) + if (!file.exists()) { + throw new RuntimeException(s"Parameters file not found: $jsonPath") + } + + val jsonText = Source.fromFile(file).mkString + parse(jsonText).asInstanceOf[Map[String, Any]] + } + + private def parse(json: String): Any = { + var pos = 0 + + def skipWhitespace(): Unit = { + while (pos < json.length && json(pos).isWhitespace) pos += 1 + } + + def peek: Char = { + skipWhitespace() + if (pos < json.length) json(pos) else throw new RuntimeException("Unexpected end of JSON") + } + + def consume(expected: Char): Unit = { + skipWhitespace() + if (pos >= json.length || json(pos) != expected) { + throw new RuntimeException(s"Expected '$expected' at position $pos") + } + pos += 1 + } + + def parseValue(): Any = { + peek match { + case '"' => parseString() + case '{' => parseObject() + case '[' => parseArray() + case 't' | 'f' => parseBoolean() + case 'n' => parseNull() + case c if c == '-' || c.isDigit => parseNumber() + case c => throw new RuntimeException(s"Unexpected character '$c' at position $pos") + } + } + + def parseString(): String = { + consume('"') + val sb = new StringBuilder + while (pos < json.length && json(pos) != '"') { + if (json(pos) == '\\') { + pos += 1 + if (pos >= json.length) throw new RuntimeException("Unterminated string escape") + json(pos) match { + case 'n' => sb.append('\n') + case 't' => sb.append('\t') + case 'r' => sb.append('\r') + case 'b' => sb.append('\b') + case 'f' => sb.append('\f') + case '\\' => sb.append('\\') + case '"' => sb.append('"') + case '/' => sb.append('/') + case 'u' => + if (pos + 4 >= json.length) throw new RuntimeException("Invalid unicode escape") + val hex = json.substring(pos + 1, pos + 5) + sb.append(Integer.parseInt(hex, 16).toChar) + pos += 4 + case c => sb.append(c) + } + } else { + sb.append(json(pos)) + } + pos += 1 + } + consume('"') + sb.toString + } + + def parseNumber(): Any = { + val start = pos + if (json(pos) == '-') pos += 1 + while (pos < json.length && json(pos).isDigit) pos += 1 + + val hasDecimal = pos < json.length && json(pos) == '.' + if (hasDecimal) { + pos += 1 + while (pos < json.length && json(pos).isDigit) pos += 1 + } + + val hasExponent = pos < json.length && (json(pos) == 'e' || json(pos) == 'E') + if (hasExponent) { + pos += 1 + if (pos < json.length && (json(pos) == '+' || json(pos) == '-')) pos += 1 + while (pos < json.length && json(pos).isDigit) pos += 1 + } + + val numStr = json.substring(start, pos) + if (hasDecimal || hasExponent) { + numStr.toDouble + } else { + val n = BigInt(numStr) + if (n.isValidInt) n.toInt + else if (n.isValidLong) n.toLong + else n.toDouble + } + } + + def parseBoolean(): Boolean = { + if (json.substring(pos).startsWith("true")) { + pos += 4 + true + } else if (json.substring(pos).startsWith("false")) { + pos += 5 + false + } else { + throw new RuntimeException(s"Invalid boolean at position $pos") + } + } + + def parseNull(): Null = { + if (json.substring(pos).startsWith("null")) { + pos += 4 + null + } else { + throw new RuntimeException(s"Invalid null at position $pos") + } + } + + def parseArray(): List[Any] = { + consume('[') + if (peek == ']') { + consume(']') + return Nil + } + + val items = scala.collection.mutable.ListBuffer[Any]() + items += parseValue() + while (peek == ',') { + consume(',') + items += parseValue() + } + consume(']') + items.toList + } + + def parseObject(): Map[String, Any] = { + consume('{') + if (peek == '}') { + consume('}') + return Map.empty + } + + val entries = scala.collection.mutable.Map[String, Any]() + def parseEntry(): Unit = { + val key = parseString() + consume(':') + entries(key) = parseValue() + } + + parseEntry() + while (peek == ',') { + consume(',') + parseEntry() + } + consume('}') + entries.toMap + } + + parseValue() + } +} diff --git a/src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf b/src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf index f9ed8e40d..e31faa786 100644 --- a/src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf +++ b/src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf @@ -204,19 +204,20 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { // create dirs for output files (based on BashWrapper.createParentFiles) def createParentStr = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .findAll { par -> par.type == "file" && par.direction == "output" && par.create_parent } .collect { par -> def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]" "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }" } .join("\n") - // construct inputFileExports - def inputFileExports = meta.config.allArguments - .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + // construct input file additions to viashPar (convert Path objects to strings, keep lists as arrays) + def inputFileAdditions = meta.config.allArguments + .findAll { par -> par.type == "file" && par.direction.toLowerCase() == "input" } .collect { par -> - def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}" - "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}" + def varName = "viash_par_${par.plainName}" + // Convert Path objects to strings, keep lists as arrays + "\n |if (!${varName}.empty) { viashPar[\"${par.plainName}\"] = ${varName} instanceof List ? ${varName}.collect{it.toString()} : ${varName}.toString() }" } // NOTE: if using docker, use /tmp instead of tmpDir! @@ -253,6 +254,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { def procStr = """nextflow.enable.dsl=2 | + |import groovy.json.JsonOutput |def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") } |process $procKey {$drctvStrs |input: @@ -265,36 +267,39 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { |$stub |\"\"\" |script:$assertStr - |def parInject = args - | .findAll{key, value -> value != null} - | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"} - | .join("\\n") - |\"\"\" - |# meta exports - |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" - |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" - |export VIASH_META_NAME="${meta.config.name}" - |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" - |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" - |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } - |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } - |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then - | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) - | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) - | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) - | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) - | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) - | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) - | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) - | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) - | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) - | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) - |fi - | - |# meta synonyms - |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" - |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + |// Construct meta map + |def viashMeta = [ + | "resources_dir": "\${resourcesDir}", + | "temp_dir": "${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}", + | "name": "${meta.config.name}", + | "config": "\${resourcesDir}/.config.vsh.yaml" + |] + |if (task.cpus) { viashMeta["cpus"] = task.cpus } + |if (task.memory?.bytes != null) { + | def memB = task.memory.bytes + | viashMeta["memory_b"] = memB + | viashMeta["memory_kb"] = (long)((memB + 999) / 1000) + | viashMeta["memory_mb"] = (long)((viashMeta["memory_kb"] + 999) / 1000) + | viashMeta["memory_gb"] = (long)((viashMeta["memory_mb"] + 999) / 1000) + | viashMeta["memory_tb"] = (long)((viashMeta["memory_gb"] + 999) / 1000) + | viashMeta["memory_pb"] = (long)((viashMeta["memory_tb"] + 999) / 1000) + | viashMeta["memory_kib"] = (long)((memB + 1023) / 1024) + | viashMeta["memory_mib"] = (long)((viashMeta["memory_kib"] + 1023) / 1024) + | viashMeta["memory_gib"] = (long)((viashMeta["memory_mib"] + 1023) / 1024) + | viashMeta["memory_tib"] = (long)((viashMeta["memory_gib"] + 1023) / 1024) + | viashMeta["memory_pib"] = (long)((viashMeta["memory_tib"] + 1023) / 1024) + |} + |// Define args + |def viashPar = args + [:]${inputFileAdditions.join()} | + |// Construct full params object + |def viashParams = [ + | "par": viashPar, + | "meta": viashMeta, + | "dep": [:] + |] + |def paramsJson = JsonOutput.prettyPrint(JsonOutput.toJson(viashParams)) + |\"\"\" |# create output dirs if need be |function mkdir_parent { | for file in "\\\$@"; do @@ -303,8 +308,16 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { |} |$createParentStr | - |# argument exports${inputFileExports.join()} - |\$parInject + |# Write params.json file + |cat > .viash_params.json << 'VIASH_PARAMS_JSON' + |\$paramsJson + |VIASH_PARAMS_JSON + |export VIASH_WORK_PARAMS=".viash_params.json" + | + |# Also export VIASH_META_TEMP_DIR for backwards compatibility + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" | |# process script |${escapedScript} diff --git a/src/main/scala/io/viash/Main.scala b/src/main/scala/io/viash/Main.scala index 711405c16..5b71974e5 100644 --- a/src/main/scala/io/viash/Main.scala +++ b/src/main/scala/io/viash/Main.scala @@ -363,7 +363,7 @@ object Main extends Logging { addOptMainScript = false, applyRunnerAndEngine = false ) - ViashConfig.inject(config.config) + ViashConfig.inject(config.config, force = cli.config.inject.force()) 0 case List(cli.`export`, cli.`export`.cli_schema) => val output = cli.`export`.cli_schema.output.toOption.map(Paths.get(_)) diff --git a/src/main/scala/io/viash/ViashConfig.scala b/src/main/scala/io/viash/ViashConfig.scala index fc79409a1..4611c27c1 100644 --- a/src/main/scala/io/viash/ViashConfig.scala +++ b/src/main/scala/io/viash/ViashConfig.scala @@ -23,8 +23,10 @@ import scala.sys.process.Process import io.circe.syntax.EncoderOps import io.viash.config.Config +import io.viash.config.arguments._ import io.viash.helpers.{IO, Logging} import io.viash.helpers.circe._ +import io.viash.helpers.data_structures._ import io.viash.runners.DebugRunner import io.viash.config.ConfigMeta import io.viash.exceptions.ExitException @@ -32,6 +34,36 @@ import io.viash.runners.Runner object ViashConfig extends Logging{ + /** + * Augment required arguments with placeholder examples if they don't have examples or defaults. + * This ensures consistency across languages when generating config inject code. + */ + private def addPlaceholderExamples(config: Config): Config = { + val newGrps = config.argument_groups.map(grp => + val newArgs = grp.arguments.map { arg => + // Only add placeholder if required and missing both example and default + if (arg.required && arg.example.toList.isEmpty && arg.default.toList.isEmpty) { + val placeholder = arg match { + case a: StringArgument => a.copy(example = OneOrMore("placeholder")) + case a: FileArgument => a.copy(example = OneOrMore(Paths.get("path/to/file"))) + case a: IntegerArgument => a.copy(example = OneOrMore(123)) + case a: LongArgument => a.copy(example = OneOrMore(123456L)) + case a: DoubleArgument => a.copy(example = OneOrMore(12.34)) + case a: BooleanArgument => a.copy(example = OneOrMore(true)) + // BooleanTrueArgument and BooleanFalseArgument automatically have a default + case other => other + } + placeholder.asInstanceOf[Argument[_]] + } else { + arg + } + } + grp.copy(arguments = newArgs) + ) + + config.copy(argument_groups = newGrps) + } + def view(config: Config, format: String): Unit = { val json = ConfigMeta.configToCleanJson(config) infoOut(json.toFormattedString(format)) @@ -42,23 +74,25 @@ object ViashConfig extends Logging{ infoOut(jsons.asJson.toFormattedString(format)) } - def inject(config: Config): Unit = { + def inject(config: Config, force: Boolean = false): Unit = { // check if config has a main script if (config.mainScript.isEmpty) { infoOut("Could not find a main script in the Viash config.") throw new ExitException(1) } + val mainScript = config.mainScript.get + // check if we can read code - if (config.mainScript.get.readSome.isEmpty) { + if (mainScript.readSome.isEmpty) { infoOut("Could not read main script in the Viash config.") throw new ExitException(1) } // check if main script has a path - if (config.mainScript.get.uri.isEmpty) { + if (mainScript.uri.isEmpty) { infoOut("Main script should have a path.") throw new ExitException(1) } - val uri = config.mainScript.get.uri.get + val uri = mainScript.uri.get // check if main script is a local file if (uri.getScheme != "file") { @@ -67,22 +101,35 @@ object ViashConfig extends Logging{ } val path = Paths.get(uri.getPath()) - // debugFun - val debugRunner = DebugRunner(path = uri.getPath()) - val resources = debugRunner.generateRunner(config, testing = false) - - // create temporary directory - val dir = IO.makeTemp("viash_inject_" + config.name) - - // build regular executable - Files.createDirectories(dir) - IO.writeResources(resources.resources, dir) - - // run command, collect output - val executable = Paths.get(dir.toString, config.name).toString - val exitValue = Process(Seq(executable), cwd = dir.toFile).! + // Augment config with placeholder examples for required arguments + val augmentedConfig = addPlaceholderExamples(config) + + // Generate args, meta, and deps maps + val argsMetaAndDeps = augmentedConfig.getArgumentLikesGroupedByDest( + includeMeta = true, + includeDependencies = true, + filterInputs = true + ) + + // Generate injected code + val script = mainScript.asInstanceOf[io.viash.config.resources.Script] + val newCode = script.readWithConfigInject(argsMetaAndDeps, augmentedConfig) + + // Ask for confirmation unless --force is specified + if (!force) { + infoOut(s"About to modify: $path") + infoOut("This will inject parameter definitions into the script between VIASH START and VIASH END markers.") + infoOut("Do you want to continue? (y/N): ") + val response = scala.io.StdIn.readLine() + if (response.toLowerCase != "y" && response.toLowerCase != "yes") { + infoOut("Cancelled.") + return + } + } - // TODO: remove tempdir + // Write the modified script back to the file + IO.write(newCode, path) + infoOut(s"Successfully injected Viash header into: $path") } diff --git a/src/main/scala/io/viash/cli/CLIConf.scala b/src/main/scala/io/viash/cli/CLIConf.scala index b05817d1f..946355564 100644 --- a/src/main/scala/io/viash/cli/CLIConf.scala +++ b/src/main/scala/io/viash/cli/CLIConf.scala @@ -311,6 +311,13 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin "viash config inject", "Inject a Viash header into the main script of a Viash component.", "viash config inject config.vsh.yaml") + + val force = registerOpt[Boolean]( + name = "force", + short = Some('f'), + default = Some(false), + descr = "Modify the script without asking for confirmation." + ) } addSubcommand(view) diff --git a/src/main/scala/io/viash/config/ComputationalRequirements.scala b/src/main/scala/io/viash/config/ComputationalRequirements.scala index 29cef36ee..87ec62311 100644 --- a/src/main/scala/io/viash/config/ComputationalRequirements.scala +++ b/src/main/scala/io/viash/config/ComputationalRequirements.scala @@ -37,7 +37,17 @@ case class ComputationalRequirements( @description("A list of commands which should be present on the system for the script to function.") @example("commands: [ which, bash, awk, date, grep, egrep, ps, sed, tail, tee ]", "yaml") @default("Empty") - commands: List[String] = Nil + commands: List[String] = Nil, + @description("A list of Python packages which should be available for the script to function.") + @example("python_packages: [ numpy, pandas, scikit-learn ]", "yaml") + @default("Empty") + @since("Viash 0.10.0") + python_packages: List[String] = Nil, + @description("A list of R packages which should be available for the script to function.") + @example("r_packages: [ tidyverse, ggplot2, dplyr ]", "yaml") + @default("Empty") + @since("Viash 0.10.0") + r_packages: List[String] = Nil ) { def memoryAsBytes: Option[BigInt] = { diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index 98eb890b0..9a90461a2 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -65,7 +65,7 @@ import io.viash.helpers.data_structures.oneOrMoreToList | - type: executable |engines: | - type: docker - | image: "bash:4.0" + | image: "bash:3.2" |""", "yaml") case class Config( @description("Name of the component and the filename of the executable when built with `viash build`.") @@ -564,7 +564,7 @@ object Config extends Logging { val configUriStr = configUri.toString // get extension - val extension = configUriStr.substring(configUriStr.lastIndexOf(".") + 1).toLowerCase() + val extension = configUriStr.substring(configUriStr.lastIndexOf(".")).toLowerCase() // get basename val basenameRegex = ".*/".r @@ -572,7 +572,7 @@ object Config extends Logging { // detect whether a script (with joined header) was passed or a joined yaml // using the extension - if ((extension == "yml" || extension == "yaml") && configStr.contains("name:")) { + if ((extension == ".yml" || extension == ".yaml") && configStr.contains("name:")) { (configStr, None) } else if (Script.extensions.contains(extension)) { // detect scripting language from extension @@ -591,7 +591,8 @@ object Config extends Logging { val yaml = header.map(s => s.drop(3)).mkString("\n") val code = body.mkString("\n") - val script = Script(dest = Some(basename), text = Some(code), `type` = scriptObj.`type`) + val scriptType = s"${scriptObj.id}_script" + val script = Script(dest = Some(basename), text = Some(code), `type` = scriptType) (yaml, Some(script)) } else { diff --git a/src/main/scala/io/viash/config/arguments/Argument.scala b/src/main/scala/io/viash/config/arguments/Argument.scala index d95f5bcf5..a49e90f5b 100644 --- a/src/main/scala/io/viash/config/arguments/Argument.scala +++ b/src/main/scala/io/viash/config/arguments/Argument.scala @@ -19,6 +19,7 @@ package io.viash.config.arguments import io.circe.Json import io.viash.helpers.data_structures._ +import io.viash.helpers.Bash import io.viash.schemas._ import java.nio.file.Paths @@ -82,8 +83,8 @@ abstract class Argument[Type] { /** Common parameter name for this argument */ val par: String = dest + "_" + plainName - /** Parameter name in bash scripts */ - val VIASH_PAR: String = "VIASH_" + dest.toUpperCase + "_" + plainName.toUpperCase() + /** Parameter name in bash scripts - see Bash.viashVarName for naming convention */ + val VIASH_PAR: String = Bash.viashVarName(dest, plainName) def copyArg( `type`: String = this.`type`, diff --git a/src/main/scala/io/viash/config/dependencies/Dependency.scala b/src/main/scala/io/viash/config/dependencies/Dependency.scala index 127d25841..ecacaeba0 100644 --- a/src/main/scala/io/viash/config/dependencies/Dependency.scala +++ b/src/main/scala/io/viash/config/dependencies/Dependency.scala @@ -112,8 +112,8 @@ case class Dependency( // So after that step we must always use the righthand object. This makes that much easier. def workRepository: Option[Repository] = repository.toOption - // Name in BashWrapper - def VIASH_DEP: String = s"VIASH_DEP_${alias.getOrElse(name).replace("/", "_").toUpperCase()}" + // Name in BashWrapper - see Bash.viashVarName for naming convention + def VIASH_DEP: String = io.viash.helpers.Bash.viashVarName("dep", alias.getOrElse(name)) // Name to be used in scripts def scriptName: String = alias.getOrElse(name).replace("/", "_") // Part of the folder structure where dependencies should be written to, contains the repository & dependency name diff --git a/src/main/scala/io/viash/config/resources/BashScript.scala b/src/main/scala/io/viash/config/resources/BashScript.scala index fc819a37b..0cec7527f 100644 --- a/src/main/scala/io/viash/config/resources/BashScript.scala +++ b/src/main/scala/io/viash/config/resources/BashScript.scala @@ -17,13 +17,10 @@ package io.viash.config.resources -import io.viash.config.arguments._ -import io.viash.wrapper.BashWrapper import io.viash.schemas._ import java.net.URI -import io.viash.helpers.Bash -import io.viash.config.Config +import io.viash.languages.{Bash => BashLang} @description("""An executable Bash script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. @@ -36,28 +33,19 @@ case class BashScript( is_executable: Option[Boolean] = Some(true), parent: Option[URI] = None, + @description("""Whether to use jq for JSON parameter parsing and store multiple-value arguments as Bash arrays. + | - `true`: Use jq for JSON parsing. Arguments with `multiple: true` are stored as Bash arrays (e.g. `par_input=("a" "b" "c")`). Requires jq to be installed. + | - `false`: Use the built-in JSON parser. Arguments with `multiple: true` are stored as separator-delimited strings (e.g. `par_input="a;b;c"`), using the argument's `multiple_sep` (default `";"`). + | - Not specified (default): Same behavior as `false`, but a deprecation warning is shown at build time indicating that the default will change to `true` in a future version of Viash.""") + @example("use_jq: true", "yaml") + @since("Viash 0.10.0") + use_jq: Option[Boolean] = None, + @description("Specifies the resource as a Bash script.") - `type`: String = BashScript.`type` + `type`: String = "bash_script" ) extends Script { - val companion = BashScript + val language = BashLang def copyResource(path: Option[String], text: Option[String], dest: Option[String], is_executable: Option[Boolean], parent: Option[URI]): Resource = { copy(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } - - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { - val parSet = argsMetaAndDeps.values.flatten.map { par => - val slash = "\\VIASH_SLASH\\" - s"""$$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$${${par.VIASH_PAR}}" | sed "s#'#'$slash"'$slash"'#g;s#.*#${par.par}='&'#" ; else echo "# ${par.par}="; fi )""" - } - - val paramsCode = parSet.mkString("", "\n", "\n") - ScriptInjectionMods(params = paramsCode) - } -} - -object BashScript extends ScriptCompanion { - val commentStr = "#" - val extension = "sh" - val `type` = "bash_script" - val executor = Seq("bash") } diff --git a/src/main/scala/io/viash/config/resources/CSharpScript.scala b/src/main/scala/io/viash/config/resources/CSharpScript.scala index 2fc286060..edf063486 100644 --- a/src/main/scala/io/viash/config/resources/CSharpScript.scala +++ b/src/main/scala/io/viash/config/resources/CSharpScript.scala @@ -17,13 +17,10 @@ package io.viash.config.resources -import io.viash.config.arguments._ -import io.viash.wrapper.BashWrapper import io.viash.schemas._ import java.net.URI -import io.viash.helpers.Bash -import io.viash.config.Config +import io.viash.languages.CSharp @description("""An executable C# script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. @@ -37,83 +34,10 @@ case class CSharpScript( parent: Option[URI] = None, @description("Specifies the resource as a C# script.") - `type`: String = CSharpScript.`type` + `type`: String = "csharp_script" ) extends Script { - val companion = CSharpScript + val language = CSharp def copyResource(path: Option[String], text: Option[String], dest: Option[String], is_executable: Option[Boolean], parent: Option[URI]): Resource = { copy(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } - - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { - val quo = "\"'\"'\"" - - val paramsCode = argsMetaAndDeps.map { case (dest, params) => - val parSet = params.map{ par => - // val env_name = par.VIASH_PAR - val env_name = Bash.getEscapedArgument(par.VIASH_PAR, "@\"'\"'\"", quo, """\"""", """\"\"""") - val parse = { par match { - case a: BooleanArgumentBase if a.multiple => - s"""$env_name.Split($quo${a.multiple_sep}$quo).Select(x => bool.Parse(x.ToLower())).ToArray()""" - case a: IntegerArgument if a.multiple => - s"""$env_name.Split($quo${a.multiple_sep}$quo).Select(x => Convert.ToInt32(x)).ToArray()""" - case a: LongArgument if a.multiple => - s"""$env_name.Split($quo${a.multiple_sep}$quo).Select(x => Convert.ToInt64(x)).ToArray()""" - case a: DoubleArgument if a.multiple => - s"""$env_name.Split($quo${a.multiple_sep}$quo).Select(x => Convert.ToDouble(x)).ToArray()""" - case a: FileArgument if a.multiple => - s"""$env_name.Split($quo${a.multiple_sep}$quo).ToArray()""" - case a: StringArgument if a.multiple => - s"""$env_name.Split($quo${a.multiple_sep}$quo).ToArray()""" - case _: BooleanArgumentBase => s"""bool.Parse($env_name.ToLower())""" - case _: IntegerArgument => s"""Convert.ToInt32($env_name)""" - case _: LongArgument => s"""Convert.ToInt64($env_name)""" - case _: DoubleArgument => s"""Convert.ToDouble($env_name)""" - case _: FileArgument => s"""$env_name""" - case _: StringArgument => s"""$env_name""" - }} - - val class_ = par match { - case _: BooleanArgumentBase => "bool" - case _: IntegerArgument => "int" - case _: LongArgument => "long" - case _: DoubleArgument => "double" - case _: FileArgument => "string" - case _: StringArgument => "string" - } - - // TODO: set as null if not found, not an empty array - val notFound = par match { - //case a: Argument[_] if a.multiple => Some(s"new $class_[0]") - case a: Argument[_] if a.multiple => Some(s"(${class_}) null") - case a: StringArgument if !a.required => Some(s"(${class_}) null") - case a: FileArgument if !a.required => Some(s"(${class_}) null") - case a: Argument[_] if !a.required => Some(s"(${class_}?) null") - case _: Argument[_] => None - } - - val setter = notFound match { - case Some(nf) => - s"""$$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$parse"; else echo "$nf"; fi )""" - case None => parse.replaceAll(quo, "\"") - } - - s"${par.plainName} = $setter" - } - - s"""var $dest = new { - | ${parSet.mkString(",\n ")} - |}; - |""".stripMargin - } - - ScriptInjectionMods(params = paramsCode.mkString) - } } - -object CSharpScript extends ScriptCompanion { - val commentStr = "//" - val extension = "csx" - val `type` = "csharp_script" - val executor = Seq("dotnet", "script") -} - diff --git a/src/main/scala/io/viash/config/resources/Executable.scala b/src/main/scala/io/viash/config/resources/Executable.scala index 9a1c7ac23..5c54d9adf 100644 --- a/src/main/scala/io/viash/config/resources/Executable.scala +++ b/src/main/scala/io/viash/config/resources/Executable.scala @@ -22,6 +22,7 @@ import java.nio.file.Path import io.viash.config.arguments.Argument import io.viash.schemas._ import io.viash.config.Config +import io.viash.languages.Language @description("An executable file.") @subclass("executable") @@ -35,23 +36,18 @@ case class Executable( @description("Specifies the resource as an executable.") `type`: String = "executable" ) extends Script { - val companion = Executable + val language: Language = null // todo: deprecate executable def copyResource(path: Option[String], text: Option[String], dest: Option[String], is_executable: Option[Boolean], parent: Option[URI]): Resource = { copy(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = ScriptInjectionMods() + override def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = ScriptInjectionMods() override def readSome: Option[String] = None override def write(path: Path, overwrite: Boolean): Unit = {} override def command(script: String): String = script -} - -object Executable extends ScriptCompanion { - val commentStr = "#" - val extension = "*" - val `type` = "executable" - val executor = Nil + + override def commandSeq(script: String): Seq[String] = Seq(script) } diff --git a/src/main/scala/io/viash/config/resources/JavaScriptScript.scala b/src/main/scala/io/viash/config/resources/JavaScriptScript.scala index b35fd7bb5..4278c08f3 100644 --- a/src/main/scala/io/viash/config/resources/JavaScriptScript.scala +++ b/src/main/scala/io/viash/config/resources/JavaScriptScript.scala @@ -17,13 +17,10 @@ package io.viash.config.resources -import io.viash.wrapper.BashWrapper import io.viash.schemas._ import java.net.URI -import io.viash.helpers.Bash -import io.viash.config.Config -import io.viash.config.arguments.{Argument, StringArgument, IntegerArgument, BooleanArgumentBase, LongArgument, DoubleArgument, FileArgument} +import io.viash.languages.JavaScript @description("""An executable JavaScript script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. @@ -37,56 +34,10 @@ case class JavaScriptScript( parent: Option[URI] = None, @description("Specifies the resource as a JavaScript script.") - `type`: String = JavaScriptScript.`type` + `type`: String = "javascript_script" ) extends Script { - val companion = JavaScriptScript + val language = JavaScript def copyResource(path: Option[String], text: Option[String], dest: Option[String], is_executable: Option[Boolean], parent: Option[URI]): Resource = { copy(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } - - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { - val paramsCode = argsMetaAndDeps.map { case (dest, params) => - val parSet = params.map { par => - // val env_name = par.VIASH_PAR - val env_name = Bash.getEscapedArgument(par.VIASH_PAR, "String.raw`", "`", """`""", """`+\"`\"+String.raw`""") - - val parse = par match { - case a: BooleanArgumentBase if a.multiple => - s"""$env_name.split('${a.multiple_sep}').map(x => x.toLowerCase() === 'true')""" - case a: IntegerArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}').map(x => parseInt(x))""" - case a: LongArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}').map(x => parseInt(x))""" - case a: DoubleArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}').map(x => parseFloat(x))""" - case a: FileArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}')""" - case a: StringArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}')""" - case _: BooleanArgumentBase => s"""$env_name.toLowerCase() === 'true'""" - case _: IntegerArgument => s"""parseInt($env_name)""" - case _: LongArgument => s"""parseInt($env_name)""" - case _: DoubleArgument => s"""parseFloat($env_name)""" - case _: FileArgument => s"""$env_name""" - case _: StringArgument => s"""$env_name""" - } - - val notFound = "undefined" - - s"""'${par.plainName}': $$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$parse"; else echo $notFound; fi )""" - } - s"""let $dest = { - | ${parSet.mkString(",\n ")} - |}; - |""".stripMargin - } - ScriptInjectionMods(params = paramsCode.mkString) - } -} - -object JavaScriptScript extends ScriptCompanion { - val commentStr = "//" - val extension = "js" - val `type` = "javascript_script" - val executor = Seq("node") } diff --git a/src/main/scala/io/viash/config/resources/NextflowScript.scala b/src/main/scala/io/viash/config/resources/NextflowScript.scala index e41d733ed..7030cbb24 100644 --- a/src/main/scala/io/viash/config/resources/NextflowScript.scala +++ b/src/main/scala/io/viash/config/resources/NextflowScript.scala @@ -20,15 +20,7 @@ package io.viash.config.resources import io.viash.schemas._ import java.net.URI -import java.nio.file.Path -import java.nio.file.Paths -import io.viash.config.Config -import io.viash.config.arguments.Argument -import io.viash.runners.nextflow.NextflowHelper -import io.circe.syntax._ -import io.viash.helpers.circe._ -import io.viash.ViashNamespace -import io.viash.config.dependencies.Dependency +import io.viash.languages.Nextflow @description("""A Nextflow script. Work in progress; added mainly for annotation at the moment.""") @subclass("nextflow_script") @@ -43,24 +35,12 @@ case class NextflowScript( entrypoint: String, @description("Specifies the resource as a Nextflow script.") - `type`: String = NextflowScript.`type` + `type`: String = "nextflow_script" ) extends Script { - val companion = NextflowScript + val language = Nextflow def copyResource(path: Option[String], text: Option[String], dest: Option[String], is_executable: Option[Boolean], parent: Option[URI]): Resource = { copy(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } - - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { - ScriptInjectionMods() - } -} - -object NextflowScript extends ScriptCompanion { - val commentStr = "//" - val extension = "nf" - val `type` = "nextflow_script" - val executor = Seq("nextflow", "run", ".", "-main-script") - } diff --git a/src/main/scala/io/viash/config/resources/PythonScript.scala b/src/main/scala/io/viash/config/resources/PythonScript.scala index feba17735..93bbc6a0c 100644 --- a/src/main/scala/io/viash/config/resources/PythonScript.scala +++ b/src/main/scala/io/viash/config/resources/PythonScript.scala @@ -17,13 +17,10 @@ package io.viash.config.resources -import io.viash.config.arguments._ -import io.viash.wrapper.BashWrapper +import io.viash.schemas._ import java.net.URI -import _root_.io.viash.helpers.Bash -import io.viash.schemas._ -import io.viash.config.Config +import io.viash.languages.Python @description("""An executable Python script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. @@ -37,60 +34,10 @@ case class PythonScript( parent: Option[URI] = None, @description("Specifies the resource as a Python script.") - `type`: String = PythonScript.`type` + `type`: String = "python_script" ) extends Script { - val companion = PythonScript + val language = Python def copyResource(path: Option[String], text: Option[String], dest: Option[String], is_executable: Option[Boolean], parent: Option[URI]): Resource = { copy(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } - - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { - val paramsCode = argsMetaAndDeps.map { case (dest, params) => - val parSet = params.map { par => - // val env_name = par.VIASH_PAR - val env_name = Bash.getEscapedArgument(par.VIASH_PAR, "r'", "'", """\'""", """\'\"\'\"r\'""") - - val parse = par match { - case a: BooleanArgumentBase if a.multiple => - s"""list(map(lambda x: (x.lower() == 'true'), $env_name.split('${a.multiple_sep}')))""" - case a: IntegerArgument if a.multiple => - s"""list(map(int, $env_name.split('${a.multiple_sep}')))""" - case a: LongArgument if a.multiple => - s"""list(map(int, $env_name.split('${a.multiple_sep}')))""" - case a: DoubleArgument if a.multiple => - s"""list(map(float, $env_name.split('${a.multiple_sep}')))""" - case a: FileArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}')""" - case a: StringArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}')""" - case _: BooleanArgumentBase => s"""$env_name.lower() == 'true'""" - case _: IntegerArgument => s"""int($env_name)""" - case _: LongArgument => s"""int($env_name)""" - case _: DoubleArgument => s"""float($env_name)""" - case _: FileArgument => s"""$env_name""" - case _: StringArgument => s"""$env_name""" - } - - val notFound = "None" - - s"""'${par.plainName}': $$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$parse"; else echo $notFound; fi )""" - } - - s"""$dest = { - | ${parSet.mkString(",\n ")} - |} - |""".stripMargin - } - - ScriptInjectionMods(params = paramsCode.mkString) - } } - -object PythonScript extends ScriptCompanion { - val commentStr = "#" - val extension = "py" - val `type` = "python_script" - // The -B argument stops Python from creating .pyc or .pyo files - // on importing functions from other files. - val executor = Seq("python", "-B") -} \ No newline at end of file diff --git a/src/main/scala/io/viash/config/resources/RScript.scala b/src/main/scala/io/viash/config/resources/RScript.scala index 02eef256b..41b3723b5 100644 --- a/src/main/scala/io/viash/config/resources/RScript.scala +++ b/src/main/scala/io/viash/config/resources/RScript.scala @@ -17,13 +17,10 @@ package io.viash.config.resources -import io.viash.helpers.Bash -import io.viash.config.arguments._ -import io.viash.wrapper.BashWrapper import io.viash.schemas._ import java.net.URI -import io.viash.config.Config +import io.viash.languages.R @description("""An executable R script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. @@ -36,69 +33,19 @@ case class RScript( is_executable: Option[Boolean] = Some(true), parent: Option[URI] = None, + @description("""Whether to use the jsonlite R package for JSON parameter parsing. + | - `true`: Use only jsonlite for JSON parsing. An error will be raised if jsonlite is not installed. + | - `false`: Use only the built-in JSON parser. No deprecation warning will be shown. + | - Not specified (default): Use jsonlite if available, otherwise fall back to the built-in parser with a deprecation warning that jsonlite will be required in a future version of Viash.""") + @example("use_jsonlite: true", "yaml") + @since("Viash 0.10.0") + use_jsonlite: Option[Boolean] = None, + @description("Specifies the resource as a R script.") - `type`: String = RScript.`type` + `type`: String = "r_script" ) extends Script { - val companion = RScript + val language = R def copyResource(path: Option[String], text: Option[String], dest: Option[String], is_executable: Option[Boolean], parent: Option[URI]): Resource = { copy(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } - - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { - val paramsCode = argsMetaAndDeps.map { case (dest, params) => - - val parSet = params.map { par => - - // todo: escape multiple_sep? - val (lhs, rhs) = par match { - case a: BooleanArgumentBase if a.multiple => - ("as.logical(strsplit(toupper(", s"), split = '${a.multiple_sep}')[[1]])") - case a: IntegerArgument if a.multiple => - ("as.integer(strsplit(", s", split = '${a.multiple_sep}')[[1]])") - case a: LongArgument if a.multiple => - ("bit64::as.integer64(strsplit(", s", split = '${a.multiple_sep}')[[1]])") - case a: DoubleArgument if a.multiple => - ("as.numeric(strsplit(", s", split = '${a.multiple_sep}')[[1]])") - case a: FileArgument if a.multiple => - ("strsplit(", s", split = '${a.multiple_sep}')[[1]]") - case a: StringArgument if a.multiple => - ("strsplit(", s", split = '${a.multiple_sep}')[[1]]") - case _: BooleanArgumentBase => ("as.logical(toupper(", "))") - case _: IntegerArgument => ("as.integer(", ")") - case _: LongArgument => ("bit64::as.integer64(", ")") - case _: DoubleArgument => ("as.numeric(", ")") - case _: FileArgument => ("", "") - case _: StringArgument => ("", "") - } - val sl = "\\VIASH_SLASH\\" // used instead of "\\", as otherwise the slash gets escaped automatically. - - val notFound = "NULL" - - s""""${par.plainName}" = $$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo -n "$lhs'"; echo -n "$$${par.VIASH_PAR}" | sed "s#['$sl$sl]#$sl$sl$sl$sl&#g"; echo "'$rhs"; else echo $notFound; fi )""" - } - - s"""$dest <- list( - | ${parSet.mkString(",\n ")} - |) - |""".stripMargin - } - - val outCode = s"""# treat warnings as errors - |.viash_orig_warn <- options(warn = 2) - | - |${paramsCode.mkString} - | - |# restore original warn setting - |options(.viash_orig_warn) - |rm(.viash_orig_warn) - |""".stripMargin - ScriptInjectionMods(params = outCode) - } } - -object RScript extends ScriptCompanion { - val commentStr = "#" - val extension = "R" - val `type` = "r_script" - val executor = Seq("Rscript") -} \ No newline at end of file diff --git a/src/main/scala/io/viash/config/resources/ScalaScript.scala b/src/main/scala/io/viash/config/resources/ScalaScript.scala index 8a41e1949..88cf65bf5 100644 --- a/src/main/scala/io/viash/config/resources/ScalaScript.scala +++ b/src/main/scala/io/viash/config/resources/ScalaScript.scala @@ -17,13 +17,10 @@ package io.viash.config.resources -import io.viash.config.arguments._ -import io.viash.wrapper.BashWrapper import io.viash.schemas._ import java.net.URI -import io.viash.helpers.Bash -import io.viash.config.Config +import io.viash.languages.{Scala => ScalaLang} @description("""An executable Scala script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. @@ -37,103 +34,10 @@ case class ScalaScript( parent: Option[URI] = None, @description("Specifies the resource as a Scala script.") - `type`: String = ScalaScript.`type` + `type`: String = "scala_script" ) extends Script { - val companion = ScalaScript + val language = ScalaLang def copyResource(path: Option[String], text: Option[String], dest: Option[String], is_executable: Option[Boolean], parent: Option[URI]): Resource = { copy(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } - - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { - val quo = "\"'\"\"\"'\"" - val paramsCode = argsMetaAndDeps.map { case (dest, params) => - val parClassTypes = params.map { par => - val classType = par match { - case a: BooleanArgumentBase if a.multiple => "List[Boolean]" - case a: IntegerArgument if a.multiple => "List[Int]" - case a: LongArgument if a.multiple => "List[Long]" - case a: DoubleArgument if a.multiple => "List[Double]" - case a: FileArgument if a.multiple => "List[String]" - case a: StringArgument if a.multiple => "List[String]" - // we could argue about whether these should be options or not - case a: BooleanArgumentBase if !a.required && a.flagValue.isEmpty => "Option[Boolean]" - case a: IntegerArgument if !a.required => "Option[Int]" - case a: LongArgument if !a.required => "Option[Long]" - case a: DoubleArgument if !a.required => "Option[Double]" - case a: FileArgument if !a.required => "Option[String]" - case a: StringArgument if !a.required => "Option[String]" - case _: BooleanArgumentBase => "Boolean" - case _: IntegerArgument => "Int" - case _: LongArgument => "Long" - case _: DoubleArgument => "Double" - case _: FileArgument => "String" - case _: StringArgument => "String" - } - par.plainName + ": " + classType - } - val parSet = params.map { par => - // val env_name = par.VIASH_PAR - val env_name = Bash.getEscapedArgument(par.VIASH_PAR, quo, """\"""", """\"\"\"+\"\\\"\"+\"\"\"""") - - val parse = { par match { - case a: BooleanArgumentBase if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toLowerCase.toBoolean).toList""" - case a: IntegerArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toInt).toList""" - case a: LongArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toLong).toList""" - case a: DoubleArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toDouble).toList""" - case a: FileArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).toList""" - case a: StringArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).toList""" - case a: BooleanArgumentBase if !a.required && a.flagValue.isEmpty => s"""Some($env_name.toLowerCase.toBoolean)""" - case a: IntegerArgument if !a.required => s"""Some($env_name.toInt)""" - case a: LongArgument if !a.required => s"""Some($env_name.toLong)""" - case a: DoubleArgument if !a.required => s"""Some($env_name.toDouble)""" - case a: FileArgument if !a.required => s"""Some($env_name)""" - case a: StringArgument if !a.required => s"""Some($env_name)""" - case _: BooleanArgumentBase => s"""$env_name.toLowerCase.toBoolean""" - case _: IntegerArgument => s"""$env_name.toInt""" - case _: LongArgument => s"""$env_name.toLong""" - case _: DoubleArgument => s"""$env_name.toDouble""" - case _: FileArgument => s"""$env_name""" - case _: StringArgument => s"""$env_name""" - }} - - // Todo: set as None if multiple is undefined - val notFound = par match { - case a: Argument[_] if a.multiple => Some("Nil") - case a: BooleanArgumentBase if a.flagValue.isDefined => None - case a: Argument[_] if !a.required => Some("None") - case _: Argument[_] => None - } - - notFound match { - case Some(nf) => - s"""$$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$parse"; else echo "$nf"; fi )""" - case None => - parse.replaceAll(quo, "\"\"\"") // undo quote escape as string is not part of echo - } - } - - s"""case class Viash${dest.capitalize}( - | ${parClassTypes.mkString(",\n ")} - |) - |val $dest = Viash${dest.capitalize}( - | ${parSet.mkString(",\n ")} - |) - |""".stripMargin - } - - ScriptInjectionMods(params = paramsCode.mkString) - } } - -object ScalaScript extends ScriptCompanion { - val commentStr = "//" - val extension = "scala" - val `type` = "scala_script" - val executor = Seq("scala", "-nc") -} \ No newline at end of file diff --git a/src/main/scala/io/viash/config/resources/Script.scala b/src/main/scala/io/viash/config/resources/Script.scala index 10be5671f..0ef45a44c 100644 --- a/src/main/scala/io/viash/config/resources/Script.scala +++ b/src/main/scala/io/viash/config/resources/Script.scala @@ -20,11 +20,19 @@ package io.viash.config.resources import java.net.URI import io.viash.config.arguments.Argument import io.viash.config.Config +import io.viash.languages.Language +import io.viash.languages.{Bash, Python, R, JavaScript, Nextflow, Scala, CSharp} trait Script extends Resource { - val companion: ScriptCompanion + val language: Language - def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + language.generateInjectionMods(argsMetaAndDeps, config) + } + + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + language.generateConfigInjectMods(argsMetaAndDeps, config) + } def readWithInjection(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): String = { val code = read @@ -36,7 +44,7 @@ trait Script extends Resource { val mods = generateInjectionMods(argsMetaAndDeps, config) val viashLines = Array( - companion.commentStr + " The following code has been auto-generated by Viash.", + language.commentStr + " The following code has been auto-generated by Viash.", mods.params ) @@ -54,9 +62,9 @@ trait Script extends Resource { viashLinesWDelimiter ++ lines.slice(endIndex, lines.length) } else { - Array(companion.commentStr + companion.commentStr + " VIASH START") ++ + Array(language.commentStr + language.commentStr + " VIASH START") ++ viashLines ++ - Array(companion.commentStr + companion.commentStr + " VIASH END") ++ + Array(language.commentStr + language.commentStr + " VIASH END") ++ lines } val li2 = @@ -81,34 +89,55 @@ trait Script extends Resource { li.mkString("\n") } - def command(script: String): String = (companion.executor :+ s"\"$script\"").mkString(" ") - def commandSeq(script: String): Seq[String] = companion.executor ++ Seq(script) -} + def readWithConfigInject(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): String = { + val code = read + val lines = code.split("\n") + val startIndex = lines.indexWhere(_.contains("VIASH START")) + val endIndex = lines.indexWhere(_.contains("VIASH END")) + + // compute mods using config inject (static dictionaries, not JSON parser) + val mods = generateConfigInjectMods(argsMetaAndDeps, config) + + val viashLines = Array( + language.commentStr + " The following code has been auto-generated by Viash.", + mods.params + ) + + val li = + if (startIndex >= 0 && endIndex >= 0) { + val Whitespace = raw"^(\s+).*".r + val viashLinesWDelimiter = + lines(startIndex) match { + case Whitespace(prefix) => + viashLines.flatMap(_.split("\n")).map(prefix + _) + case _ => viashLines + } + + lines.slice(0, startIndex + 1) ++ + viashLinesWDelimiter ++ + lines.slice(endIndex, lines.length) + } else { + Array(language.commentStr + language.commentStr + " VIASH START") ++ + viashLines ++ + Array(language.commentStr + language.commentStr + " VIASH END") ++ + lines + } + + li.mkString("\n") + } -trait ScriptCompanion { - val commentStr: String - val extension: String - val `type`: String - val executor: Seq[String] - // def apply( - // path: Option[String] = None, - // text: Option[String] = None, - // dest: Option[String] = None, - // is_executable: Option[Boolean] = Some(true), - // parent: Option[URI] = None, - // entrypoint: Option[String] = None, - // `type`: String = `type` - // ): Script + def command(script: String): String = (language.executor :+ s"\"$script\"").mkString(" ") + def commandSeq(script: String): Seq[String] = language.executor ++ Seq(script) } object Script { - val companions = List(BashScript, PythonScript, RScript, JavaScriptScript, NextflowScript, ScalaScript, CSharpScript) + val languages = List(Bash, Python, R, JavaScript, Nextflow, Scala, CSharp) val extensions = - companions - .map(x => (x.extension.toLowerCase, x)) + languages + .flatMap(lang => lang.extensions.map(ext => (ext.toLowerCase, lang))) .toMap - def fromExt(extension: String): ScriptCompanion = { + def fromExt(extension: String): Language = { extensions(extension.toLowerCase) } @@ -121,25 +150,26 @@ object Script { entrypoint: Option[String] = None, `type`: String ): Script = { + val languageId = `type`.replaceAll("_script$", "") - if (`type` != NextflowScript.`type`) + if (languageId != Nextflow.id) assert(entrypoint.isEmpty, message = s"Entrypoints are not (yet) supported for resources of type ${`type`}.") - `type` match { - case BashScript.`type` => + languageId match { + case Bash.id => BashScript(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) - case CSharpScript.`type` => + case CSharp.id => CSharpScript(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) - case JavaScriptScript.`type` => + case JavaScript.id => JavaScriptScript(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) - case NextflowScript.`type` => + case Nextflow.id => assert(entrypoint.isDefined, "In a Nextflow script, the 'entrypoint' argument needs to be specified.") NextflowScript(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent, entrypoint = entrypoint.get) - case PythonScript.`type` => + case Python.id => PythonScript(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) - case RScript.`type` => + case R.id => RScript(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) - case ScalaScript.`type` => + case Scala.id => ScalaScript(path = path, text = text, dest = dest, is_executable = is_executable, parent = parent) } } diff --git a/src/main/scala/io/viash/config/resources/package.scala b/src/main/scala/io/viash/config/resources/package.scala index 9a84004ee..3e96c8dd6 100644 --- a/src/main/scala/io/viash/config/resources/package.scala +++ b/src/main/scala/io/viash/config/resources/package.scala @@ -101,7 +101,8 @@ package object resources { case Right("executable") => decodeExecutable.widen case Right("file") => decodePlainFile.widen case Right(typ) => - DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[BashScript](typ, Script.companions.map(c => c.`type`) ++ List("executable", "file")).widen + val validTypes = Script.languages.map(lang => lang.id + "_script") :+ "file" + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[BashScript](typ, validTypes).widen case Left(_) => decodePlainFile.widen // default is a simple file } diff --git a/src/main/scala/io/viash/engines/DockerEngine.scala b/src/main/scala/io/viash/engines/DockerEngine.scala index 37818ed55..67b6b81ca 100644 --- a/src/main/scala/io/viash/engines/DockerEngine.scala +++ b/src/main/scala/io/viash/engines/DockerEngine.scala @@ -36,7 +36,7 @@ import io.viash.helpers.DockerImageInfo @example( """engines: | - type: docker - | image: "bash:4.0" + | image: "bash:3.2" | setup: | - type: apt | packages: [ curl ] @@ -50,7 +50,7 @@ final case class DockerEngine( id: String = "docker", @description("The base container to start from. You can also add the tag here if you wish.") - @example("image: \"bash:4.0\"", "yaml") + @example("image: \"bash:3.2\"", "yaml") image: String, @description("If anything is specified in the setup section, running the `---setup` will result in an image with the name of `:`. If nothing is specified in the `setup` section, simply `image` will be used. Advanced usage only.") diff --git a/src/main/scala/io/viash/engines/Engine.scala b/src/main/scala/io/viash/engines/Engine.scala index 7c45ad543..69e78efe0 100644 --- a/src/main/scala/io/viash/engines/Engine.scala +++ b/src/main/scala/io/viash/engines/Engine.scala @@ -30,7 +30,7 @@ import io.viash.schemas._ @example( """engines: | - type: docker - | image: "bash:4.0" + | image: "bash:3.2" | - type: native |""", "yaml") diff --git a/src/main/scala/io/viash/helpers/Bash.scala b/src/main/scala/io/viash/helpers/Bash.scala index aa472adea..e68c17315 100644 --- a/src/main/scala/io/viash/helpers/Bash.scala +++ b/src/main/scala/io/viash/helpers/Bash.scala @@ -30,41 +30,13 @@ object Bash { lazy val ViashRemoveFlags: String = readUtils("ViashRemoveFlags") lazy val ViashAbsolutePath: String = readUtils("ViashAbsolutePath") lazy val ViashDockerAutodetectMount: String = readUtils("ViashDockerAutodetectMount") - // backwards compatibility, for now - lazy val ViashAutodetectMount: String = ViashDockerAutodetectMount.replace("ViashDocker", "Viash") lazy val ViashSourceDir: String = readUtils("ViashSourceDir") lazy val ViashFindTargetDir: String = readUtils("ViashFindTargetDir") lazy val ViashDockerFuns: String = readUtils("ViashDockerFuns") lazy val ViashLogging: String = readUtils("ViashLogging") - - def save(saveVariable: String, args: Seq[String]): String = { - saveVariable + "=\"$" + saveVariable + " " + args.mkString(" ") + "\"" - } - - // generate strings in the form of: - // SAVEVARIABLE="$SAVEVARIABLE $(Quote arg1) $(Quote arg2)" - def quoteSave(saveVariable: String, args: Seq[String]): String = { - saveVariable + "=\"$" + saveVariable + - args.map(" $(ViashQuote \"" + _ + "\")").mkString + - "\"" - } - - def argStore(name: String, plainName: String, store: String, argsConsumed: Int, storeUnparsed: Option[String]): String = { - val passStr = - if (storeUnparsed.isDefined) { - "\n " + quoteSave(storeUnparsed.get, (1 to argsConsumed).map("$" + _)) - } else { - "" - } - s""" $name) - | $plainName=$store$passStr - | shift $argsConsumed - | ;;""".stripMargin - } - - def argStoreSed(name: String, plainName: String, storeUnparsed: Option[String]): String = { - argStore(name + "=*", plainName, "$(ViashRemoveFlags \"$1\")", 1, storeUnparsed) - } + lazy val ViashCleanupRegistry: String = readUtils("ViashCleanupRegistry") + lazy val ViashRenderJson: String = readUtils("ViashRenderJson") + lazy val ViashParseArgumentValue: String = readUtils("ViashParseArgumentValue") /** * Access the parameters contents in a bash script, @@ -104,4 +76,21 @@ object Bash { .replaceWith("\\\\\\$VIASH_", "\\$VIASH_", allowUnescape) .replaceWith("\\\\\\$\\{VIASH_", "\\${VIASH_", allowUnescape) } -} + /** + * Generate a VIASH variable name for use in bash scripts. + * + * For backwards compatibility, meta variables use uppercase names. + * Par and dep variables use lowercase names so variables with similar names + * (e.g. "--input" and "--INPUT") do not clash. + * + * Examples: VIASH_META_NAME, VIASH_PAR_input, VIASH_DEP_my_dependency + * + * @param dest The destination type: "par", "meta", or "dep" + * @param name The variable name (may contain slashes which are replaced with underscores) + * @return The formatted VIASH variable name + */ + def viashVarName(dest: String, name: String): String = { + val sanitizedName = name.replace("/", "_") + val formattedName = if (dest == "meta") sanitizedName.toUpperCase() else sanitizedName + s"VIASH_${dest.toUpperCase()}_$formattedName" + }} diff --git a/src/main/scala/io/viash/helpers/Resources.scala b/src/main/scala/io/viash/helpers/Resources.scala new file mode 100644 index 000000000..f2da657bc --- /dev/null +++ b/src/main/scala/io/viash/helpers/Resources.scala @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers + +import scala.io.Source + +object Resources { + def read(path: String): String = { + Source.fromResource(s"io/viash/$path").getLines().mkString("\n") + } +} diff --git a/src/main/scala/io/viash/languages/Bash.scala b/src/main/scala/io/viash/languages/Bash.scala new file mode 100644 index 000000000..d95d461bf --- /dev/null +++ b/src/main/scala/io/viash/languages/Bash.scala @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.languages + +import io.viash.helpers.{Resources, Logger} +import io.viash.config.arguments._ +import io.viash.config.Config +import io.viash.config.resources.{ScriptInjectionMods, BashScript} + +object Bash extends Language { + val id: String = "bash" + val name: String = "Bash" + val extensions: Seq[String] = Seq(".sh") + val commentStr: String = "#" + val executor: Seq[String] = Seq("bash") + val viashParseJsonCode: String = Resources.read("languages/bash/ViashParseJson.sh") + val viashParseJsonCompatCode: String = Resources.read("languages/bash/ViashParseJsonCompatibility.sh") + + private val logger = Logger("Bash") + + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Determine use_jq setting from the BashScript resource + val useJq = config.resources.collectFirst { + case bs: BashScript => bs.use_jq + }.flatten + + useJq match { + case Some(true) => + val parseCode = s"""|${viashParseJsonCode} + | + |# Parse JSON parameters using jq + |_viash_json_content=$$(cat "$$VIASH_WORK_PARAMS") + |ViashParseJsonBash <<< "$$_viash_json_content" + |""".stripMargin + ScriptInjectionMods(params = parseCode) + case Some(false) => + generateCompatInjectionMods(argsMetaAndDeps) + case None => + logger.warn( + "Deprecation warning: 'use_jq' is not set for bash_script resource. " + + "Currently defaulting to compatibility mode (built-in parser, separator-delimited strings for multiple-value arguments). " + + "In a future version of Viash, the default will change to 'use_jq: true', " + + "which requires jq to be installed. " + + "Please set 'use_jq: true' or 'use_jq: false' explicitly in your bash_script resource to silence this warning." + ) + generateCompatInjectionMods(argsMetaAndDeps) + } + } + + /** + * Generate injection mods for compatibility mode: + * Uses the built-in bash JSON parser, then converts multiple-value + * arguments from bash arrays to separator-delimited strings. + */ + private def generateCompatInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]]): ScriptInjectionMods = { + val parseCode = s"""${viashParseJsonCompatCode} + +# Parse JSON parameters +_viash_json_content=$$(cat "$$VIASH_WORK_PARAMS") +ViashParseJsonBash <<< "$$_viash_json_content" +""" + + // Convert multiple-value arguments from arrays to IFS-separated strings. + // Note: We must unset the array variable before reassigning as a scalar, + // because assigning a string to a bash array variable only sets index [0] + // while leaving other indices intact. + val multipleArgs = argsMetaAndDeps.toList.flatMap { case (_, args) => + args.collect { + case arg if arg.multiple => + val sep = arg.multiple_sep + val par = arg.par + s"""|_viash_tmp="$$(IFS='${sep}'; printf '%s' "$${${par}[*]}")" + |unset ${par} + |${par}="$$_viash_tmp" + |unset _viash_tmp""".stripMargin + } + } + + val fullCode = if (multipleArgs.nonEmpty) { + parseCode + "\n# Convert arrays to separator-delimited strings for compatibility\n" + + multipleArgs.mkString("\n") + "\n" + } else { + parseCode + } + + ScriptInjectionMods(params = fullCode) + } + + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Determine use_jq setting from the BashScript resource + val useJq = config.resources.collectFirst { + case bs: BashScript => bs.use_jq + }.flatten + val useArrays = useJq.contains(true) + + val parSet = argsMetaAndDeps.flatMap { case (_, params) => + params.flatMap { par => + val value = getExampleValue(par, useArrays) + if (value.isEmpty) { + None + } else if (par.multiple && par.direction != Output && useArrays) { + // jq mode with arrays + Some(s"""${par.par}=($value)""") + } else { + Some(s"""${par.par}='$value'""") + } + } + } + + val paramsCode = parSet.mkString("\n") + ScriptInjectionMods(params = paramsCode) + } + + private def getExampleValue(arg: Argument[_], useArrays: Boolean = true): String = { + val values = getArgumentValues(arg) + + if (arg.multiple && arg.direction != Output) { + if (useArrays) { + values.map(v => s"'$v'").mkString(" ") + } else { + values.mkString(arg.multiple_sep) + } + } else { + values.headOption.getOrElse("") + } + } +} diff --git a/src/main/scala/io/viash/languages/CSharp.scala b/src/main/scala/io/viash/languages/CSharp.scala new file mode 100644 index 000000000..a88d2bb95 --- /dev/null +++ b/src/main/scala/io/viash/languages/CSharp.scala @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.languages + +import io.viash.helpers.Resources +import io.viash.config.arguments._ +import io.viash.config.Config +import io.viash.config.resources.ScriptInjectionMods + +object CSharp extends Language { + val id: String = "csharp" + val name: String = "C#" + val extensions: Seq[String] = Seq(".csx", ".cs") + val commentStr: String = "//" + val executor: Seq[String] = Seq("dotnet", "script") + val viashParseJsonCode: String = Resources.read("languages/csharp/ViashParseJson.csx") + + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Extract only the class and functions, not the main execution part + val helperFunctions = viashParseJsonCode + .split("\n") + .takeWhile(line => !line.contains("if (Args.Length == 0)")) + .mkString("\n") + + val paramsCode = if (argsMetaAndDeps.nonEmpty) { + // Parse JSON once + val parseOnce = "// Parse JSON parameters\nvar _viashJsonData = ViashJsonParser.ParseJson();\n\n" + + // Generate anonymous object for each section (par, meta, dep) + val sections = argsMetaAndDeps.map { case (dest, params) => + // Generate JSON extraction and anonymous object creation + val jsonExtract = s"""var _${dest}Json = _viashJsonData.ContainsKey("$dest") ? (Dictionary)_viashJsonData["$dest"] : new Dictionary();""" + + // Generate field assignments for anonymous object + val fieldAssignments = params.map { par => + val jsonKey = par.plainName + val fieldAssignment = par match { + // Multiple values - extract as arrays + case a: BooleanArgumentBase if a.multiple => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? ((List)_${dest}Json["$jsonKey"]).Select(x => Convert.ToBoolean(x)).ToArray() : new bool[0]""" + case a: IntegerArgument if a.multiple => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? ((List)_${dest}Json["$jsonKey"]).Select(x => Convert.ToInt32(x)).ToArray() : new int[0]""" + case a: LongArgument if a.multiple => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? ((List)_${dest}Json["$jsonKey"]).Select(x => Convert.ToInt64(x)).ToArray() : new long[0]""" + case a: DoubleArgument if a.multiple => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? ((List)_${dest}Json["$jsonKey"]).Select(x => Convert.ToDouble(x)).ToArray() : new double[0]""" + case a: FileArgument if a.multiple => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? ((List)_${dest}Json["$jsonKey"]).Select(x => x?.ToString()).ToArray() : new string[0]""" + case a: StringArgument if a.multiple => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? ((List)_${dest}Json["$jsonKey"]).Select(x => x?.ToString()).ToArray() : new string[0]""" + + // Optional values (nullable types) + case a: BooleanArgumentBase if !a.required && a.flagValue.isEmpty => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? (bool?)Convert.ToBoolean(_${dest}Json["$jsonKey"]) : null""" + case a: IntegerArgument if !a.required => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? (int?)Convert.ToInt32(_${dest}Json["$jsonKey"]) : null""" + case a: LongArgument if !a.required => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? (long?)Convert.ToInt64(_${dest}Json["$jsonKey"]) : null""" + case a: DoubleArgument if !a.required => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? (double?)Convert.ToDouble(_${dest}Json["$jsonKey"]) : null""" + case a: FileArgument if !a.required => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? _${dest}Json["$jsonKey"]?.ToString() : null""" + case a: StringArgument if !a.required => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? _${dest}Json["$jsonKey"]?.ToString() : null""" + + // Required values + case _: BooleanArgumentBase => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? Convert.ToBoolean(_${dest}Json["$jsonKey"]) : false""" + case _: IntegerArgument => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? Convert.ToInt32(_${dest}Json["$jsonKey"]) : 0""" + case _: LongArgument => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? Convert.ToInt64(_${dest}Json["$jsonKey"]) : 0L""" + case _: DoubleArgument => + s""" $jsonKey = _${dest}Json.ContainsKey("$jsonKey") && _${dest}Json["$jsonKey"] != null ? Convert.ToDouble(_${dest}Json["$jsonKey"]) : 0.0""" + case _: FileArgument => + " " + jsonKey + " = _" + dest + "Json.ContainsKey(\"" + jsonKey + "\") && _" + dest + "Json[\"" + jsonKey + "\"] != null ? _" + dest + "Json[\"" + jsonKey + "\"]?.ToString() : \"\"" + case _: StringArgument => + " " + jsonKey + " = _" + dest + "Json.ContainsKey(\"" + jsonKey + "\") && _" + dest + "Json[\"" + jsonKey + "\"] != null ? _" + dest + "Json[\"" + jsonKey + "\"]?.ToString() : \"\"" + } + fieldAssignment + } + + // Generate the anonymous object + val anonObject = if (params.nonEmpty) { + s"""var $dest = new { +${fieldAssignments.mkString(",\n")} +};""" + } else { + s"var $dest = new {};" + } + + jsonExtract + "\n" + anonObject + } + + parseOnce + sections.mkString("\n\n") + } else { + "" + } + + ScriptInjectionMods( + params = helperFunctions + "\n\n" + paramsCode + ) + } + + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + val paramsCode = argsMetaAndDeps.map { case (dest, params) => + val parSet = params.map { par => + val value = formatCSharpValue(par) + s"${par.plainName} = $value" + } + + s"""var $dest = new { + | ${parSet.mkString(",\n ")} + |};""".stripMargin + } + + ScriptInjectionMods(params = paramsCode.mkString("\n")) + } + + private def getCSharpArrayType(arg: Argument[_]): String = { + arg match { + case _: BooleanArgumentBase => "bool" + case _: IntegerArgument => "int" + case _: LongArgument => "long" + case _: DoubleArgument => "double" + case _: FileArgument => "string" + case _: StringArgument => "string" + } + } + + private def formatCSharpValue(arg: Argument[_]): String = { + val rawValues = getArgumentValues(arg) + + if (rawValues.isEmpty) { + // Return null with appropriate cast + val class_ = getCSharpArrayType(arg) + return if (arg.multiple) s"($class_[]) null" else if (arg.isInstanceOf[StringArgument] || arg.isInstanceOf[FileArgument]) s"($class_) null" else s"($class_?) null" + } + + if (arg.multiple) { + val arrayType = getCSharpArrayType(arg) + val formattedValues = rawValues.map(v => formatSingleCSharpValue(arg, v)) + s"new $arrayType[] { ${formattedValues.mkString(", ")} }" + } else { + formatSingleCSharpValue(arg, rawValues.headOption.getOrElse("")) + } + } + + private def formatSingleCSharpValue(arg: Argument[_], value: String): String = { + arg match { + case _: BooleanArgumentBase => if (value.toLowerCase == "true") "true" else "false" + case _: IntegerArgument | _: LongArgument => value + case _: DoubleArgument => value + case _ => s"@\"${value.replace("\"", "\"\"")}\"" + } + } +} diff --git a/src/main/scala/io/viash/languages/JavaScript.scala b/src/main/scala/io/viash/languages/JavaScript.scala new file mode 100644 index 000000000..2edb26b24 --- /dev/null +++ b/src/main/scala/io/viash/languages/JavaScript.scala @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.languages + +import io.viash.helpers.Resources +import io.viash.config.arguments._ +import io.viash.config.Config +import io.viash.config.resources.ScriptInjectionMods + +object JavaScript extends Language { + val id: String = "javascript" + val name: String = "JavaScript" + val extensions: Seq[String] = Seq(".js") + val commentStr: String = "//" + val executor: Seq[String] = Seq("node") + val viashParseJsonCode: String = Resources.read("languages/javascript/ViashParseJson.js") + + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Extract only the functions, not the main execution part or module exports + val helperFunctions = viashParseJsonCode + .split("\n") + .takeWhile(line => !line.contains("if (require.main === module)")) + .filterNot(line => line.contains("module.exports")) + .mkString("\n") + + val paramsCode = if (argsMetaAndDeps.nonEmpty) { + // Parse JSON once and extract all sections + val parseOnce = "// Parse JSON parameters once and extract all sections\nconst _viashJsonData = viashParseJson();\n" + val extractSections = argsMetaAndDeps.map { case (dest, _) => + s"const $dest = _viashJsonData['$dest'] || {};" + }.mkString("\n") + + parseOnce + extractSections + } else { + "" + } + + ScriptInjectionMods( + params = helperFunctions + "\n\n" + paramsCode + ) + } + + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + val paramsCode = argsMetaAndDeps.map { case (dest, params) => + val parSet = params.map { par => + val value = formatJSValue(par) + s" '${par.plainName}': $value" + } + + s"""let $dest = { + |${parSet.mkString(",\n")} + |};""".stripMargin + } + + ScriptInjectionMods(params = paramsCode.mkString("\n")) + } + + private def formatJSValue(arg: Argument[_]): String = { + val rawValues = getArgumentValues(arg) + if (rawValues.isEmpty) return "undefined" + + if (arg.multiple) { + val formattedValues = rawValues.map(v => formatSingleJSValue(arg, v)) + s"[${formattedValues.mkString(", ")}]" + } else { + formatSingleJSValue(arg, rawValues.headOption.getOrElse("")) + } + } + + private def formatSingleJSValue(arg: Argument[_], value: String): String = { + arg match { + case _: BooleanArgumentBase => if (value.toLowerCase == "true") "true" else "false" + case _: IntegerArgument | _: LongArgument => value + case _: DoubleArgument => value + case _ => s"String.raw`${value.replace("`", "\\`")}`" + } + } +} diff --git a/src/main/scala/io/viash/languages/Language.scala b/src/main/scala/io/viash/languages/Language.scala new file mode 100644 index 000000000..e78a964ea --- /dev/null +++ b/src/main/scala/io/viash/languages/Language.scala @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.languages + +import io.viash.config.arguments.Argument +import io.viash.config.Config +import io.viash.config.resources.ScriptInjectionMods + +/** + * Represents a programming language. + * + * Each language implementation provides: + * - Code generation for runtime parameter parsing via JSON + * - Static code injection for `viash config inject` (development convenience) + * - Language-specific value formatting + */ +trait Language { + // The unique identifier for the programming language + val id: String + + // A short, human-readable name for the programming language + val name: String + + // The file extensions associated with the programming language + val extensions: Seq[String] + + // The comment string used for single-line comments in the programming language + val commentStr: String + + // The command(s) used to execute a script written in the programming language + val executor: Seq[String] + + // The code to parse Viash param JSON files in the programming language + val viashParseJsonCode: String + + def scriptTypeId = id + "_script" + + /** + * Generate the code injection modifications for this language. + * This includes the JSON parser helper functions and type-aware extraction code. + * + * @param argsMetaAndDeps Map of destination names to lists of arguments + * @param config The component configuration + * @return ScriptInjectionMods containing header, params, and footer code + */ + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods + + /** + * Generate static code for `viash config inject`. + * This generates simple dictionaries/classes with example/default/placeholder values, + * not a JSON parser. This is intended for development convenience when editing scripts. + * + * @param argsMetaAndDeps Map of destination names to lists of arguments + * @param config The component configuration + * @return ScriptInjectionMods containing the static dictionary/class definitions + */ + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods + + /** + * Get the raw example/default values for an argument. + * + * Priority order: example > default > empty list + * + * This is a common helper used across language implementations to extract + * values for static code injection (`viash config inject`). + * + * @param arg The argument to get values from + * @return List of string values (may be empty if no example or default exists) + */ + protected def getArgumentValues(arg: Argument[_]): List[String] = { + arg.example.toList match { + case Nil => arg.default.toList match { + case Nil => Nil + case defaults => defaults.map(_.toString) + } + case examples => examples.map(_.toString) + } + } +} diff --git a/src/main/scala/io/viash/languages/Nextflow.scala b/src/main/scala/io/viash/languages/Nextflow.scala new file mode 100644 index 000000000..b6da1d799 --- /dev/null +++ b/src/main/scala/io/viash/languages/Nextflow.scala @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.languages + +import io.viash.helpers.Resources +import io.viash.config.arguments.Argument +import io.viash.config.Config +import io.viash.config.resources.ScriptInjectionMods + +object Nextflow extends Language { + val id: String = "nextflow" + val name: String = "Nextflow" + val extensions: Seq[String] = Seq(".nf") + val commentStr: String = "//" + val executor: Seq[String] = Seq("nextflow", "run", ".", "-main-script") + // this is processed in a different way + val viashParseJsonCode: String = "" + + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Nextflow scripts are processed differently + ScriptInjectionMods() + } + + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Config inject is not supported for Nextflow scripts + ScriptInjectionMods() + } +} diff --git a/src/main/scala/io/viash/languages/Python.scala b/src/main/scala/io/viash/languages/Python.scala new file mode 100644 index 000000000..936a7c197 --- /dev/null +++ b/src/main/scala/io/viash/languages/Python.scala @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.languages + +import io.viash.helpers.Resources +import io.viash.config.arguments._ +import io.viash.config.Config +import io.viash.config.resources.ScriptInjectionMods + +object Python extends Language { + val id: String = "python" + val name: String = "Python" + val extensions: Seq[String] = Seq(".py") + val commentStr: String = "#" + val executor: Seq[String] = Seq("python", "-B") + val viashParseJsonCode: String = Resources.read("languages/python/ViashParseJson.py") + + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Extract only the functions, not the main execution part + val helperFunctions = viashParseJsonCode + .split("\n") + .takeWhile(line => !line.startsWith("if __name__ == \"__main__\":")) + .mkString("\n") + + val paramsCode = if (argsMetaAndDeps.nonEmpty) { + // Parse JSON once and extract all sections + val parseOnce = "# Parse JSON parameters once and extract all sections\n_viash_json_data = viash_parse_json()\n" + val extractSections = argsMetaAndDeps.map { case (dest, _) => + s"$dest = _viash_json_data.get('$dest', {})" + }.mkString("\n") + + parseOnce + extractSections + } else { + "" + } + + ScriptInjectionMods( + params = helperFunctions + "\n\n" + paramsCode + ) + } + + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + val paramsCode = argsMetaAndDeps.map { case (dest, params) => + val parSet = params.map { par => + val value = formatPythonValue(par) + s" '${par.plainName}': $value" + } + + s"""$dest = { + |${parSet.mkString(",\n")} + |}""".stripMargin + } + + ScriptInjectionMods(params = paramsCode.mkString("\n")) + } + + private def formatPythonValue(arg: Argument[_]): String = { + val rawValues = getArgumentValues(arg) + if (rawValues.isEmpty) return "None" + + if (arg.multiple) { + val formattedValues = rawValues.map(v => formatSingleValue(arg, v)) + s"[${formattedValues.mkString(", ")}]" + } else { + formatSingleValue(arg, rawValues.headOption.getOrElse("")) + } + } + + private def formatSingleValue(arg: Argument[_], value: String): String = { + arg match { + case _: BooleanArgumentBase => if (value.toLowerCase == "true") "True" else "False" + case _: IntegerArgument | _: LongArgument => value + case _: DoubleArgument => value + case _ => s"r'${value.replace("'", "\\'")}'" + } + } +} diff --git a/src/main/scala/io/viash/languages/R.scala b/src/main/scala/io/viash/languages/R.scala new file mode 100644 index 000000000..174a46041 --- /dev/null +++ b/src/main/scala/io/viash/languages/R.scala @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.languages + +import io.viash.helpers.{Resources, Logger} +import io.viash.config.arguments._ +import io.viash.config.Config +import io.viash.config.resources.{ScriptInjectionMods, RScript} + +object R extends Language { + val id: String = "r" + val name: String = "R" + val extensions: Seq[String] = Seq(".R", ".r") + val commentStr: String = "#" + val executor: Seq[String] = Seq("Rscript") + val viashParseJsonHybridCode: String = Resources.read("languages/r/ViashParseJsonHybrid.R") + val viashParseJsonCode: String = Resources.read("languages/r/ViashParseJson.R") + + private val logger = Logger("R") + + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Determine use_jsonlite setting from the RScript resource + val useJsonlite = config.resources.collectFirst { + case rs: RScript => rs.use_jsonlite + }.flatten + + // Select the appropriate parser code based on the use_jsonlite setting: + // Some(true) -> jsonlite only (no fallback) + // Some(false) -> hybrid (jsonlite preferred, custom parser fallback), no warning + // None -> hybrid + build-time deprecation warning + val helperFunctions = useJsonlite match { + case Some(true) => + // jsonlite-only: small wrapper, no fallback parser + viashParseJsonCode + case Some(false) => + // Hybrid: jsonlite preferred with custom parser fallback, no warning + viashParseJsonHybridCode + case None => + // Hybrid + build-time deprecation warning + logger.warn( + "Deprecation warning: 'use_jsonlite' is not set for r_script resource. " + + "Currently defaulting to a hybrid mode (jsonlite preferred, built-in parser fallback). " + + "In a future version of Viash, the default will change to 'use_jsonlite: true', " + + "which requires the jsonlite R package to be installed. " + + "Please set 'use_jsonlite: true' or 'use_jsonlite: false' explicitly in your r_script resource to silence this warning." + ) + viashParseJsonHybridCode + } + + val paramsCode = if (argsMetaAndDeps.nonEmpty) { + // Parse JSON once and extract all sections + val parseOnce = "# Parse JSON parameters once and extract all sections\n.viash_json_data <- viash_parse_json()\n" + + val extractSections = argsMetaAndDeps.map { case (dest, args) => + // Extract the section + val sectionExtract = s"$dest <- if (is.null(.viash_json_data[['$dest']])) list() else .viash_json_data[['$dest']]" + + // Generate type conversions for long arguments (which are parsed as + // character strings for values > 2^53 to preserve precision) + val longConversions = args.collect { + case arg: LongArgument => + val name = arg.plainName + if (arg.multiple) { + s"if (!is.null($dest[['$name']])) $dest[['$name']] <- bit64::as.integer64($dest[['$name']])" + } else { + s"if (!is.null($dest[['$name']])) $dest[['$name']] <- bit64::as.integer64($dest[['$name']])" + } + } + + if (longConversions.nonEmpty) { + sectionExtract + "\n" + longConversions.mkString("\n") + } else { + sectionExtract + } + }.mkString("\n") + + parseOnce + extractSections + } else { + "" + } + + val outCode = s"""# treat warnings as errors + |.viash_orig_warn <- options(warn = 2) + | + |$helperFunctions + | + |$paramsCode + | + |# restore original warn setting + |options(.viash_orig_warn) + |rm(.viash_orig_warn) + |rm(.viash_json_data) + |""".stripMargin + ScriptInjectionMods(params = outCode) + } + + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + val paramsCode = argsMetaAndDeps.map { case (dest, params) => + val parSet = params.map { par => + val value = formatRValue(par) + s""" "${par.plainName}" = $value""" + } + + s"""$dest <- list( + |${parSet.mkString(",\n")} + |)""".stripMargin + } + + ScriptInjectionMods(params = paramsCode.mkString("\n")) + } + + private def formatRValue(arg: Argument[_]): String = { + val rawValues = getArgumentValues(arg) + if (rawValues.isEmpty) return "NULL" + + if (arg.multiple) { + val formattedValues = rawValues.map(v => formatSingleRValue(arg, v)) + s"c(${formattedValues.mkString(", ")})" + } else { + formatSingleRValue(arg, rawValues.headOption.getOrElse("")) + } + } + + private def formatSingleRValue(arg: Argument[_], value: String): String = { + arg match { + case _: BooleanArgumentBase => if (value.toLowerCase == "true") "TRUE" else "FALSE" + case _: IntegerArgument => s"${value}L" + case _: LongArgument => s"bit64::as.integer64('$value')" + case _: DoubleArgument => value + case _ => s""""${value.replace("\\", "\\\\").replace("\"", "\\\"")}"""" + } + } +} diff --git a/src/main/scala/io/viash/languages/Scala.scala b/src/main/scala/io/viash/languages/Scala.scala new file mode 100644 index 000000000..2643605af --- /dev/null +++ b/src/main/scala/io/viash/languages/Scala.scala @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.languages + +import io.viash.helpers.Resources +import io.viash.config.arguments._ +import io.viash.config.Config +import io.viash.config.resources.ScriptInjectionMods + +object Scala extends Language { + val id: String = "scala" + val name: String = "Scala" + val extensions: Seq[String] = Seq(".scala") + val commentStr: String = "//" + val executor: Seq[String] = Seq("scala", "-nc") + val viashParseJsonCode: String = Resources.read("languages/scala/ViashParseJson.scala") + + private def getScalaType(arg: Argument[_]): String = { + arg match { + case a: BooleanArgumentBase if a.multiple => "List[Boolean]" + case a: IntegerArgument if a.multiple => "List[Int]" + case a: LongArgument if a.multiple => "List[Long]" + case a: DoubleArgument if a.multiple => "List[Double]" + case a: FileArgument if a.multiple => "List[String]" + case a: StringArgument if a.multiple => "List[String]" + case a: BooleanArgumentBase if !a.required && a.flagValue.isEmpty => "Option[Boolean]" + case a: IntegerArgument if !a.required => "Option[Int]" + case a: LongArgument if !a.required => "Option[Long]" + case a: DoubleArgument if !a.required => "Option[Double]" + case a: FileArgument if !a.required => "Option[String]" + case a: StringArgument if !a.required => "Option[String]" + case _: BooleanArgumentBase => "Boolean" + case _: IntegerArgument => "Int" + case _: LongArgument => "Long" + case _: DoubleArgument => "Double" + case _: FileArgument => "String" + case _: StringArgument => "String" + } + } + + private def generateCaseClass(className: String, params: List[Argument[_]]): String = { + val classTypes = params.map { par => + s"${par.plainName}: ${getScalaType(par)}" + } + s"""case class $className( + ${classTypes.mkString(",\n ")} +)""" + } + + def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + // Extract only the object and functions, not the main execution part + val helperFunctions = viashParseJsonCode + .split("\n") + .takeWhile(line => !line.contains("if (sys.props.get(\"viash.run.main\").contains(\"true\")")) + .mkString("\n") + + val paramsCode = if (argsMetaAndDeps.nonEmpty) { + // Parse JSON once + val parseOnce = "// Parse JSON parameters\nval _viashJsonData = ViashJsonParser.parseJson()\n\n" + + // Generate case class and instance for each section (par, meta, dep) + val sections = argsMetaAndDeps.map { case (dest, params) => + val className = s"Viash${dest.capitalize}" + + // Generate JSON extraction code for each parameter + val extractors = params.map { par => + val jsonKey = par.plainName + val extractor = par match { + // Multiple values - extract as List (handle null as empty list) + case a: BooleanArgumentBase if a.multiple => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.asInstanceOf[List[Any]].map(_.toString.toBoolean)).getOrElse(Nil)""" + case a: IntegerArgument if a.multiple => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.asInstanceOf[List[Any]].map(v => v match { case i: Int => i; case d: Double => d.toInt; case s => s.toString.toInt })).getOrElse(Nil)""" + case a: LongArgument if a.multiple => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.asInstanceOf[List[Any]].map(v => v match { case i: Int => i.toLong; case l: Long => l; case d: Double => d.toLong; case s => s.toString.toLong })).getOrElse(Nil)""" + case a: DoubleArgument if a.multiple => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.asInstanceOf[List[Any]].map(v => v match { case d: Double => d; case i: Int => i.toDouble; case s => s.toString.toDouble })).getOrElse(Nil)""" + case a: FileArgument if a.multiple => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.asInstanceOf[List[Any]].map(_.toString)).getOrElse(Nil)""" + case a: StringArgument if a.multiple => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.asInstanceOf[List[Any]].map(_.toString)).getOrElse(Nil)""" + + // Optional values + case a: BooleanArgumentBase if !a.required && a.flagValue.isEmpty => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.toString.toBoolean)""" + case a: IntegerArgument if !a.required => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(v => v match { case i: Int => i; case d: Double => d.toInt; case s => s.toString.toInt })""" + case a: LongArgument if !a.required => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(v => v match { case i: Int => i.toLong; case l: Long => l; case d: Double => d.toLong; case s => s.toString.toLong })""" + case a: DoubleArgument if !a.required => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(v => v match { case d: Double => d; case i: Int => i.toDouble; case s => s.toString.toDouble })""" + case a: FileArgument if !a.required => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.toString)""" + case a: StringArgument if !a.required => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.toString)""" + + // Required values (handle null as default value) + case _: BooleanArgumentBase => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.toString.toBoolean).getOrElse(false)""" + case _: IntegerArgument => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(v => v match { case i: Int => i; case d: Double => d.toInt; case s => s.toString.toInt }).getOrElse(0)""" + case _: LongArgument => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(v => v match { case i: Int => i.toLong; case l: Long => l; case d: Double => d.toLong; case s => s.toString.toLong }).getOrElse(0L)""" + case _: DoubleArgument => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(v => v match { case d: Double => d; case i: Int => i.toDouble; case s => s.toString.toDouble }).getOrElse(0.0)""" + case _: FileArgument => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.toString).getOrElse("")""" + case _: StringArgument => + s"""_${dest}Json.get("$jsonKey").flatMap(v => Option(v)).map(_.toString).getOrElse("")""" + } + s" $extractor" + } + + // Generate the case class definition + val caseClassDef = generateCaseClass(className, params) + + // Generate the JSON extraction and instance creation + val extraction = s"""val _${dest}Json = _viashJsonData.getOrElse("$dest", Map.empty[String, Any]).asInstanceOf[Map[String, Any]] +val $dest = $className( +${extractors.mkString(",\n")} +)""" + + caseClassDef + "\n" + extraction + } + + parseOnce + sections.mkString("\n\n") + } else { + "" + } + + ScriptInjectionMods( + params = helperFunctions + "\n\n" + paramsCode + ) + } + + def generateConfigInjectMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { + val quo = "\"\"\"" + val paramsCode = argsMetaAndDeps.map { case (dest, params) => + val className = s"Viash${dest.capitalize}" + val caseClassDef = generateCaseClass(className, params) + + val parSet = params.map { par => + formatScalaValue(par) + } + + s"""$caseClassDef +val $dest = $className( + ${parSet.mkString(",\n ")} +)""" + } + + ScriptInjectionMods(params = paramsCode.mkString("\n")) + } + + private def formatScalaValue(arg: Argument[_]): String = { + val quo = "\"\"\"" + val rawValues = getArgumentValues(arg) + + if (rawValues.isEmpty) { + // Return appropriate null value based on type + return arg match { + case a: Argument[_] if a.multiple => "Nil" + case a: BooleanArgumentBase if a.flagValue.isDefined => formatSingleScalaValue(arg, a.flagValue.get.toString) + case _: Argument[_] if !arg.required => "None" + case _ => "??? // Required argument without default" + } + } + + if (arg.multiple) { + val formattedValues = rawValues.map(v => formatSingleScalaValue(arg, v)) + s"List(${formattedValues.mkString(", ")})" + } else if (!arg.required) { + s"Some(${formatSingleScalaValue(arg, rawValues.headOption.getOrElse(""))})" + } else { + formatSingleScalaValue(arg, rawValues.headOption.getOrElse("")) + } + } + + private def formatSingleScalaValue(arg: Argument[_], value: String): String = { + val quo = "\"\"\"" + arg match { + case _: BooleanArgumentBase => if (value.toLowerCase == "true") "true" else "false" + case _: IntegerArgument => value + case _: LongArgument => s"${value}L" + case _: DoubleArgument => value + case _ => s"$quo${value.replace(quo, "\\" + quo)}$quo" + } + } +} diff --git a/src/main/scala/io/viash/runners/ExecutableRunner.scala b/src/main/scala/io/viash/runners/ExecutableRunner.scala index 3861c581b..67d279cc7 100644 --- a/src/main/scala/io/viash/runners/ExecutableRunner.scala +++ b/src/main/scala/io/viash/runners/ExecutableRunner.scala @@ -119,7 +119,7 @@ final case class ExecutableRunner( val mainScript = Some(BashScript( dest = Some(config.name), text = Some(BashWrapper.wrapScript( - executor = "eval $VIASH_CMD", + executor = "eval $VIASH_RUN_CMD", mods = mods, config = config )) @@ -192,12 +192,12 @@ final case class ExecutableRunner( } private def nativeConfigMods(config: Config): BashWrapperMods = { - val engines = config.engines.flatMap{ + val nativeEngines = config.engines.flatMap{ case e: NativeEngine => Some(e) case _ => None } - if (engines.isEmpty) { + if (nativeEngines.isEmpty) { return BashWrapperMods() } @@ -209,9 +209,9 @@ final case class ExecutableRunner( val preRun = s""" - |if ${oneOfEngines(engines)} ; then + |if ${oneOfEngines(nativeEngines)} ; then | if [ "$$VIASH_MODE" == "run" ]; then - | VIASH_CMD="$cmd" + | VIASH_RUN_CMD="$cmd" | else | ViashError "Engine '$$VIASH_ENGINE_ID' does not support mode '$$VIASH_MODE'." | exit 1 @@ -363,6 +363,7 @@ final case class ExecutableRunner( | VIASH_DOCKER_IMAGE_ID='${engine.getTargetIdentifier(config).toString()}'""".stripMargin }.mkString("if ", "\n elif ", "\n fi") + // NOTE: regarding ---debug, should we create the viash_work_dir already? val postParse = s""" |if [[ "$$VIASH_ENGINE_TYPE" == "docker" ]]; then @@ -383,9 +384,9 @@ final case class ExecutableRunner( | | # enter docker container | elif [[ "$$VIASH_MODE" == "debug" ]]; then - | VIASH_CMD="docker run --entrypoint=bash $${VIASH_DOCKER_RUN_ARGS[@]} -v '$$(pwd)':/pwd --workdir /pwd -t $$VIASH_DOCKER_IMAGE_ID" - | ViashNotice "+ $$VIASH_CMD" - | eval $$VIASH_CMD + | VIASH_DEBUG_CMD="docker run --entrypoint=bash $${VIASH_DOCKER_RUN_ARGS[@]} -v '$$(pwd)':/pwd --workdir /pwd -t $$VIASH_DOCKER_IMAGE_ID" + | ViashNotice "+ $$VIASH_DEBUG_CMD" + | eval $$VIASH_DEBUG_CMD | exit | | # build docker image @@ -415,22 +416,26 @@ final case class ExecutableRunner( val args = config.getArgumentLikes(includeMeta = true) val detectMounts = args.flatMap { - case arg: FileArgument if arg.multiple => - // resolve arguments with multiplicity different from singular args - val viash_temp = "VIASH_TEST_" + arg.plainName.toUpperCase() - val chownIfOutput = if (arg.direction == Output) "\n VIASH_CHOWN_VARS+=( \"$var\" )" else "" + case arg: FileArgument if arg.multiple && arg.direction == Input => + // resolve input arguments with multiplicity different from singular args Some( s""" |if [ ! -z "$$${arg.VIASH_PAR}" ]; then - | $viash_temp=() - | IFS='${Bash.escapeString(arg.multiple_sep, quote = true)}' - | for var in $$${arg.VIASH_PAR}; do - | unset IFS - | VIASH_DIRECTORY_MOUNTS+=( "$$(ViashDockerAutodetectMountArg "$$var")" ) - | var=$$(ViashDockerAutodetectMount "$$var") - | $viash_temp+=( "$$var" )$chownIfOutput + | for i in "$${!${arg.VIASH_PAR}[@]}"; do + | VIASH_DIRECTORY_MOUNTS+=( "$$(ViashDockerAutodetectMountArg $${${arg.VIASH_PAR}[$$i]})" ) + | ${arg.VIASH_PAR}[$$i]=$$(ViashDockerAutodetectMount $${${arg.VIASH_PAR}[$$i]}) | done - | ${arg.VIASH_PAR}=$$(IFS='${Bash.escapeString(arg.multiple_sep, quote = true)}' ; echo "$${$viash_temp[*]}") + |fi""".stripMargin + ) + case arg: FileArgument if arg.multiple && arg.direction == Output => + // For multiple output arguments, the value is a pattern (e.g., "output_*.txt") + // Add it to chown vars so created files get ownership changed + Some( + s""" + |if [ ! -z "$$${arg.VIASH_PAR}" ]; then + | VIASH_DIRECTORY_MOUNTS+=( "$$(ViashDockerAutodetectMountArg "$$${arg.VIASH_PAR}")" ) + | ${arg.VIASH_PAR}=$$(ViashDockerAutodetectMount "$$${arg.VIASH_PAR}") + | VIASH_CHOWN_VARS+=( "$$${arg.VIASH_PAR}" ) |fi""".stripMargin ) case arg: FileArgument => @@ -467,6 +472,10 @@ final case class ExecutableRunner( |if [[ "$$VIASH_ENGINE_TYPE" == "docker" ]]; then | # detect volumes from file arguments | VIASH_CHOWN_VARS=()${detectMounts.mkString("")} + | + | # Add viash work dir to mounts + | VIASH_DIRECTORY_MOUNTS+=( "$$(ViashDockerAutodetectMountArg "$$VIASH_WORK_DIR")" ) + | VIASH_WORK_DIR=$$(ViashDockerAutodetectMount "$$VIASH_WORK_DIR") | | # get unique mounts | VIASH_UNIQUE_MOUNTS=($$(for val in "$${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$$val"; done | sort -u)) @@ -477,17 +486,12 @@ final case class ExecutableRunner( val stripAutomounts = args.flatMap { case arg: FileArgument if arg.multiple && arg.direction == Input => // resolve arguments with multiplicity different from singular args - val viash_temp = "VIASH_TEST_" + arg.plainName.toUpperCase() Some( s""" | if [ ! -z "$$${arg.VIASH_PAR}" ]; then - | unset $viash_temp - | IFS='${Bash.escapeString(arg.multiple_sep, quote = true)}' - | for var in $$${arg.VIASH_PAR}; do - | unset IFS - | ${BashWrapper.store("ViashDockerStripAutomount", viash_temp, "\"$(ViashDockerStripAutomount \"$var\")\"", Some(arg.multiple_sep)).mkString("\n ")} + | for i in "$${!${arg.VIASH_PAR}[@]}"; do + | ${arg.VIASH_PAR}[i]=$$(ViashDockerStripAutomount "$${${arg.VIASH_PAR}[i]}") | done - | ${arg.VIASH_PAR}="$$$viash_temp" | fi""".stripMargin ) case arg: FileArgument => @@ -526,13 +530,13 @@ final case class ExecutableRunner( | function ViashPerformChown { | if (( $${#VIASH_CHOWN_VARS[@]} )); then | set +e - | VIASH_CMD="docker run --entrypoint=bash --rm $${VIASH_UNIQUE_MOUNTS[@]} $$VIASH_DOCKER_IMAGE_ID -c 'chown $$(id -u):$$(id -g) --silent --recursive $${VIASH_CHOWN_VARS[@]}'" - | ViashDebug "+ $$VIASH_CMD" - | eval $$VIASH_CMD + | VIASH_CHMOD_CMD="docker run --entrypoint=bash --rm $${VIASH_UNIQUE_MOUNTS[@]} $$VIASH_DOCKER_IMAGE_ID -c 'chown $$(id -u):$$(id -g) --silent --recursive $${VIASH_CHOWN_VARS[@]}'" + | ViashDebug "+ $$VIASH_CHMOD_CMD" + | eval $$VIASH_CHMOD_CMD | set -e | fi | } - | trap ViashPerformChown EXIT + | ViashRegisterCleanup ViashPerformChown |fi""".stripMargin BashWrapperMods( @@ -584,7 +588,7 @@ final case class ExecutableRunner( val preRun = s""" |if [[ "$$VIASH_ENGINE_TYPE" == "docker" ]]; then - | VIASH_CMD="docker run$entrypointStr$workdirStr $${VIASH_DOCKER_RUN_ARGS[@]} $${VIASH_UNIQUE_MOUNTS[@]} $$VIASH_DOCKER_IMAGE_ID" + | VIASH_RUN_CMD="docker run$entrypointStr$workdirStr $${VIASH_DOCKER_RUN_ARGS[@]} $${VIASH_UNIQUE_MOUNTS[@]} $$VIASH_DOCKER_IMAGE_ID" |fi""".stripMargin BashWrapperMods( diff --git a/src/main/scala/io/viash/runners/NextflowRunner.scala b/src/main/scala/io/viash/runners/NextflowRunner.scala index f44564ad4..f073d49be 100644 --- a/src/main/scala/io/viash/runners/NextflowRunner.scala +++ b/src/main/scala/io/viash/runners/NextflowRunner.scala @@ -30,6 +30,7 @@ import io.viash.runners.{Runner, RunnerResources} import io.viash.engines.DockerEngine import io.viash.runners.nextflow._ import io.viash.config.resources.{Executable, NextflowScript, PlainFile} +import io.viash.languages.Nextflow @description( """Run a Viash component on a Nextflow backend engine. @@ -162,7 +163,7 @@ final case class NextflowRunner( // TODO: define profiles val profileStr = - if (containerDirective.isDefined || config.mainScript.map(_.`type`) == Some(NextflowScript.`type`)) { + if (containerDirective.isDefined || config.mainScript.map(_.`type`) == Some(Nextflow.scriptTypeId)) { "\n\n" + NextflowHelper.profilesHelper } else { "" diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala b/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala index 6ed1c8f62..7b078332b 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala @@ -78,7 +78,7 @@ object NextflowHelper { val executionCode = s"""set -e - |tempscript=".viash_script.${res.companion.extension}" + |tempscript=".viash_script${res.language.extensions.head}" |cat > "$scriptPath" << VIASHMAIN |$escapedCode |VIASHMAIN diff --git a/src/main/scala/io/viash/wrapper/BashWrapper.scala b/src/main/scala/io/viash/wrapper/BashWrapper.scala index 7d86e9bc7..ee86c85ca 100644 --- a/src/main/scala/io/viash/wrapper/BashWrapper.scala +++ b/src/main/scala/io/viash/wrapper/BashWrapper.scala @@ -25,9 +25,13 @@ import java.nio.file.Paths import io.viash.ViashNamespace import io.viash.config.arguments._ import io.viash.config.resources.Executable +import io.viash.config.resources.Script import io.viash.helpers.data_structures.oneOrMoreToList object BashWrapper { + // Add pipes after each newline. Prevents pipes being stripped when a string starts with a pipe (with optional leading spaces). + private def escapePipes(s: String) = s.replaceAll("\n", "\n|") + val metaArgs: List[Argument[_]] = { List( StringArgument("name", required = true, dest = "meta"), @@ -51,49 +55,86 @@ object BashWrapper { ) } - def store(name: String, env: String, value: String, multiple_sep: Option[String]): Array[String] = { - if (multiple_sep.isDefined) { - s"""if [ -z "$$$env" ]; then - | $env=$value - |else - | $env="$$$env${multiple_sep.get}"$value - |fi""".stripMargin.split("\n") - } else { - Array( - s"""[ -n "$$$env" ] && ViashError Bad arguments for option \\'$name\\': \\'$$$env\\' \\& \\'$$2\\' - you should provide exactly one argument for this option. && exit 1""", - env + "=" + value - ) - } + /** + * Generate a flag parser for arguments of the form --arg value (by default) + * + * @param argName The name of the argument. + * @param envName The name of the environment variable to store the value in. + * @param value Where the value of the argument is stored during parsing. + * @param argsConsumed The number of arguments consumed by this argument. + * @param matchKey The key to match the argument with. + * @param multiple_sep The separator to use when splitting the value into multiple values. + */ + private def generateParser( + matchKey: String, + argName: String, + envName: String, + value: String, + argsConsumed: Int, + multiple_sep: Option[String] = None + ): String = { + + s""" ${matchKey}) + | ViashParseArgumentValue "${argName}" "${envName}" "${multiple_sep.isDefined}" "${value}" + | shift ${argsConsumed} + | ;;""".stripMargin } - def argStore( - name: String, - plainName: String, - store: String, - argsConsumed: Int, + /** + * Helper function for generating a flag parser for arguments of the form --arg ... + */ + private def generateFlagParser( + argName: String, + envName: String, multiple_sep: Option[String] = None ): String = { - argsConsumed match { - case num if num > 1 => - s""" $name) - | ${this.store(name, plainName, store, multiple_sep).mkString("\n ")} - | [ $$# -lt $argsConsumed ] && ViashError Not enough arguments passed to $name. Use "--help" to get more information on the parameters. && exit 1 - | shift $argsConsumed - | ;;""".stripMargin - case _ => - s""" $name) - | ${this.store(name, plainName, store, multiple_sep).mkString("\n ")} - | shift $argsConsumed - | ;;""".stripMargin - } - + generateParser( + argName = argName, + envName = envName, + matchKey = argName, + value = "$2", + argsConsumed = 2, + multiple_sep = multiple_sep + ) + } + + /** + * Helper function for generating a flag parser for arguments of the form --arg=... + */ + private def generateFlagWithEqualsParser( + argName: String, + envName: String, + multiple_sep: Option[String] = None + ): String = { + generateParser( + argName = argName, + matchKey = argName + "=*", + envName = envName, + value = "$(ViashRemoveFlags \"$1\")", + argsConsumed = 1, + multiple_sep = multiple_sep + ) } - def argStoreSed(name: String, plainName: String, multiple_sep: Option[String] = None): String = { - argStore(name + "=*", plainName, "$(ViashRemoveFlags \"$1\")", 1, multiple_sep) + /** + * Helper function for generating a flag parser for boolean arguments of the form --arg + */ + private def generateBooleanFlagParser( + argName: String, + envName: String, + value: Boolean + ): String = { + generateParser( + argName = argName, + envName = envName, + matchKey = argName, + value = value.toString, + argsConsumed = 1, + multiple_sep = None + ) } - def spaceCode(str: String): String = { + private def spaceCode(str: String): String = { if (str != "") { "\n" + str + "\n" } else { @@ -138,20 +179,7 @@ object BashWrapper { mods: BashWrapperMods = BashWrapperMods(), debugPath: Option[String] = None, ): String = { - // Add pipes after each newline. Prevents pipes being stripped when a string starts with a pipe (with optional leading spaces). - def escapePipes(s: String) = s.replaceAll("\n", "\n|") - - val mainResource = config.mainScript - - // check whether the wd needs to be set to the resources dir - val cdToResources = - if (config.set_wd_to_resources_dir) { - s""" - |cd "$$VIASH_META_RESOURCES_DIR"""".stripMargin - } else { - "" - } - + // gather arguments val argsMetaAndDeps = if (debugPath.isDefined) { config.getArgumentLikesGroupedByDest( @@ -168,95 +196,26 @@ object BashWrapper { } val args = argsMetaAndDeps.flatMap(_._2).toList - // DETERMINE HOW TO RUN THE CODE - val executionCode = mainResource match { - // if mainResource is empty (shouldn't be the case) - case None => "" - - // if mainResource is simply an executable - case Some(e: Executable) => " " + e.path.get + " $VIASH_EXECUTABLE_ARGS" - - // if we want to debug our code - case Some(res) if debugPath.isDefined => - val code = res.readWithInjection(argsMetaAndDeps, config) - val escapedCode = Bash.escapeString(code, allowUnescape = true) - - s""" - |set -e - |cat > "${debugPath.get}" << 'VIASHMAIN' - |${escapePipes(escapedCode)} - |VIASHMAIN - |""".stripMargin - - // if mainResource is a script - case Some(res) => - val code = res.readWithInjection(argsMetaAndDeps, config) - val escapedCode = Bash.escapeString(code, allowUnescape = true) - - // check whether the script can be written to a temprorary location or - // whether it needs to be a specific path - val scriptSetup = - s""" - |tempscript=\\$$(mktemp "$$VIASH_META_TEMP_DIR/viash-run-${config.name}-XXXXXX").${res.companion.extension} - |function clean_up { - | rm "\\$$tempscript" - |} - |function interrupt { - | echo -e "\\nCTRL-C Pressed..." - | exit 1 - |} - |trap clean_up EXIT - |trap interrupt INT SIGINT""".stripMargin - val scriptPath = "\\$tempscript" - - s""" - |set -e$scriptSetup - |cat > "$scriptPath" << 'VIASHMAIN' - |${escapePipes(escapedCode)} - |VIASHMAIN$cdToResources - |${res.command(scriptPath)} & - |wait "\\$$!" - |""".stripMargin - } - - // generate bash document - val (heredocStart, heredocEnd) = mainResource match { - case None => ("", "") - case Some(_: Executable) => ("", "") - case _ => ("cat << VIASHEOF | ", "\nVIASHEOF") - } - // generate script modifiers val helpMods = generateHelp(config) - val computationalRequirementMods = generateComputationalRequirements(config) + val reqMods = generateComputationalRequirements(config) val parMods = generateParsers(args) - val execMods = mainResource match { - case Some(_: Executable) => generateExecutableArgs(args) + val execMods = config.mainScript match { + // For executables, only pass the 'par' arguments, not 'meta' or 'dep' + case Some(_: Executable) => generateExecutableArgs(argsMetaAndDeps.getOrElse("par", List.empty)) case _ => BashWrapperMods() } + val depMods = generateDependencies(config) + val runMods = generateRun(config, argsMetaAndDeps, debugPath) // combine - val allMods = helpMods ++ parMods ++ mods ++ execMods ++ computationalRequirementMods + val allMods = helpMods ++ parMods ++ mods ++ execMods ++ reqMods ++ depMods ++ runMods // generate header val header = Helper.generateScriptHeader(config) .map(h => Escaper(h, newline = true)) .mkString("# ", "\n# ", "") - val (localDependencies, remoteDependencies) = config.dependencies - .partition(d => d.isLocalDependency) - val localDependenciesStrings = localDependencies.map{ d => - // relativize the path of the main component to the local dependency - // TODO ideally we'd already have 'thisPath' precalculated but until that day, calculate it here - val thisPath = ViashNamespace.targetOutputPath("", "invalid_runner_name", config) - val relativePath = Paths.get(thisPath).relativize(Paths.get(d.configInfo.getOrElse("executable", ""))) - s"${d.VIASH_DEP}=\"$$VIASH_META_RESOURCES_DIR/$relativePath\"" - } - val remoteDependenciesStrings = remoteDependencies.map{ d => - s"${d.VIASH_DEP}=\"$$VIASH_TARGET_DIR/dependencies/${d.subOutputPath.get}/${Paths.get(d.configInfo.getOrElse("executable", "not_found")).getFileName()}\"" - } - val dependenciesStr = (localDependenciesStrings ++ remoteDependenciesStrings).mkString("\n") - /* GENERATE BASH SCRIPT */ s"""#!/usr/bin/env bash | @@ -275,12 +234,16 @@ object BashWrapper { | VIASH_TEMP=$${VIASH_TEMP:-/tmp} |fi | - |# define helper functions + |# bash helper functions start ----------------------------------- |${Bash.ViashQuote} |${Bash.ViashRemoveFlags} |${Bash.ViashSourceDir} |${Bash.ViashFindTargetDir} |${Bash.ViashLogging} + |${Bash.ViashCleanupRegistry} + |${Bash.ViashParseArgumentValue} + |${Bash.ViashRenderJson} + |# bash helper functions end ------------------------------------- | |# find source folder of this component |VIASH_META_RESOURCES_DIR=`ViashSourceDir $${BASH_SOURCE[0]}` @@ -294,10 +257,13 @@ object BashWrapper { |VIASH_META_CONFIG="$$VIASH_META_RESOURCES_DIR/${ConfigMeta.metaFilename}" |VIASH_META_TEMP_DIR="$$VIASH_TEMP" | + |# preparse bashwrapper mods start ------------------------------ |${spaceCode(allMods.preParse)} + |# preparse bashwrapper mods end -------------------------------- + | |${generateHelp(allMods.helpStrings)} |# initialise array - |VIASH_POSITIONAL_ARGS='' + |VIASH_POSITIONAL_ARGS=() | |while [[ $$# -gt 0 ]]; do | case "$$1" in @@ -322,25 +288,37 @@ object BashWrapper { | exit | ;; |${allMods.parsers} - | *) # positional arg or unknown option - | # since the positional args will be eval'd, can we always quote, instead of using ViashQuote - | VIASH_POSITIONAL_ARGS="$$VIASH_POSITIONAL_ARGS '$$1'" - | [[ $$1 == -* ]] && ViashWarning $$1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. + | *) + | # positional arg or unknown option + | VIASH_POSITIONAL_ARGS+=("$$1") + | [[ $$1 == -* ]] && ViashWarning "Value '$$1' looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use --help to get more information on the parameters." | shift # past argument | ;; | esac |done | |# parse positional parameters - |eval set -- $$VIASH_POSITIONAL_ARGS - |${spaceCode(allMods.postParse)}${spaceCode(allMods.preRun)} + |set -- "$${VIASH_POSITIONAL_ARGS[@]}" + | + |# postparse bashwrapper mods start ----------------------------- + |${spaceCode(allMods.postParse)} + |# postparse bashwrapper mods end ------------------------------- + | + |# prerun bashwrapper mods start -------------------------------- + |${spaceCode(allMods.preRun)} + |# prerun bashwrapper mods end ---------------------------------- + | + |# run bashwrapper mods start ---------------------------------- + |${spaceCode(allMods.run)} + |# run bashwrapper mods end ------------------------------------ | - |# set dependency paths - |$dependenciesStr + |# postrun bashwrapper mods start ------------------------------- + |${spaceCode(allMods.postRun)} + |# postrun bashwrapper mods end --------------------------------- | - |ViashDebug "Running command: ${executor.replaceAll("^eval (.*)", "\\$(echo $1)")}" - |$heredocStart$executor${escapePipes(executionCode)}$heredocEnd - |${spaceCode(allMods.postRun)}${spaceCode(allMods.last)} + |# last bashwrapper mods start ---------------------------------- + |${spaceCode(allMods.last)} + |# last bashwrapper mods end ------------------------------------ | |exit 0 |""".stripMargin @@ -353,22 +331,32 @@ object BashWrapper { } private def generateParsers(params: List[Argument[_]]) = { - // gather parse code for params + // no parsers should be generated for positional arguments, so remove these first val wrapperParams = params.filterNot(_.flags == "") + + // gather parse code for params val parseStrs = wrapperParams.map { - case bo: BooleanArgumentBase if bo.flagValue.isDefined => - val fv = bo.flagValue.get + case param: BooleanArgumentBase if param.flagValue.isDefined => + val flagValue = param.flagValue.get // params of the form --param - val part1 = argStore(bo.name, bo.VIASH_PAR, fv.toString, 1) + val part1 = generateBooleanFlagParser( + argName = param.name, + envName = param.VIASH_PAR, + value = flagValue + ) // Alternatives - val moreParts = bo.alternatives.map(alt => { - argStore(alt, bo.VIASH_PAR, fv.toString, 1) + val moreParts = param.alternatives.map(alt => { + generateBooleanFlagParser( + argName = alt, + envName = param.VIASH_PAR, + value = flagValue + ) }) (part1 :: moreParts).mkString("\n") case param => - val multisep = + val multiple_sep = if (param.multiple && param.direction == Input) { Some(param.multiple_sep) } else { @@ -376,18 +364,31 @@ object BashWrapper { } // params of the form --param ... - val part1 = param.flags match { - case "---" | "--" | "-" => argStore(param.name, param.VIASH_PAR, "\"$2\"", 2, multisep) - case "" => Nil - } + val part1 = generateFlagParser( + argName = param.name, + envName = param.VIASH_PAR, + multiple_sep = multiple_sep + ) + // params of the form --param=..., except -param=... is not allowed val part2 = param.flags match { - case "---" | "--" => List(argStoreSed(param.name, param.VIASH_PAR, multisep)) - case "-" | "" => Nil + case "---" | "--" => + List( + generateFlagWithEqualsParser( + argName = param.name, + envName = param.VIASH_PAR, + multiple_sep = multiple_sep + ) + ) + case "-" => Nil } // Alternatives - val moreParts = param.alternatives.map(alt => { - argStore(alt, param.VIASH_PAR, "\"$2\"", 2, multisep) + val moreParts = param.alternatives.map(alternativeFlag => { + generateFlagParser( + argName = alternativeFlag, + envName = param.VIASH_PAR, + multiple_sep = multiple_sep + ) }) (part1 :: part2 ::: moreParts).mkString("\n") @@ -401,17 +402,21 @@ object BashWrapper { } else { "\n# storing leftover values in positionals\n" + positionals.map { param => - if (param.multiple && param.direction == Input) { - s"""while [[ $$# -gt 0 ]]; do - | ${store("positionalArg", param.VIASH_PAR, "\"$1\"", Some(param.multiple_sep)).mkString("\n ")} - | shift 1 - |done""".stripMargin - } else { - s"""if [[ $$# -gt 0 ]]; then - | ${param.VIASH_PAR}="$$1" - | shift 1 - |fi""" - } + val storeStr = + s"""ViashParseArgumentValue "${param.name}" "${param.VIASH_PAR}" "${param.multiple}" "$$1"""" + + val (begin, mid, end) = + if (param.multiple && param.direction == Input) { + ("while", "do", "done") + } else { + ("if", "then", "fi") + } + + s"""# processing positional values for '${param.name}' + |${begin} [[ $$# -gt 0 ]]; ${mid} + | ${storeStr} + | shift 1 + |${end}""".stripMargin }.mkString("\n") } @@ -423,7 +428,7 @@ object BashWrapper { } else { "\n# check whether required parameters exist\n" + reqParams.map { param => - s"""if [ -z $${${param.VIASH_PAR}+x} ]; then + s"""if [ -z $${${param.VIASH_PAR}+x} ] || [ "$${${param.VIASH_PAR}}" == "@@VIASH_UNDEFINED@@" ]; then | ViashError '${param.name}' is a required argument. Use "--help" to get more information on the parameters. | exit 1 |fi""".stripMargin @@ -475,10 +480,8 @@ object BashWrapper { |fi""".stripMargin } else if (direction == Input) { s"""if [ ! -z "$$${param.VIASH_PAR}" ]; then - | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' | set -f - | for file in $$${param.VIASH_PAR}; do - | unset IFS + | for file in $${${param.VIASH_PAR}[@]}; do | if [ ! -e "$$file" ]; then | ViashError "$direction file '$$file' does not exist." | exit 1 @@ -511,10 +514,8 @@ object BashWrapper { createParentFiles.map { param => if (param.multiple && param.direction == Input) { s"""if [ ! -z "$$${param.VIASH_PAR}" ]; then - | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' | set -f - | for file in $$${param.VIASH_PAR}; do - | unset IFS + | for file in $${${param.VIASH_PAR}[@]}; do | if [ ! -d "$$(dirname "$$file")" ]; then | mkdir -p "$$(dirname "$$file")" | fi @@ -616,14 +617,12 @@ object BashWrapper { case param if param.multiple && param.direction == Input => val checkStart = s"""if [ -n "$$${param.VIASH_PAR}" ]; then - | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' | set -f - | for val in $$${param.VIASH_PAR}; do + | for val in $${${param.VIASH_PAR}[@]}; do |""" val checkEnd = s""" done | set +f - | unset IFS |fi |""".stripMargin // TODO add extra spaces for typeCheck, minCheck, maxCheck @@ -662,37 +661,43 @@ object BashWrapper { } def checkChoices[T](param: Argument[T], allowedChoices: List[T]) = { - val allowedChoicesString = allowedChoices.mkString(param.multiple_sep.toString) + val allowedChoicesString = allowedChoices.map(choice => "\"" + Bash.escapeString(choice.toString, quote = true) + "\"").mkString(" ") param match { case _ if param.multiple && param.direction == Input => - s"""if [ ! -z "$$${param.VIASH_PAR}" ]; then - | ${param.VIASH_PAR}_CHOICES=("$allowedChoicesString") - | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' - | set -f - | for val in $$${param.VIASH_PAR}; do - | if ! [[ "${param.multiple_sep}$${${param.VIASH_PAR}_CHOICES[*]}${param.multiple_sep}" =~ "${param.multiple_sep}$${val}${param.multiple_sep}" ]]; then - | ViashError '${param.name}' specified value of \\'$${val}\\' is not in the list of allowed values. Use "--help" to get more information on the parameters. - | exit 1 - | fi - | done - | set +f - | unset IFS - |fi - |""".stripMargin + s"""if [ $${#${param.VIASH_PAR}[@]} -gt 0 ]; then + | ${param.VIASH_PAR}_CHOICES=($allowedChoicesString) + | for val in "$${${param.VIASH_PAR}[@]}"; do + | found=0 + | for choice in "$${${param.VIASH_PAR}_CHOICES[@]}"; do + | if [ "$$val" == "$$choice" ]; then + | found=1 + | break + | fi + | done + | if [ $$found -eq 0 ]; then + | ViashError '${param.name}' specified value of \\'$$val\\' is not in the list of allowed values. Use "--help" to get more information on the parameters. + | exit 1 + | fi + | done + |fi + |""".stripMargin case _ => s"""if [ ! -z "$$${param.VIASH_PAR}" ]; then - | ${param.VIASH_PAR}_CHOICES=("$allowedChoicesString") - | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' - | set -f - | if ! [[ "${param.multiple_sep}$${${param.VIASH_PAR}_CHOICES[*]}${param.multiple_sep}" =~ "${param.multiple_sep}$$${param.VIASH_PAR}${param.multiple_sep}" ]]; then - | ViashError '${param.name}' specified value of \\'$$${param.VIASH_PAR}\\' is not in the list of allowed values. Use "--help" to get more information on the parameters. - | exit 1 - | fi - | set +f - | unset IFS - |fi - |""".stripMargin + | ${param.VIASH_PAR}_CHOICES=($allowedChoicesString) + | found=0 + | for choice in "$${${param.VIASH_PAR}_CHOICES[@]}"; do + | if [ "$$${param.VIASH_PAR}" == "$$choice" ]; then + | found=1 + | break + | fi + | done + | if [ $$found -eq 0 ]; then + | ViashError '${param.name}' specified value of \\'$$${param.VIASH_PAR}\\' is not in the list of allowed values. Use "--help" to get more information on the parameters. + | exit 1 + | fi + |fi + |""".stripMargin } } val choicesCheckList = @@ -715,10 +720,19 @@ object BashWrapper { choicesCheckList.mkString("\n") } + // unset variables that are set to @@VIASH_UNDEFINED@@ + val unsetUndefinedStr = + "\n# unset variables that are set to @@VIASH_UNDEFINED@@\n" + + params.map { param => + s"""if [ "$$${param.VIASH_PAR}" == "@@VIASH_UNDEFINED@@" ]; then + | unset ${param.VIASH_PAR} + |fi""".stripMargin + }.mkString("\n") + // return output BashWrapperMods( parsers = parseStrs, - preRun = joinSections(List(positionalStr, reqCheckStr, defaultsStrs, reqInputFilesStr, typeMinMaxCheckStr, choiceCheckStr, createParentStr)), + preRun = joinSections(List(positionalStr, reqCheckStr, defaultsStrs, unsetUndefinedStr, reqInputFilesStr, typeMinMaxCheckStr, choiceCheckStr, createParentStr)), last = reqOutputFilesStr ) } @@ -745,8 +759,8 @@ object BashWrapper { val parsers = compArgs.flatMap{ case (flag, env, _) => List( - argStore(flag, env, "\"$2\"", 2), - argStoreSed(flag, env) + generateFlagParser(argName = flag, envName = env), + generateFlagWithEqualsParser(argName = flag, envName = env) ) }.map("\n" + _).mkString @@ -788,7 +802,7 @@ object BashWrapper { | fi |} |# compute memory in different units - |if [ ! -z ${VIASH_META_MEMORY+x} ]; then + |if [ ! -z ${VIASH_META_MEMORY+x} ] && [ "$VIASH_META_MEMORY" != "@@VIASH_UNDEFINED@@" ]; then | VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` | # do not define other variables if memory_b is an empty string | if [ ! -z "$VIASH_META_MEMORY_B" ]; then @@ -802,14 +816,15 @@ object BashWrapper { | VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) | VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) | VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) - | else - | # unset memory if string is empty - | unset $VIASH_META_MEMORY_B | fi |fi - |# unset nproc if string is empty - |if [ -z "$VIASH_META_CPUS" ]; then - | unset $VIASH_META_CPUS + |# unset cpus if undefined or empty + |if [ -z "$VIASH_META_CPUS" ] || [ "$VIASH_META_CPUS" == "@@VIASH_UNDEFINED@@" ]; then + | unset VIASH_META_CPUS + |fi + |# unset memory if undefined + |if [ "$VIASH_META_MEMORY" == "@@VIASH_UNDEFINED@@" ]; then + | unset VIASH_META_MEMORY |fi""".stripMargin // return output @@ -833,10 +848,8 @@ object BashWrapper { if (param.multiple && param.direction == Input) { s""" |if [ ! -z "$$${param.VIASH_PAR}" ]; then - | IFS='${Bash.escapeString(param.multiple_sep, quote = true)}' | set -f - | for val in $$${param.VIASH_PAR}; do - | unset IFS + | for val in $${${param.VIASH_PAR}[@]}; do | VIASH_EXECUTABLE_ARGS="$$VIASH_EXECUTABLE_ARGS$flag '$$val'" | done | set +f @@ -854,4 +867,179 @@ object BashWrapper { ) } + private def generateDependencies( + config: Config + ): BashWrapperMods = { + if (config.dependencies.isEmpty) { + return BashWrapperMods() + } + + val (localDependencies, remoteDependencies) = config.dependencies + .partition(d => d.isLocalDependency) + + val localDependenciesStrings = localDependencies.map{ d => + // relativize the path of the main component to the local dependency + // TODO ideally we'd already have 'thisPath' precalculated but until that day, calculate it here + val thisPath = ViashNamespace.targetOutputPath("", "invalid_runner_name", config) + val relativePath = Paths.get(thisPath).relativize(Paths.get(d.configInfo.getOrElse("executable", ""))) + s"${d.VIASH_DEP}=\"$$VIASH_META_RESOURCES_DIR/$relativePath\"" + } + val remoteDependenciesStrings = remoteDependencies.map{ d => + s"${d.VIASH_DEP}=\"$$VIASH_TARGET_DIR/dependencies/${d.subOutputPath.get}/${Paths.get(d.configInfo.getOrElse("executable", "not_found")).getFileName()}\"" + } + val dependenciesStr = (localDependenciesStrings ++ remoteDependenciesStrings).mkString("\n") + + BashWrapperMods( + preRun = "\n# set dependency paths\n" + dependenciesStr + ) + } + + private def generateRun( + config: Config, + argsMetaAndDeps: Map[String, List[Argument[_]]], + debugPath: Option[String] + ): BashWrapperMods = { + // check whether the wd needs to be set to the resources dir + val cdToResources = + if (config.set_wd_to_resources_dir) { + "cd \"$VIASH_META_RESOURCES_DIR\"; " + } else { + "" + } + + config.mainScript match { + // if mainResource is empty (shouldn't be the case) + case None => BashWrapperMods() + + // if mainResource is an executable + case Some(e: Executable) => + // Execute the executable + // For native engine, run directly. For docker, use -c to pass to shell + val runCode = s""" + |# Run executable + |if [[ "$$VIASH_ENGINE_TYPE" == "docker" ]]; then + | VIASH_RUN_CMD+=" -c '${cdToResources}${e.path.get} $$VIASH_EXECUTABLE_ARGS'" + | ViashDebug "Running command: $$(echo $$VIASH_RUN_CMD)" + | eval $$VIASH_RUN_CMD & + | wait "$$!" + |else + | ViashDebug "Running command: ${cdToResources}${e.path.get} $$VIASH_EXECUTABLE_ARGS" + | eval "${cdToResources}${e.path.get} $$VIASH_EXECUTABLE_ARGS" + |fi + |""".stripMargin + + BashWrapperMods( + run = runCode + ) + + // if we want to debug our code + case Some(res) if debugPath.isDefined => + val code = res.readWithInjection(argsMetaAndDeps, config) + val runCode = s""" + |cat > "${debugPath.get}" << 'VIASHMAIN' + |${escapePipes(code)} + |VIASHMAIN + |""".stripMargin + BashWrapperMods( + run = runCode + ) + + // if mainResource is a script + case Some(res) => + val code = res.readWithInjection(argsMetaAndDeps, config) + + // determine script extension and path + val scriptPath = s"$$VIASH_WORK_DIR/script${res.language.extensions.head}" + + // Generate work directory setup code + val renderStrs = argsMetaAndDeps.map{case (key, args) => + val renderJsonStrs = args.map(arg => { + val multiple = arg.multiple && arg.direction == Input + val value = s"$${${arg.VIASH_PAR}[@]:-@@VIASH_UNDEFINED@@}" + s"""ViashRenderJsonKeyValue '${arg.plainName}' '${arg.`type`}' "${multiple}" "${value}"""" + }) + // Add section header with proper JSON object syntax + val sectionStart = s"""echo ' "${key}": {' >> "$$VIASH_WORK_PARAMS"""" + val isLastSection = key == argsMetaAndDeps.keys.last + val sectionEnd = if (isLastSection) { + s"""echo ' }' >> "$$VIASH_WORK_PARAMS"""" + } else { + s"""echo ' },' >> "$$VIASH_WORK_PARAMS"""" + } + + // Join key-value pairs with commas except for the last one + val renderedWithCommas = renderJsonStrs.zipWithIndex.map { case (str, idx) => + if (idx < renderJsonStrs.length - 1) { + s"""$str | sed 's/$$/,/' >> "$$VIASH_WORK_PARAMS"""" + } else { + s"""$str >> "$$VIASH_WORK_PARAMS"""" + } + } + + s"""$sectionStart + |${renderedWithCommas.mkString("\n")} + |$sectionEnd""".stripMargin + } + + val postParseCode = + s"""VIASH_WORK_DIR=$$(mktemp -d "$$VIASH_META_TEMP_DIR/viash-run-${config.name}-workdir-XXXXXX") + |VIASH_WORK_PARAMS="$$VIASH_WORK_DIR/params.json" + |VIASH_WORK_SCRIPT="${scriptPath}" + |# Store original work dir path before docker remapping + |VIASH_WORK_DIR_ORIGINAL="$$VIASH_WORK_DIR" + | + |function ViashCleanupWorkDir { + | if [ -z "$${VIASH_KEEP_WORK_DIR+x}" ]; then + | rm -rf "$$VIASH_WORK_DIR_ORIGINAL" + | else + | # Only print notice if VIASH_KEEP_WORK_DIR is not set to "silent" + | if [ "$$VIASH_KEEP_WORK_DIR" != "silent" ]; then + | ViashNotice "Keeping work directory at '$$VIASH_WORK_DIR_ORIGINAL' because VIASH_KEEP_WORK_DIR is set." + | fi + | fi + |} + |ViashRegisterCleanup ViashCleanupWorkDir + | + |function interrupt { + | echo -e "\nCTRL-C Pressed..." + | exit 1 + |} + |trap interrupt INT SIGINT + |""".stripMargin + + // Create params json and script file in work directory + val preRunCode = + s"""# Create params json + |ViashDebug "Creating params.json in work dir at $$VIASH_WORK_PARAMS" + |echo '{' > "$$VIASH_WORK_PARAMS" + |${renderStrs.mkString("\n")} + |echo '}' >> "$$VIASH_WORK_PARAMS" + | + |# Create script file in work directory + |ViashDebug "Creating script in work dir at $$VIASH_WORK_SCRIPT" + |cat > "$$VIASH_WORK_SCRIPT" << 'VIASHMAIN' + |${escapePipes(code)} + |VIASHMAIN + |chmod +x "$$VIASH_WORK_SCRIPT" + | + |""".stripMargin + + // Execute the script + val runCode = s""" + |# Add context + |VIASH_RUN_CMD+=" -c '${cdToResources}VIASH_WORK_PARAMS=\\\"$$VIASH_WORK_DIR/params.json\\\" ${res.command(scriptPath).replace("\"", "\\\"")}'" + | + |# Run command + |ViashDebug "Running command: $$(echo $$VIASH_RUN_CMD)" + |eval $$VIASH_RUN_CMD & + |wait "$$!" + |""".stripMargin + + BashWrapperMods( + postParse = postParseCode, + preRun = preRunCode, + run = runCode + ) + } + } } diff --git a/src/main/scala/io/viash/wrapper/BashWrapperMods.scala b/src/main/scala/io/viash/wrapper/BashWrapperMods.scala index 37b4e4dcb..56c859033 100644 --- a/src/main/scala/io/viash/wrapper/BashWrapperMods.scala +++ b/src/main/scala/io/viash/wrapper/BashWrapperMods.scala @@ -25,6 +25,7 @@ case class BashWrapperMods( parsers: String = "", postParse: String = "", preRun: String = "", + run: String = "", postRun: String = "", last: String = "", extraParams: String = "" @@ -36,6 +37,7 @@ case class BashWrapperMods( parsers = BashWrapper.joinSections(List(parsers, other.parsers), middle = "\n"), postParse = BashWrapper.joinSections(List(postParse, other.postParse)), preRun = BashWrapper.joinSections(List(preRun, other.preRun)), + run = BashWrapper.joinSections(List(run, other.run)), postRun = BashWrapper.joinSections(List(postRun, other.postRun)), last = BashWrapper.joinSections(List(last, other.last)), extraParams = extraParams + other.extraParams diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashAbsolutePath.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashAbsolutePath.test.sh new file mode 100644 index 000000000..68ea7f9bf --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashAbsolutePath.test.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashAbsolutePath.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + +# Store current directory for reference +ORIG_PWD="$PWD" + + +## TEST1: test relative paths + +# TEST1a: Simple relative file +output=$(ViashAbsolutePath "file.txt") +assert_value_equal "test1a_output" "$ORIG_PWD/file.txt" "$output" + +# TEST1b: Relative path with subdirectory +output=$(ViashAbsolutePath "subdir/file.txt") +assert_value_equal "test1b_output" "$ORIG_PWD/subdir/file.txt" "$output" + +# TEST1c: Relative path with multiple subdirectories +output=$(ViashAbsolutePath "a/b/c/file.txt") +assert_value_equal "test1c_output" "$ORIG_PWD/a/b/c/file.txt" "$output" + + +## TEST2: test absolute paths + +# TEST2a: Simple absolute path +output=$(ViashAbsolutePath "/foo/bar/file.txt") +assert_value_equal "test2a_output" "/foo/bar/file.txt" "$output" + +# TEST2b: Root path +output=$(ViashAbsolutePath "/") +assert_value_equal "test2b_output" "/" "$output" + +# TEST2c: Single directory absolute path +output=$(ViashAbsolutePath "/foo") +assert_value_equal "test2c_output" "/foo" "$output" + + +## TEST3: test path normalization with .. + +# TEST3a: Path with parent directory reference +output=$(ViashAbsolutePath "/foo/bar/..") +assert_value_equal "test3a_output" "/foo" "$output" + +# TEST3b: Path with multiple parent references +output=$(ViashAbsolutePath "/foo/bar/baz/../..") +assert_value_equal "test3b_output" "/foo" "$output" + +# TEST3c: Path with parent reference in the middle +output=$(ViashAbsolutePath "/foo/bar/../baz/file.txt") +assert_value_equal "test3c_output" "/foo/baz/file.txt" "$output" + +# TEST3d: Relative path with parent reference +output=$(ViashAbsolutePath "foo/bar/../file.txt") +assert_value_equal "test3d_output" "$ORIG_PWD/foo/file.txt" "$output" + +# TEST3e: Too many parent references (should stay at root) +output=$(ViashAbsolutePath "/foo/../../../bar") +assert_value_equal "test3e_output" "/bar" "$output" + + +## TEST4: test path normalization with . + +# TEST4a: Path with current directory reference +output=$(ViashAbsolutePath "/foo/./bar") +assert_value_equal "test4a_output" "/foo/bar" "$output" + +# TEST4b: Path with multiple current directory references +output=$(ViashAbsolutePath "/foo/././bar/./baz") +assert_value_equal "test4b_output" "/foo/bar/baz" "$output" + +# TEST4c: Path with mixed . and .. +output=$(ViashAbsolutePath "/foo/./bar/../baz/./file.txt") +assert_value_equal "test4c_output" "/foo/baz/file.txt" "$output" + + +## TEST5: test edge cases + +# TEST5a: Path with trailing slash +output=$(ViashAbsolutePath "/foo/bar/") +assert_value_equal "test5a_output" "/foo/bar" "$output" + +# TEST5b: Path with multiple consecutive slashes +output=$(ViashAbsolutePath "/foo//bar///baz") +assert_value_equal "test5b_output" "/foo/bar/baz" "$output" + +# TEST5c: Just a dot (current directory) +output=$(ViashAbsolutePath ".") +assert_value_equal "test5c_output" "$ORIG_PWD" "$output" + +# TEST5d: Just two dots (parent directory) +output=$(ViashAbsolutePath "..") +expected_parent=$(dirname "$ORIG_PWD") +assert_value_equal "test5d_output" "$expected_parent" "$output" + + +## TEST6: test relative paths starting with ./ + +# TEST6a: Relative path starting with ./ +output=$(ViashAbsolutePath "./file.txt") +assert_value_equal "test6a_output" "$ORIG_PWD/file.txt" "$output" + +# TEST6b: Relative path starting with ./subdir +output=$(ViashAbsolutePath "./subdir/file.txt") +assert_value_equal "test6b_output" "$ORIG_PWD/subdir/file.txt" "$output" + +print_test_summary diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashCleanupRegistry.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashCleanupRegistry.test.sh new file mode 100644 index 000000000..c56954641 --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashCleanupRegistry.test.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashLogging.sh +source src/main/resources/io/viash/helpers/bashutils/ViashCleanupRegistry.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + +# Disable the global EXIT trap for testing purposes +# We'll test the functions directly instead +trap - EXIT + +## TEST1: test registering cleanup handlers + +# TEST1a: Register a single handler +VIASH_CLEANUP_HANDLERS=() +ViashRegisterCleanup "handler1" +assert_value_equal "test1a_count" "1" "${#VIASH_CLEANUP_HANDLERS[@]}" +assert_value_equal "test1a_handler" "handler1" "${VIASH_CLEANUP_HANDLERS[0]}" + +# TEST1b: Register multiple handlers +VIASH_CLEANUP_HANDLERS=() +ViashRegisterCleanup "handler1" +ViashRegisterCleanup "handler2" +ViashRegisterCleanup "handler3" +assert_value_equal "test1b_count" "3" "${#VIASH_CLEANUP_HANDLERS[@]}" +assert_value_equal "test1b_handler0" "handler1" "${VIASH_CLEANUP_HANDLERS[0]}" +assert_value_equal "test1b_handler1" "handler2" "${VIASH_CLEANUP_HANDLERS[1]}" +assert_value_equal "test1b_handler2" "handler3" "${VIASH_CLEANUP_HANDLERS[2]}" + + +## TEST2: test running cleanup handlers + +# TEST2a: Run handlers in reverse order (LIFO) +VIASH_CLEANUP_HANDLERS=() +CLEANUP_ORDER="" +function test_handler1 { CLEANUP_ORDER="${CLEANUP_ORDER}1"; } +function test_handler2 { CLEANUP_ORDER="${CLEANUP_ORDER}2"; } +function test_handler3 { CLEANUP_ORDER="${CLEANUP_ORDER}3"; } +ViashRegisterCleanup "test_handler1" +ViashRegisterCleanup "test_handler2" +ViashRegisterCleanup "test_handler3" +ViashRunCleanupHandlers +assert_value_equal "test2a_order" "321" "$CLEANUP_ORDER" + +# TEST2b: Skip non-existent handlers gracefully +VIASH_CLEANUP_HANDLERS=() +CLEANUP_ORDER="" +function test_handler_exists { CLEANUP_ORDER="${CLEANUP_ORDER}E"; } +ViashRegisterCleanup "test_handler_exists" +ViashRegisterCleanup "test_handler_nonexistent" +ViashRunCleanupHandlers +assert_value_equal "test2b_order" "E" "$CLEANUP_ORDER" + +# TEST2c: Empty handler list runs without error +VIASH_CLEANUP_HANDLERS=() +ViashRunCleanupHandlers +# If we get here without error, the test passes + + +## TEST3: test cleanup handlers can access variables from their definition context + +# TEST3a: Handler can access variable captured at definition time +VIASH_CLEANUP_HANDLERS=() +CAPTURED_VALUE="" +MY_VAR="original_value" +eval 'function test_capture_handler { CAPTURED_VALUE="$MY_VAR"; }' +ViashRegisterCleanup "test_capture_handler" +MY_VAR="changed_value" +ViashRunCleanupHandlers +assert_value_equal "test3a_capture" "changed_value" "$CAPTURED_VALUE" + +# TEST3b: Handler works with local-like pattern (store value at registration time) +VIASH_CLEANUP_HANDLERS=() +RESULT_VALUE="" +WORK_DIR="/tmp/test_workdir_123" +# Create handler that captures current value +eval "function test_workdir_handler { RESULT_VALUE=\"$WORK_DIR\"; }" +ViashRegisterCleanup "test_workdir_handler" +WORK_DIR="/remapped/path" # Simulate docker remapping +ViashRunCleanupHandlers +assert_value_equal "test3b_stored" "/tmp/test_workdir_123" "$RESULT_VALUE" + + +## TEST4: test real-world usage pattern (simulating BashWrapper + ExecutableRunner) + +# TEST4a: Multiple cleanup handlers from different sources +VIASH_CLEANUP_HANDLERS=() +CLEANUP_LOG="" + +# Simulate BashWrapper's work directory cleanup +VIASH_WORK_DIR_ORIGINAL="/tmp/workdir" +function ViashCleanupWorkDir { + CLEANUP_LOG="${CLEANUP_LOG}[workdir:$VIASH_WORK_DIR_ORIGINAL]" +} +ViashRegisterCleanup ViashCleanupWorkDir + +# Simulate ExecutableRunner's chown cleanup +function ViashPerformChown { + CLEANUP_LOG="${CLEANUP_LOG}[chown]" +} +ViashRegisterCleanup ViashPerformChown + +# Run all handlers +ViashRunCleanupHandlers + +# Chown should run first (LIFO), then workdir cleanup +assert_value_equal "test4a_log" "[chown][workdir:/tmp/workdir]" "$CLEANUP_LOG" + +print_test_summary diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashDockerAutodetectMount.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashDockerAutodetectMount.test.sh new file mode 100644 index 000000000..cfca1f1e7 --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashDockerAutodetectMount.test.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashAbsolutePath.sh +source src/main/resources/io/viash/helpers/bashutils/ViashLogging.sh +source src/main/resources/io/viash/helpers/bashutils/ViashDockerAutodetectMount.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + +# Set up test environment +TEST_DIR=$(mktemp -d) +cleanup() { + rm -rf "$TEST_DIR" +} +trap cleanup EXIT + +# Create test directory structure +mkdir -p "$TEST_DIR/subdir" +touch "$TEST_DIR/file.txt" +touch "$TEST_DIR/subdir/nested.txt" + + +## TEST1: test ViashDockerAutodetectMount with default prefix + +# Set default automount prefix +VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" + +# TEST1a: File path +output=$(ViashDockerAutodetectMount "$TEST_DIR/file.txt") +assert_value_equal "test1a_output" "/viash_automount$TEST_DIR/file.txt" "$output" + +# TEST1b: Directory path +output=$(ViashDockerAutodetectMount "$TEST_DIR/subdir") +assert_value_equal "test1b_output" "/viash_automount$TEST_DIR/subdir" "$output" + +# TEST1c: Nested file path +output=$(ViashDockerAutodetectMount "$TEST_DIR/subdir/nested.txt") +assert_value_equal "test1c_output" "/viash_automount$TEST_DIR/subdir/nested.txt" "$output" + + +## TEST2: test ViashDockerAutodetectMount with custom prefix + +VIASH_DOCKER_AUTOMOUNT_PREFIX="/custom_mount" + +# TEST2a: File with custom prefix +output=$(ViashDockerAutodetectMount "$TEST_DIR/file.txt") +assert_value_equal "test2a_output" "/custom_mount$TEST_DIR/file.txt" "$output" + +# TEST2b: Directory with custom prefix +output=$(ViashDockerAutodetectMount "$TEST_DIR/subdir") +assert_value_equal "test2b_output" "/custom_mount$TEST_DIR/subdir" "$output" + + +## TEST3: test ViashDockerAutodetectMount with empty prefix + +VIASH_DOCKER_AUTOMOUNT_PREFIX="" + +# TEST3a: File with empty prefix (path should be unchanged) +output=$(ViashDockerAutodetectMount "$TEST_DIR/file.txt") +assert_value_equal "test3a_output" "$TEST_DIR/file.txt" "$output" + + +## TEST4: test ViashDockerAutodetectMountArg + +VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" + +# TEST4a: Volume mount for file +output=$(ViashDockerAutodetectMountArg "$TEST_DIR/file.txt") +assert_value_equal "test4a_output" "--volume=\"$TEST_DIR:/viash_automount$TEST_DIR\"" "$output" + +# TEST4b: Volume mount for directory +output=$(ViashDockerAutodetectMountArg "$TEST_DIR/subdir") +assert_value_equal "test4b_output" "--volume=\"$TEST_DIR/subdir:/viash_automount$TEST_DIR/subdir\"" "$output" + + +## TEST5: test ViashDockerStripAutomount + +VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" + +# TEST5a: Strip prefix from automounted path +output=$(ViashDockerStripAutomount "/viash_automount$TEST_DIR/file.txt") +assert_value_equal "test5a_output" "$TEST_DIR/file.txt" "$output" + +# TEST5b: Path without prefix should remain unchanged +output=$(ViashDockerStripAutomount "$TEST_DIR/file.txt") +assert_value_equal "test5b_output" "$TEST_DIR/file.txt" "$output" + +# TEST5c: Path with different prefix +output=$(ViashDockerStripAutomount "/other_prefix$TEST_DIR/file.txt") +assert_value_equal "test5c_output" "/other_prefix$TEST_DIR/file.txt" "$output" + + +## TEST6: test with relative paths (should be converted to absolute) + +VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" + +# TEST6a: Relative path +cd "$TEST_DIR" +touch "relative_file.txt" +output=$(ViashDockerAutodetectMount "relative_file.txt") +assert_value_equal "test6a_output" "/viash_automount$TEST_DIR/relative_file.txt" "$output" + +# Go back to original directory +cd - > /dev/null + +print_test_summary diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashLogging.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashLogging.test.sh new file mode 100644 index 000000000..b442e936b --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashLogging.test.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashLogging.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + +## TEST1: test log level constants + +assert_value_equal "VIASH_LOGCODE_EMERGENCY" "0" "$VIASH_LOGCODE_EMERGENCY" +assert_value_equal "VIASH_LOGCODE_ALERT" "1" "$VIASH_LOGCODE_ALERT" +assert_value_equal "VIASH_LOGCODE_CRITICAL" "2" "$VIASH_LOGCODE_CRITICAL" +assert_value_equal "VIASH_LOGCODE_ERROR" "3" "$VIASH_LOGCODE_ERROR" +assert_value_equal "VIASH_LOGCODE_WARNING" "4" "$VIASH_LOGCODE_WARNING" +assert_value_equal "VIASH_LOGCODE_NOTICE" "5" "$VIASH_LOGCODE_NOTICE" +assert_value_equal "VIASH_LOGCODE_INFO" "6" "$VIASH_LOGCODE_INFO" +assert_value_equal "VIASH_LOGCODE_DEBUG" "7" "$VIASH_LOGCODE_DEBUG" + + +## TEST2: test default verbosity level + +assert_value_equal "VIASH_VERBOSITY" "$VIASH_LOGCODE_NOTICE" "$VIASH_VERBOSITY" + + +## TEST3: test ViashLog outputs at correct verbosity levels + +# TEST3a: Log at notice level (should output at default verbosity) +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE +output=$(ViashLog $VIASH_LOGCODE_NOTICE notice "Test message" 2>&1) +assert_value_equal "test3a_output" "[notice] Test message" "$output" + +# TEST3b: Log at info level (should NOT output at notice verbosity) +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE +output=$(ViashLog $VIASH_LOGCODE_INFO info "Test message" 2>&1) +assert_value_equal "test3b_output" "" "$output" + +# TEST3c: Log at info level (should output at info verbosity) +VIASH_VERBOSITY=$VIASH_LOGCODE_INFO +output=$(ViashLog $VIASH_LOGCODE_INFO info "Test message" 2>&1) +assert_value_equal "test3c_output" "[info] Test message" "$output" + +# TEST3d: Log at debug level (should NOT output at info verbosity) +VIASH_VERBOSITY=$VIASH_LOGCODE_INFO +output=$(ViashLog $VIASH_LOGCODE_DEBUG debug "Test message" 2>&1) +assert_value_equal "test3d_output" "" "$output" + +# TEST3e: Log at error level (should output at notice verbosity) +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE +output=$(ViashLog $VIASH_LOGCODE_ERROR error "Test message" 2>&1) +assert_value_equal "test3e_output" "[error] Test message" "$output" + + +## TEST4: test convenience functions + +# Reset to default verbosity +VIASH_VERBOSITY=$VIASH_LOGCODE_DEBUG + +# TEST4a: ViashEmergency +output=$(ViashEmergency "Emergency test" 2>&1) +assert_value_equal "test4a_output" "[emergency] Emergency test" "$output" + +# TEST4b: ViashAlert +output=$(ViashAlert "Alert test" 2>&1) +assert_value_equal "test4b_output" "[alert] Alert test" "$output" + +# TEST4c: ViashCritical +output=$(ViashCritical "Critical test" 2>&1) +assert_value_equal "test4c_output" "[critical] Critical test" "$output" + +# TEST4d: ViashError +output=$(ViashError "Error test" 2>&1) +assert_value_equal "test4d_output" "[error] Error test" "$output" + +# TEST4e: ViashWarning +output=$(ViashWarning "Warning test" 2>&1) +assert_value_equal "test4e_output" "[warning] Warning test" "$output" + +# TEST4f: ViashNotice +output=$(ViashNotice "Notice test" 2>&1) +assert_value_equal "test4f_output" "[notice] Notice test" "$output" + +# TEST4g: ViashInfo +output=$(ViashInfo "Info test" 2>&1) +assert_value_equal "test4g_output" "[info] Info test" "$output" + +# TEST4h: ViashDebug +output=$(ViashDebug "Debug test" 2>&1) +assert_value_equal "test4h_output" "[debug] Debug test" "$output" + + +## TEST5: test multiple word messages + +VIASH_VERBOSITY=$VIASH_LOGCODE_DEBUG + +# TEST5a: Multiple words in message +output=$(ViashInfo "This is a multi-word message" 2>&1) +assert_value_equal "test5a_output" "[info] This is a multi-word message" "$output" + +# TEST5b: Message with special characters +output=$(ViashInfo 'Message with $pecial chars: "quotes" and '\''singles'\''' 2>&1) +assert_value_equal "test5b_output" "[info] Message with \$pecial chars: \"quotes\" and 'singles'" "$output" + + +## TEST6: test verbosity boundary conditions + +# TEST6a: Verbosity at emergency (0) should only show emergency +VIASH_VERBOSITY=$VIASH_LOGCODE_EMERGENCY +output=$(ViashEmergency "Emergency" 2>&1) +assert_value_equal "test6a_emergency" "[emergency] Emergency" "$output" +output=$(ViashAlert "Alert" 2>&1) +assert_value_equal "test6a_alert" "" "$output" + +# TEST6b: Verbosity at max (debug=7) should show everything +VIASH_VERBOSITY=$VIASH_LOGCODE_DEBUG +output=$(ViashEmergency "Emergency" 2>&1) +assert_value_equal "test6b_emergency" "[emergency] Emergency" "$output" +output=$(ViashDebug "Debug" 2>&1) +assert_value_equal "test6b_debug" "[debug] Debug" "$output" + +print_test_summary diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashParseArgumentValue.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashParseArgumentValue.test.sh new file mode 100755 index 000000000..47d40a8b2 --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashParseArgumentValue.test.sh @@ -0,0 +1,407 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashParseArgumentValue.sh +source src/main/resources/io/viash/helpers/bashutils/ViashLogging.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + +## TEST1: test simple strings + +# TEST1a simple use case +ViashParseArgumentValue "--input" "par_input" "false" 'input.txt' + +assert_value_equal "par_input" 'input.txt' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST1b multiple values +ViashParseArgumentValue "--input" "par_input" "true" 'input1.txt;input2.txt' + +assert_value_equal "par_input" 'input1.txt input2.txt' "${par_input[@]}" +assert_value_equal "par_input" 2 "${#par_input[@]}" + +unset par_input + + + + +## TEST2: test quoting + +# TEST2a resolve quotes +ViashParseArgumentValue "--input" "par_input" "false" '"input.txt"' + +assert_value_equal "par_input" 'input.txt' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST2b resolve single quotes + +ViashParseArgumentValue "--input" "par_input" "false" "'input.txt'" + +assert_value_equal "par_input" 'input.txt' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST2c resolve quotes with multiple values +ViashParseArgumentValue "--input" "par_input" "true" '"input1.txt";"input2.txt"' + +assert_value_equal "par_input" 'input1.txt input2.txt' "${par_input[@]}" +assert_value_equal "par_input" 2 "${#par_input[@]}" + +unset par_input + +# TEST2d resolve quotes with quotes in the value +ViashParseArgumentValue "--input" "par_input" "false" "\"foo'bar'\"" + +# todo: check +assert_value_equal "par_input" "foo'bar'" "${par_input[@]}" +unset par_input + + + +## TEST3: test undefined + +# TEST3a escape undefined +ViashParseArgumentValue "--input" "par_input" "false" 'UNDEFINED' + +assert_value_equal "par_input" '@@VIASH_UNDEFINED@@' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST3b do not escape undefined when quoted +ViashParseArgumentValue "--input" "par_input" "false" '"UNDEFINED"' "false" + +assert_value_equal "par_input" 'UNDEFINED' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST3c do not escape undefined when single quoted +ViashParseArgumentValue "--input" "par_input" "false" "'UNDEFINED'" "false" + +assert_value_equal "par_input" 'UNDEFINED' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST3d escape undefined when multiple values +ViashParseArgumentValue "--input" "par_input" "true" 'UNDEFINED' + +assert_value_equal "par_input" '@@VIASH_UNDEFINED@@' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST3e do not escape undefined when multiple values and quoted +ViashParseArgumentValue "--input" "par_input" "true" '"UNDEFINED"' "false" + +assert_value_equal "par_input" 'UNDEFINED' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST3f do not escape undefined when multiple values and single quoted +ViashParseArgumentValue "--input" "par_input" "true" "'UNDEFINED'" "false" + +assert_value_equal "par_input" 'UNDEFINED' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + + +## TEST4: test undefined_item + +# TEST4a do not escape undefined_item for single value +ViashParseArgumentValue "--input" "par_input" "false" 'UNDEFINED_ITEM' + +assert_value_equal "par_input" 'UNDEFINED_ITEM' "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST4b escape undefined_item for multiple values +ViashParseArgumentValue "--input" "par_input" "true" 'UNDEFINED_ITEM;a;b;UNDEFINED_ITEM' + +assert_value_equal "par_input" '@@VIASH_UNDEFINED_ITEM@@ a b @@VIASH_UNDEFINED_ITEM@@' "${par_input[@]}" +assert_value_equal "par_input" 4 "${#par_input[@]}" + +unset par_input + +# TEST4c do not escape undefined_item for multiple values when quoted +ViashParseArgumentValue "--input" "par_input" "true" "\"UNDEFINED_ITEM\";a;'UNDEFINED_ITEM';UNDEFINED_ITEM" + +assert_value_equal "par_input" 'UNDEFINED_ITEM a UNDEFINED_ITEM @@VIASH_UNDEFINED_ITEM@@' "${par_input[@]}" +assert_value_equal "par_input" 4 "${#par_input[@]}" + +unset par_input + + +## TEST5: test escaping of special characters + +# TEST5a do not escape single strings +ViashParseArgumentValue "--input" "par_input" "false" "a\;b\'\\\"c" + +assert_value_equal "par_input" "a\;b\'\\\"c" "${par_input[@]}" +assert_value_equal "par_input" 1 "${#par_input[@]}" + +unset par_input + +# TEST5b escape multiple values +ViashParseArgumentValue "--input" "par_input" "true" "a\;b\'\\\"c;d;e" + +assert_value_equal "par_input" "a;b'\"c d e" "${par_input[@]}" +assert_value_equal "par_input" 3 "${#par_input[@]}" + +unset par_input + +# TEST5c escape multiple values with quotes +ViashParseArgumentValue "--input" "par_input" "true" "\"a\;b\'\\\"c\";d;'e'" +assert_value_equal "par_input" "a;b'\"c d e" "${par_input[@]}" +assert_value_equal "par_input" 3 "${#par_input[@]}" + +unset par_input + +# TEST5d escape multiple values with single quotes +ViashParseArgumentValue "--input" "par_input" "true" "'a\;b\'\\\"c';d;'e'" + +assert_value_equal "par_input" "a;b'\"c d e" "${par_input[@]}" +assert_value_equal "par_input" 3 "${#par_input[@]}" + +unset par_input + + +## TEST6: test special shell characters that could cause command injection + +# TEST6a: backticks should not be executed +ViashParseArgumentValue "--input" "par_input" "false" 'value with `echo dangerous`' + +assert_value_equal "par_input_backtick" 'value with `echo dangerous`' "${par_input[@]}" +assert_value_equal "par_input_backtick_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST6b: dollar signs should be preserved literally +ViashParseArgumentValue "--input" "par_input" "false" 'value with $HOME variable' + +assert_value_equal "par_input_dollar" 'value with $HOME variable' "${par_input[@]}" +assert_value_equal "par_input_dollar_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST6c: command substitution syntax should not be executed +ViashParseArgumentValue "--input" "par_input" "false" 'value with $(whoami)' + +assert_value_equal "par_input_subst" 'value with $(whoami)' "${par_input[@]}" +assert_value_equal "par_input_subst_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST6d: complex string with multiple special characters +ViashParseArgumentValue "--input" "par_input" "false" 'a \ b $ c ` d " e '\'' f' + +assert_value_equal "par_input_complex" 'a \ b $ c ` d " e '\'' f' "${par_input[@]}" +assert_value_equal "par_input_complex_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST6e: newline escape sequences +ViashParseArgumentValue "--input" "par_input" "false" 'line1\nline2' + +assert_value_equal "par_input_newline" 'line1\nline2' "${par_input[@]}" +assert_value_equal "par_input_newline_len" 1 "${#par_input[@]}" + +unset par_input + + +## TEST7: test accumulating multiple values + +# TEST7a: multiple calls should accumulate values +ViashParseArgumentValue "--input" "par_input" "true" 'value1' +ViashParseArgumentValue "--input" "par_input" "true" 'value2' + +assert_value_equal "par_input_accum" 'value1 value2' "${par_input[@]}" +assert_value_equal "par_input_accum_len" 2 "${#par_input[@]}" + +unset par_input + +# TEST7b: multiple calls with semicolon-separated values +ViashParseArgumentValue "--input" "par_input" "true" 'a;b' +ViashParseArgumentValue "--input" "par_input" "true" 'c;d' + +assert_value_equal "par_input_multi_accum" 'a b c d' "${par_input[@]}" +assert_value_equal "par_input_multi_accum_len" 4 "${#par_input[@]}" + +unset par_input + + +## TEST8: test empty and whitespace values + +# TEST8a: empty string for multiple (results in 0-length array) +ViashParseArgumentValue "--input" "par_input" "true" '' + +assert_value_equal "par_input_empty_multi" '' "${par_input[@]}" +assert_value_equal "par_input_empty_multi_len" 0 "${#par_input[@]}" + +unset par_input + +# TEST8b: value with only spaces +ViashParseArgumentValue "--input" "par_input" "false" ' ' + +assert_value_equal "par_input_spaces" ' ' "${par_input[@]}" +assert_value_equal "par_input_spaces_len" 1 "${#par_input[@]}" + +unset par_input + + +## TEST9: comprehensive special character escape tests + +# TEST9a: multiple backslashes +ViashParseArgumentValue "--input" "par_input" "false" 'a\\b\\\\c' + +assert_value_equal "par_input_multibackslash" 'a\\b\\\\c' "${par_input[@]}" +assert_value_equal "par_input_multibackslash_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9b: nested backticks +ViashParseArgumentValue "--input" "par_input" "false" '`echo `nested``' + +assert_value_equal "par_input_nested_backtick" '`echo `nested``' "${par_input[@]}" +assert_value_equal "par_input_nested_backtick_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9c: nested command substitution +ViashParseArgumentValue "--input" "par_input" "false" '$(echo $(whoami))' + +assert_value_equal "par_input_nested_subst" '$(echo $(whoami))' "${par_input[@]}" +assert_value_equal "par_input_nested_subst_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9d: mixed dollar signs and brackets +ViashParseArgumentValue "--input" "par_input" "false" '$HOME/${USER}/$(date)' + +assert_value_equal "par_input_mixed_dollar" '$HOME/${USER}/$(date)' "${par_input[@]}" +assert_value_equal "par_input_mixed_dollar_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9e: arithmetic expansion syntax +ViashParseArgumentValue "--input" "par_input" "false" '$((1+2))' + +assert_value_equal "par_input_arith" '$((1+2))' "${par_input[@]}" +assert_value_equal "par_input_arith_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9f: process substitution syntax +ViashParseArgumentValue "--input" "par_input" "false" '<(cat file)' + +assert_value_equal "par_input_procsub" '<(cat file)' "${par_input[@]}" +assert_value_equal "par_input_procsub_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9g: glob patterns (should be preserved as literal) +ViashParseArgumentValue "--input" "par_input" "false" '*.txt' + +assert_value_equal "par_input_glob" '*.txt' "${par_input[@]}" +assert_value_equal "par_input_glob_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9h: brace expansion syntax +ViashParseArgumentValue "--input" "par_input" "false" '{a,b,c}' + +assert_value_equal "par_input_brace" '{a,b,c}' "${par_input[@]}" +assert_value_equal "par_input_brace_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9i: exclamation mark (history expansion) +ViashParseArgumentValue "--input" "par_input" "false" 'hello!world' + +assert_value_equal "par_input_exclaim" 'hello!world' "${par_input[@]}" +assert_value_equal "par_input_exclaim_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9j: pipe and redirect characters +ViashParseArgumentValue "--input" "par_input" "false" 'cmd | other > file' + +assert_value_equal "par_input_pipe" 'cmd | other > file' "${par_input[@]}" +assert_value_equal "par_input_pipe_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9k: ampersand (background execution) +ViashParseArgumentValue "--input" "par_input" "false" 'cmd1 && cmd2 & cmd3' + +assert_value_equal "par_input_amp" 'cmd1 && cmd2 & cmd3' "${par_input[@]}" +assert_value_equal "par_input_amp_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9l: semicolon in single value mode (should not split) +ViashParseArgumentValue "--input" "par_input" "false" 'a;b;c' + +assert_value_equal "par_input_semicolon_single" 'a;b;c' "${par_input[@]}" +assert_value_equal "par_input_semicolon_single_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9m: tab and newline escape sequences +ViashParseArgumentValue "--input" "par_input" "false" 'line1\tline2\nline3' + +assert_value_equal "par_input_escapes" 'line1\tline2\nline3' "${par_input[@]}" +assert_value_equal "par_input_escapes_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9n: null byte representation +ViashParseArgumentValue "--input" "par_input" "false" 'before\0after' + +assert_value_equal "par_input_null" 'before\0after' "${par_input[@]}" +assert_value_equal "par_input_null_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9o: unicode characters +ViashParseArgumentValue "--input" "par_input" "false" 'café résumé 日本語' + +assert_value_equal "par_input_unicode" 'café résumé 日本語' "${par_input[@]}" +assert_value_equal "par_input_unicode_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9p: special characters in multi-value mode +ViashParseArgumentValue "--input" "par_input" "true" '`cmd`;$var;$(sub)' + +assert_value_equal "par_input_special_multi" '`cmd` $var $(sub)' "${par_input[@]}" +assert_value_equal "par_input_special_multi_len" 3 "${#par_input[@]}" + +unset par_input + +# TEST9q: Windows path with backslashes +ViashParseArgumentValue "--input" "par_input" "false" 'C:\Users\test\file.txt' + +assert_value_equal "par_input_winpath" 'C:\Users\test\file.txt' "${par_input[@]}" +assert_value_equal "par_input_winpath_len" 1 "${#par_input[@]}" + +unset par_input + +# TEST9r: regex-like pattern +ViashParseArgumentValue "--input" "par_input" "false" '^[a-z]+\.[0-9]*$' + +assert_value_equal "par_input_regex" '^[a-z]+\.[0-9]*$' "${par_input[@]}" +assert_value_equal "par_input_regex_len" 1 "${#par_input[@]}" + +unset par_input + + +print_test_summary \ No newline at end of file diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashQuote.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashQuote.test.sh new file mode 100644 index 000000000..86066f9c5 --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashQuote.test.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashQuote.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + +## TEST1: test flag values (should not be quoted) + +# TEST1a: Simple flag +output=$(ViashQuote "--foo") +assert_value_equal "test1a_output" "--foo" "$output" + +# TEST1b: Flag with underscore +output=$(ViashQuote "--my_flag") +assert_value_equal "test1b_output" "--my_flag" "$output" + +# TEST1c: Flag with hyphen +output=$(ViashQuote "--my-flag") +assert_value_equal "test1c_output" "--my-flag" "$output" + +# TEST1d: Single-dash flag +output=$(ViashQuote "-f") +assert_value_equal "test1d_output" "-f" "$output" + +# TEST1e: Flag with numbers +output=$(ViashQuote "--option123") +assert_value_equal "test1e_output" "--option123" "$output" + + +## TEST2: test plain values (should be quoted) + +# TEST2a: Simple string +output=$(ViashQuote "bar") +assert_value_equal "test2a_output" "'bar'" "$output" + +# TEST2b: String with spaces +output=$(ViashQuote "hello world") +assert_value_equal "test2b_output" "'hello world'" "$output" + +# TEST2c: Numeric string +output=$(ViashQuote "123") +assert_value_equal "test2c_output" "'123'" "$output" + +# TEST2d: Path +output=$(ViashQuote "/path/to/file.txt") +assert_value_equal "test2d_output" "'/path/to/file.txt'" "$output" + +# TEST2e: Empty string +output=$(ViashQuote "") +assert_value_equal "test2e_output" "''" "$output" + + +## TEST3: test flag=value combinations + +# TEST3a: Simple flag=value +output=$(ViashQuote "--foo=bar") +assert_value_equal "test3a_output" "--foo='bar'" "$output" + +# TEST3b: Flag=value with spaces in value +output=$(ViashQuote "--message=hello world") +assert_value_equal "test3b_output" "--message='hello world'" "$output" + +# TEST3c: Flag=value with path +output=$(ViashQuote "--input=/path/to/file.txt") +assert_value_equal "test3c_output" "--input='/path/to/file.txt'" "$output" + +# TEST3d: Flag=value with numbers +output=$(ViashQuote "--count=42") +assert_value_equal "test3d_output" "--count='42'" "$output" + +# TEST3e: Single-dash flag=value +output=$(ViashQuote "-o=output.txt") +assert_value_equal "test3e_output" "-o='output.txt'" "$output" + + +## TEST4: test edge cases + +# TEST4a: String starting with equals (should be quoted, not treated as flag=value) +output=$(ViashQuote "=value") +assert_value_equal "test4a_output" "'=value'" "$output" + +# TEST4b: String with only hyphens (treated as flag-like pattern since regex allows hyphens) +output=$(ViashQuote "---") +assert_value_equal "test4b_output" "---" "$output" + +# TEST4c: Flag with trailing equals but no value (edge case) +output=$(ViashQuote "--foo=") +# This may or may not match the flag=value pattern depending on implementation +# The current implementation requires at least one char after = + +# TEST4d: Value that looks like a flag inside +output=$(ViashQuote "not--a--flag") +assert_value_equal "test4d_output" "'not--a--flag'" "$output" + +print_test_summary diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashRemoveFlags.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashRemoveFlags.test.sh new file mode 100644 index 000000000..cc7606b73 --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashRemoveFlags.test.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashRemoveFlags.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + + +## TEST1: test removing double-dash flags + +# TEST1a: Simple flag=value +output=$(ViashRemoveFlags "--foo=bar") +assert_value_equal "test1a_output" "bar" "$output" + +# TEST1b: Flag with underscore +output=$(ViashRemoveFlags "--my_flag=value") +assert_value_equal "test1b_output" "value" "$output" + +# TEST1c: Flag with hyphen +output=$(ViashRemoveFlags "--my-flag=value") +assert_value_equal "test1c_output" "value" "$output" + +# TEST1d: Flag with numbers +output=$(ViashRemoveFlags "--option123=value") +assert_value_equal "test1d_output" "value" "$output" + + +## TEST2: test removing single-dash flags + +# TEST2a: Single letter flag +output=$(ViashRemoveFlags "-f=value") +assert_value_equal "test2a_output" "value" "$output" + +# TEST2b: Multiple letter single-dash flag +output=$(ViashRemoveFlags "-abc=value") +assert_value_equal "test2b_output" "value" "$output" + + +## TEST3: test values containing special characters + +# TEST3a: Value with spaces +output=$(ViashRemoveFlags "--message=hello world") +assert_value_equal "test3a_output" "hello world" "$output" + +# TEST3b: Value with path +output=$(ViashRemoveFlags "--input=/path/to/file.txt") +assert_value_equal "test3b_output" "/path/to/file.txt" "$output" + +# TEST3c: Value with equals sign +output=$(ViashRemoveFlags "--equation=a=b+c") +assert_value_equal "test3c_output" "a=b+c" "$output" + +# TEST3d: Value with multiple equals signs +output=$(ViashRemoveFlags "--param=key=value=extra") +assert_value_equal "test3d_output" "key=value=extra" "$output" + + +## TEST4: test strings without flags (should pass through unchanged) + +# TEST4a: Plain value +output=$(ViashRemoveFlags "just_a_value") +assert_value_equal "test4a_output" "just_a_value" "$output" + +# TEST4b: Value starting with equals +output=$(ViashRemoveFlags "=value") +assert_value_equal "test4b_output" "=value" "$output" + +# TEST4c: Value with dashes in middle (not a flag) +output=$(ViashRemoveFlags "not--a-flag") +assert_value_equal "test4c_output" "not--a-flag" "$output" + + +## TEST5: test edge cases + +# TEST5a: Empty value after flag +output=$(ViashRemoveFlags "--flag=") +assert_value_equal "test5a_output" "" "$output" + +# TEST5b: Flag only (no equals sign) - should pass through unchanged +output=$(ViashRemoveFlags "--flag") +assert_value_equal "test5b_output" "--flag" "$output" + +# TEST5c: Numeric value +output=$(ViashRemoveFlags "--count=42") +assert_value_equal "test5c_output" "42" "$output" + +# TEST5d: Quoted value +output=$(ViashRemoveFlags '--message="hello world"') +assert_value_equal "test5d_output" '"hello world"' "$output" + +print_test_summary diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashRenderJson.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashRenderJson.test.sh new file mode 100644 index 000000000..b04243226 --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashRenderJson.test.sh @@ -0,0 +1,323 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashRenderJson.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + + +## TEST1: test ViashRenderJsonKeyValue with string types + +# TEST1a: Single string value +output=$(ViashRenderJsonKeyValue "input" "string" "false" "file.txt") +assert_value_equal "test1a_output" ' "input": "file.txt"' "$output" + +# TEST1b: Single string with spaces +output=$(ViashRenderJsonKeyValue "message" "string" "false" "hello world") +assert_value_equal "test1b_output" ' "message": "hello world"' "$output" + +# TEST1c: Multiple string values (array) +output=$(ViashRenderJsonKeyValue "inputs" "string" "true" "file1.txt" "file2.txt" "file3.txt") +assert_value_equal "test1c_output" ' "inputs": [ "file1.txt", "file2.txt", "file3.txt" ]' "$output" + + +## TEST2: test ViashRenderJsonKeyValue with numeric types + +# TEST2a: Integer value +output=$(ViashRenderJsonKeyValue "count" "integer" "false" "42") +assert_value_equal "test2a_output" ' "count": 42' "$output" + +# TEST2b: Double value +output=$(ViashRenderJsonKeyValue "ratio" "double" "false" "3.14159") +assert_value_equal "test2b_output" ' "ratio": 3.14159' "$output" + +# TEST2c: Multiple integer values +output=$(ViashRenderJsonKeyValue "numbers" "integer" "true" "1" "2" "3") +assert_value_equal "test2c_output" ' "numbers": [ 1, 2, 3 ]' "$output" + + +## TEST3: test ViashRenderJsonKeyValue with boolean types + +# TEST3a: Boolean true +output=$(ViashRenderJsonKeyValue "enabled" "boolean" "false" "true") +assert_value_equal "test3a_output" ' "enabled": true' "$output" + +# TEST3b: Boolean false +output=$(ViashRenderJsonKeyValue "enabled" "boolean" "false" "false") +assert_value_equal "test3b_output" ' "enabled": false' "$output" + +# TEST3c: Boolean yes (should convert to true) +output=$(ViashRenderJsonKeyValue "enabled" "boolean" "false" "yes") +assert_value_equal "test3c_output" ' "enabled": true' "$output" + +# TEST3d: Boolean no (should convert to false) +output=$(ViashRenderJsonKeyValue "enabled" "boolean" "false" "no") +assert_value_equal "test3d_output" ' "enabled": false' "$output" + +# TEST3e: Boolean TRUE (uppercase, should convert to true) +output=$(ViashRenderJsonKeyValue "enabled" "boolean" "false" "TRUE") +assert_value_equal "test3e_output" ' "enabled": true' "$output" + +# TEST3f: Multiple boolean values +output=$(ViashRenderJsonKeyValue "flags" "boolean" "true" "true" "false" "true") +assert_value_equal "test3f_output" ' "flags": [ true, false, true ]' "$output" + + +## TEST4: test ViashRenderJsonKeyValue with null values + +# TEST4a: Single undefined value (should render as null) +output=$(ViashRenderJsonKeyValue "optional" "string" "false" "@@VIASH_UNDEFINED@@") +assert_value_equal "test4a_output" ' "optional": null' "$output" + +# TEST4b: Undefined item in array (should render as null in array) +output=$(ViashRenderJsonKeyValue "items" "string" "true" "a" "@@VIASH_UNDEFINED_ITEM@@" "c") +assert_value_equal "test4b_output" ' "items": [ "a", null, "c" ]' "$output" + + +## TEST5: test ViashRenderJsonKeyValue with file type + +# TEST5a: File type (should be quoted like string) +output=$(ViashRenderJsonKeyValue "input" "file" "false" "/path/to/file.txt") +assert_value_equal "test5a_output" ' "input": "/path/to/file.txt"' "$output" + +# TEST5b: Multiple file values +output=$(ViashRenderJsonKeyValue "inputs" "file" "true" "/a.txt" "/b.txt") +assert_value_equal "test5b_output" ' "inputs": [ "/a.txt", "/b.txt" ]' "$output" + + +## TEST6: test ViashRenderJsonQuotedValue with special characters + +# TEST6a: String with quotes (should be escaped) +output=$(ViashRenderJsonQuotedValue "key" 'say "hello"') +assert_value_equal "test6a_output" '"say \"hello\""' "$output" + +# TEST6b: String with backslash (should be escaped) +output=$(ViashRenderJsonQuotedValue "key" 'path\to\file') +assert_value_equal "test6b_output" '"path\\to\\file"' "$output" + +# TEST6c: String with newline (should be escaped) +output=$(ViashRenderJsonQuotedValue "key" $'line1\nline2') +assert_value_equal "test6c_output" '"line1\nline2"' "$output" + +# TEST6d: String with all special characters +output=$(ViashRenderJsonQuotedValue "key" 'a"b\c') +assert_value_equal "test6d_output" '"a\"b\\c"' "$output" + +# TEST6e: String with backticks (should be preserved as literal) +output=$(ViashRenderJsonQuotedValue "key" 'echo `whoami`') +assert_value_equal "test6e_backticks" '"echo `whoami`"' "$output" + +# TEST6f: String with dollar sign (should be preserved as literal) +output=$(ViashRenderJsonQuotedValue "key" 'value is $HOME') +assert_value_equal "test6f_dollar" '"value is $HOME"' "$output" + +# TEST6g: String with command substitution syntax (should be preserved as literal) +output=$(ViashRenderJsonQuotedValue "key" 'run $(date)') +assert_value_equal "test6g_subst" '"run $(date)"' "$output" + +# TEST6h: String with single quotes +output=$(ViashRenderJsonQuotedValue "key" "it's a test") +assert_value_equal "test6h_singlequote" '"it'\''s a test"' "$output" + +# TEST6i: String with tab character (tab is preserved as literal tab in JSON) +output=$(ViashRenderJsonQuotedValue "key" $'col1\tcol2') +assert_value_equal "test6i_tab" $'"col1\tcol2"' "$output" + +# TEST6j: String with carriage return (cr is preserved as literal in JSON) +output=$(ViashRenderJsonQuotedValue "key" $'line1\rline2') +assert_value_equal "test6j_cr" $'"line1\rline2"' "$output" + +# TEST6k: Complex string with multiple special chars +output=$(ViashRenderJsonQuotedValue "key" 'a\b"c$d`e') +assert_value_equal "test6k_complex" '"a\\b\"c$d`e"' "$output" + + +## TEST7: test ViashRenderJsonBooleanValue + +# TEST7a: true +output=$(ViashRenderJsonBooleanValue "flag" "true") +assert_value_equal "test7a_output" "true" "$output" + +# TEST7b: false +output=$(ViashRenderJsonBooleanValue "flag" "false") +assert_value_equal "test7b_output" "false" "$output" + +# TEST7c: yes +output=$(ViashRenderJsonBooleanValue "flag" "yes") +assert_value_equal "test7c_output" "true" "$output" + +# TEST7d: no +output=$(ViashRenderJsonBooleanValue "flag" "no") +assert_value_equal "test7d_output" "false" "$output" + +# TEST7e: YES (uppercase) +output=$(ViashRenderJsonBooleanValue "flag" "YES") +assert_value_equal "test7e_output" "true" "$output" + + +## TEST8: test ViashRenderJsonUnquotedValue + +# TEST8a: Simple value +output=$(ViashRenderJsonUnquotedValue "num" "42") +assert_value_equal "test8a_output" "42" "$output" + +# TEST8b: Decimal value +output=$(ViashRenderJsonUnquotedValue "num" "3.14") +assert_value_equal "test8b_output" "3.14" "$output" + + +## TEST9: test single value array + +# TEST9a: Single element in multiple mode +output=$(ViashRenderJsonKeyValue "items" "string" "true" "only_one") +assert_value_equal "test9a_output" ' "items": [ "only_one" ]' "$output" + +# TEST9b: Empty string value +output=$(ViashRenderJsonKeyValue "empty" "string" "false" "") +assert_value_equal "test9b_output" ' "empty": ""' "$output" + + +## TEST10: test ViashRenderJsonKeyValue with dangerous shell characters + +# TEST10a: String value with backticks +output=$(ViashRenderJsonKeyValue "cmd" "string" "false" 'run `id`') +assert_value_equal "test10a_backtick" ' "cmd": "run `id`"' "$output" + +# TEST10b: String value with dollar sign +output=$(ViashRenderJsonKeyValue "env" "string" "false" 'path is $PATH') +assert_value_equal "test10b_dollar" ' "env": "path is $PATH"' "$output" + +# TEST10c: String value with command substitution +output=$(ViashRenderJsonKeyValue "sub" "string" "false" 'time $(date)') +assert_value_equal "test10c_subst" ' "sub": "time $(date)"' "$output" + +# TEST10d: String value with backslash +output=$(ViashRenderJsonKeyValue "path" "string" "false" 'C:\Users\test') +assert_value_equal "test10d_backslash" ' "path": "C:\\Users\\test"' "$output" + +# TEST10e: Array with dangerous characters +output=$(ViashRenderJsonKeyValue "items" "string" "true" 'a`b' 'c$d' 'e\f') +assert_value_equal "test10e_array" ' "items": [ "a`b", "c$d", "e\\f" ]' "$output" + +# TEST10f: Multiple backslashes +output=$(ViashRenderJsonKeyValue "path" "string" "false" 'a\\b\\\\c') +assert_value_equal "test10f_multibackslash" ' "path": "a\\\\b\\\\\\\\c"' "$output" + +# TEST10g: Nested quotes +output=$(ViashRenderJsonKeyValue "nested" "string" "false" 'He said "Hello"') +assert_value_equal "test10g_nestedquotes" ' "nested": "He said \"Hello\""' "$output" + +# TEST10h: Unicode and special chars (if supported) +output=$(ViashRenderJsonKeyValue "unicode" "string" "false" 'café') +assert_value_equal "test10h_unicode" ' "unicode": "café"' "$output" + + +## TEST11: test ViashRenderJsonBooleanValue with invalid values (should exit with error) + +# TEST11a: Invalid boolean value should exit with code 1 +set +e # Temporarily disable exit on error +stderr=$(ViashRenderJsonBooleanValue "flag" "invalid" 2>&1 >/dev/null) +exit_code=$? +set -e # Re-enable exit on error +assert_exit_code "test11a_exit_code" 1 "$exit_code" +assert_contains "test11a_stderr" "Argument 'flag' has to be a boolean" "$stderr" + +# TEST11b: Invalid boolean like "maybe" should exit with error +set +e +stderr=$(ViashRenderJsonBooleanValue "enabled" "maybe" 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test11b_exit_code" 1 "$exit_code" +assert_contains "test11b_stderr" "has to be a boolean" "$stderr" + +# TEST11c: Invalid boolean like "1" should exit with error +set +e +stderr=$(ViashRenderJsonBooleanValue "flag" "1" 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test11c_exit_code" 1 "$exit_code" +assert_contains "test11c_stderr" "has to be a boolean" "$stderr" + +# TEST11d: Empty boolean value should exit with error +set +e +stderr=$(ViashRenderJsonBooleanValue "flag" "" 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test11d_exit_code" 1 "$exit_code" +assert_contains "test11d_stderr" "has to be a boolean" "$stderr" + + +## TEST12: test ViashRenderJsonUnquotedValue with valid and invalid numeric values + +# TEST12a: Valid integer +output=$(ViashRenderJsonUnquotedValue "count" "42") +assert_value_equal "test12a_valid_int" "42" "$output" + +# TEST12b: Valid negative integer +output=$(ViashRenderJsonUnquotedValue "temp" "-273") +assert_value_equal "test12b_negative_int" "-273" "$output" + +# TEST12c: Valid decimal +output=$(ViashRenderJsonUnquotedValue "ratio" "3.14159") +assert_value_equal "test12c_decimal" "3.14159" "$output" + +# TEST12d: Valid negative decimal +output=$(ViashRenderJsonUnquotedValue "temp" "-0.5") +assert_value_equal "test12d_negative_decimal" "-0.5" "$output" + +# TEST12e: Valid scientific notation +output=$(ViashRenderJsonUnquotedValue "big" "1.5e10") +assert_value_equal "test12e_scientific" "1.5e10" "$output" + +# TEST12f: Valid negative scientific notation +output=$(ViashRenderJsonUnquotedValue "small" "-2.5E-3") +assert_value_equal "test12f_neg_scientific" "-2.5E-3" "$output" + +# TEST12g: Invalid - text instead of number +set +e +stderr=$(ViashRenderJsonUnquotedValue "count" "not-a-number" 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test12g_exit_code" 1 "$exit_code" +assert_contains "test12g_stderr" "Argument 'count' has to be a number" "$stderr" + +# TEST12h: Invalid - text with numbers +set +e +stderr=$(ViashRenderJsonUnquotedValue "value" "12abc" 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test12h_exit_code" 1 "$exit_code" +assert_contains "test12h_stderr" "has to be a number" "$stderr" + +# TEST12i: Invalid - multiple decimal points +set +e +stderr=$(ViashRenderJsonUnquotedValue "value" "1.2.3" 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test12i_exit_code" 1 "$exit_code" +assert_contains "test12i_stderr" "has to be a number" "$stderr" + +# TEST12j: Invalid - empty string +set +e +stderr=$(ViashRenderJsonUnquotedValue "value" "" 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test12j_exit_code" 1 "$exit_code" +assert_contains "test12j_stderr" "has to be a number" "$stderr" + +# TEST12k: Invalid - just a decimal point +set +e +stderr=$(ViashRenderJsonUnquotedValue "value" "." 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test12k_exit_code" 1 "$exit_code" +assert_contains "test12k_stderr" "has to be a number" "$stderr" + +# TEST12l: Invalid - multiple minus signs +set +e +stderr=$(ViashRenderJsonUnquotedValue "value" "--5" 2>&1 >/dev/null) +exit_code=$? +set -e +assert_exit_code "test12l_exit_code" 1 "$exit_code" +assert_contains "test12l_stderr" "has to be a number" "$stderr" + +print_test_summary diff --git a/src/test/resources/io/viash/helpers/bashutils/ViashSourceDir.test.sh b/src/test/resources/io/viash/helpers/bashutils/ViashSourceDir.test.sh new file mode 100644 index 000000000..8f8d7d5cf --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/ViashSourceDir.test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# load helper functions +source src/main/resources/io/viash/helpers/bashutils/ViashSourceDir.sh +source src/main/resources/io/viash/helpers/bashutils/ViashFindTargetDir.sh +source src/test/resources/io/viash/helpers/bashutils/helpers.sh + +# Create a temporary directory structure for testing +# Use cd -P && pwd to resolve symlinks (e.g. /var -> /private/var on macOS) +TEST_DIR=$(mktemp -d) +TEST_DIR=$(cd -P "$TEST_DIR" && pwd) +cleanup() { + rm -rf "$TEST_DIR" +} +trap cleanup EXIT + + +## TEST1: test ViashSourceDir + +# TEST1a: Get directory of this script +output=$(ViashSourceDir "${BASH_SOURCE[0]}") +expected=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +assert_value_equal "test1a_output" "$expected" "$output" + + +## TEST2: test ViashSourceDir with symlinks + +# TEST2a: Create a symlink to a script and resolve it +mkdir -p "$TEST_DIR/real_dir" +echo '#!/bin/bash' > "$TEST_DIR/real_dir/script.sh" +chmod +x "$TEST_DIR/real_dir/script.sh" + +mkdir -p "$TEST_DIR/link_dir" +ln -s "$TEST_DIR/real_dir/script.sh" "$TEST_DIR/link_dir/script_link.sh" + +# Source the function and test it +output=$(ViashSourceDir "$TEST_DIR/link_dir/script_link.sh") +assert_value_equal "test2a_output" "$TEST_DIR/real_dir" "$output" + + +## TEST3: test ViashFindTargetDir + +# TEST3a: Create a directory structure with .build.yaml +mkdir -p "$TEST_DIR/project/target/namespace/component" +touch "$TEST_DIR/project/target/.build.yaml" + +output=$(ViashFindTargetDir "$TEST_DIR/project/target/namespace/component") +assert_value_equal "test3a_output" "$TEST_DIR/project/target" "$output" + +# TEST3b: Find .build.yaml from direct parent +output=$(ViashFindTargetDir "$TEST_DIR/project/target/namespace") +assert_value_equal "test3b_output" "$TEST_DIR/project/target" "$output" + +# TEST3c: Find .build.yaml from the target dir itself +output=$(ViashFindTargetDir "$TEST_DIR/project/target") +assert_value_equal "test3c_output" "$TEST_DIR/project/target" "$output" + + +## TEST4: test ViashFindTargetDir when no .build.yaml exists + +# TEST4a: No .build.yaml found +mkdir -p "$TEST_DIR/no_build/subdir" +output=$(ViashFindTargetDir "$TEST_DIR/no_build/subdir") +assert_value_equal "test4a_output" "" "$output" + + +## TEST5: test ViashFindTargetDir with multiple .build.yaml files + +# TEST5a: Should find the closest (lowest) .build.yaml +mkdir -p "$TEST_DIR/multi/a/b/c" +touch "$TEST_DIR/multi/.build.yaml" +touch "$TEST_DIR/multi/a/b/.build.yaml" + +output=$(ViashFindTargetDir "$TEST_DIR/multi/a/b/c") +assert_value_equal "test5a_output" "$TEST_DIR/multi/a/b" "$output" + +# TEST5b: Starting from a location with .build.yaml +output=$(ViashFindTargetDir "$TEST_DIR/multi/a/b") +assert_value_equal "test5b_output" "$TEST_DIR/multi/a/b" "$output" + +# TEST5c: Going above the nested .build.yaml should find the parent one +output=$(ViashFindTargetDir "$TEST_DIR/multi/a") +assert_value_equal "test5c_output" "$TEST_DIR/multi" "$output" + +print_test_summary diff --git a/src/test/resources/io/viash/helpers/bashutils/helpers.sh b/src/test/resources/io/viash/helpers/bashutils/helpers.sh new file mode 100644 index 000000000..4829cbe49 --- /dev/null +++ b/src/test/resources/io/viash/helpers/bashutils/helpers.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# Test counter for tracking test results +TESTS_PASSED=0 +TESTS_FAILED=0 + +# assert_value_equal: Assert that a value equals the expected value +# $1: name of the test +# $2: expected value +# $3+: actual value(s) +assert_value_equal() { + local name="$1" + local expected="$2" + shift 2 + local values="$*" + if [ "$expected" != "$values" ]; then + echo "FAILED: $name" + echo " Expected: '$expected'" + echo " Got: '$values'" + TESTS_FAILED=$((TESTS_FAILED + 1)) + return 1 + else + TESTS_PASSED=$((TESTS_PASSED + 1)) + return 0 + fi +} + +# assert_array_equal: Assert that two arrays are equal +# $1: name of the test +# $2: name of expected array variable +# $3: name of actual array variable +assert_array_equal() { + local name="$1" + local expected_name="$2" + local actual_name="$3" + + # Get array values using eval for bash 3.2 compatibility + eval "local expected_values=(\"\${${expected_name}[@]}\")" + eval "local actual_values=(\"\${${actual_name}[@]}\")" + + # Check lengths + if [ ${#expected_values[@]} -ne ${#actual_values[@]} ]; then + echo "FAILED: $name (array length mismatch)" + echo " Expected length: ${#expected_values[@]}" + echo " Got length: ${#actual_values[@]}" + TESTS_FAILED=$((TESTS_FAILED + 1)) + return 1 + fi + + # Check each element + for i in "${!expected_values[@]}"; do + if [ "${expected_values[$i]}" != "${actual_values[$i]}" ]; then + echo "FAILED: $name (element $i mismatch)" + echo " Expected[$i]: '${expected_values[$i]}'" + echo " Got[$i]: '${actual_values[$i]}'" + TESTS_FAILED=$((TESTS_FAILED + 1)) + return 1 + fi + done + + TESTS_PASSED=$((TESTS_PASSED + 1)) + return 0 +} + +# assert_contains: Assert that a string contains a substring +# $1: name of the test +# $2: substring to find +# $3: string to search in +assert_contains() { + local name="$1" + local substring="$2" + local string="$3" + if [[ "$string" != *"$substring"* ]]; then + echo "FAILED: $name" + echo " Expected to contain: '$substring'" + echo " In string: '$string'" + TESTS_FAILED=$((TESTS_FAILED + 1)) + return 1 + else + TESTS_PASSED=$((TESTS_PASSED + 1)) + return 0 + fi +} + +# assert_not_contains: Assert that a string does not contain a substring +# $1: name of the test +# $2: substring that should not be found +# $3: string to search in +assert_not_contains() { + local name="$1" + local substring="$2" + local string="$3" + if [[ "$string" == *"$substring"* ]]; then + echo "FAILED: $name" + echo " Expected NOT to contain: '$substring'" + echo " In string: '$string'" + TESTS_FAILED=$((TESTS_FAILED + 1)) + return 1 + else + TESTS_PASSED=$((TESTS_PASSED + 1)) + return 0 + fi +} + +# assert_exit_code: Assert that the last command had a specific exit code +# $1: name of the test +# $2: expected exit code +# $3: actual exit code +assert_exit_code() { + local name="$1" + local expected="$2" + local actual="$3" + if [ "$expected" -ne "$actual" ]; then + echo "FAILED: $name" + echo " Expected exit code: $expected" + echo " Got exit code: $actual" + TESTS_FAILED=$((TESTS_FAILED + 1)) + return 1 + else + TESTS_PASSED=$((TESTS_PASSED + 1)) + return 0 + fi +} + +# print_test_summary: Print a summary of test results and exit with proper code +print_test_summary() { + local total=$((TESTS_PASSED + TESTS_FAILED)) + echo "" + echo "==================================" + echo "Test Summary: $TESTS_PASSED/$total passed" + if [ $TESTS_FAILED -gt 0 ]; then + echo "FAILURES: $TESTS_FAILED" + exit 1 + else + echo "All tests passed!" + exit 0 + fi +} diff --git a/src/test/resources/io/viash/helpers/languages/bash/test_ViashParseJson.sh b/src/test/resources/io/viash/helpers/languages/bash/test_ViashParseJson.sh new file mode 100644 index 000000000..d94f20528 --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/bash/test_ViashParseJson.sh @@ -0,0 +1,339 @@ +#!/usr/bin/env bash + +# Test suite for viash_parse_json Bash function +echo "Running viash_parse_json Bash unit tests..." + +# Load the parser +source src/main/resources/io/viash/languages/bash/ViashParseJson.sh + +# Colors for test output +RED='\033[0;31m' +GREEN='\033[0;32m' +RESET='\033[0m' + +test_passed=0 +test_failed=0 + +# Test helper function +test_equal() { + local description="$1" + local actual="$2" + local expected="$3" + + if [ "$actual" = "$expected" ]; then + echo -e "${GREEN}PASS${RESET}: $description" + ((test_passed++)) + return 0 + else + echo -e "${RED}FAIL${RESET}: $description" + echo " Expected: $expected" + echo " Got: $actual" + ((test_failed++)) + return 1 + fi +} + +# Create test JSON file +test_json=$(mktemp) +cat >"$test_json" <<'EOF' +{ + "par": { + "input": "file.txt", + "number": 42, + "flag": true, + "empty_value": "", + "array_simple": ["a", "b", "c"], + "array_numbers": [1, 2, 3], + "array_mixed": ["text", 123, true, null], + "nested": { + "level1": { + "nasty_val": "{nasty}", + "level2": "deep_value" + } + }, + "path_with_spaces": "/path/with spaces/file.txt", + "quotes": "She said \"hello\"", + "newlines": "line1\\nline2", + "tabs": "col1\\tcol2", + "string": "text", + "integer": 123, + "float": 3.14, + "bool_true": true, + "bool_false": false, + "null_value": null, + "empty_string": "", + "zero": 0, + "empty_array": [], + "empty_object": {} + }, + "meta": { + "name": "test_component", + "version": "1.0" + }, + "simple_key": "value", + "number_key": 99, + "bool_key": false +} +EOF + +# Parse the JSON +ViashParseJsonBash <"$test_json" + +echo "=== Test 1: Basic key-value pairs ===" +test_equal "par_input" "$par_input" "file.txt" +test_equal "par_number" "$par_number" "42" +test_equal "par_flag" "$par_flag" "true" +test_equal "par_empty_value" "$par_empty_value" "" +test_equal "meta_name" "$meta_name" "test_component" +test_equal "meta_version" "$meta_version" "1.0" + +echo +echo "=== Test 2: Arrays ===" +test_equal "par_array_simple[0]" "${par_array_simple[0]}" "a" +test_equal "par_array_simple[1]" "${par_array_simple[1]}" "b" +test_equal "par_array_simple[2]" "${par_array_simple[2]}" "c" +test_equal "par_array_numbers[0]" "${par_array_numbers[0]}" "1" +test_equal "par_array_numbers[1]" "${par_array_numbers[1]}" "2" +test_equal "par_array_numbers[2]" "${par_array_numbers[2]}" "3" + +echo +echo "=== Test 3: Nested objects (stored as JSON strings) ===" +# Nested objects should be stored as JSON strings +test_equal "par_nested exists" "${par_nested:+set}" "set" +# Simple check that it looks like JSON +if [[ "$par_nested" =~ "level1" ]] && [[ "$par_nested" =~ "level2" ]]; then + echo -e "${GREEN}PASS${RESET}: par_nested contains expected nested structure" + ((test_passed++)) +else + echo -e "${RED}FAIL${RESET}: par_nested contains expected nested structure" + echo " Got: $par_nested" + ((test_failed++)) +fi + +echo +echo "=== Test 4: Quoted strings ===" +test_equal "par_path_with_spaces" "$par_path_with_spaces" "/path/with spaces/file.txt" +test_equal "par_quotes" "$par_quotes" "She said \"hello\"" +# Note: JSON \\n (literal backslash-n) is correctly preserved by jq +# This matches Viash YAML parser behavior where \n in configs is a literal backslash-n +test_equal "par_newlines" "$par_newlines" 'line1\nline2' +test_equal "par_tabs" "$par_tabs" 'col1\tcol2' + +echo +echo "=== Test 5: Type conversions ===" +test_equal "par_string" "$par_string" "text" +test_equal "par_integer" "$par_integer" "123" +test_equal "par_float" "$par_float" "3.14" +test_equal "par_bool_true" "$par_bool_true" "true" +test_equal "par_bool_false" "$par_bool_false" "false" +# null values should leave the variable unset +test_equal "par_null_value (unset)" "${par_null_value-UNSET}" "UNSET" + +echo +echo "=== Test 6: Root-level values ===" +test_equal "simple_key" "$simple_key" "value" +test_equal "number_key" "$number_key" "99" +test_equal "bool_key" "$bool_key" "false" + +echo +echo "=== Test 7: Edge cases ===" +test_equal "par_empty_string" "$par_empty_string" "" +test_equal "par_zero" "$par_zero" "0" +test_equal "par_empty_array length" "${#par_empty_array[@]}" "0" + +# Clean up +rm -f "$test_json" + +## =================================================================== +## TEST 8: Compact / minified JSON (no whitespace) +## =================================================================== +echo +echo "=== Test 8: Compact JSON (minified) ===" + +ViashParseJsonBash <<< '{"par":{"x":"hello","y":42,"z":true,"arr":["a","b"],"nested":{"k":"v"}},"meta":{"name":"comp"}}' + +test_equal "compact: par_x" "$par_x" "hello" +test_equal "compact: par_y" "$par_y" "42" +test_equal "compact: par_z" "$par_z" "true" +test_equal "compact: par_arr[0]" "${par_arr[0]}" "a" +test_equal "compact: par_arr[1]" "${par_arr[1]}" "b" +test_equal "compact: par_nested contains k" "$(echo "$par_nested" | grep -c '"k"')" "1" +test_equal "compact: meta_name" "$meta_name" "comp" + +## =================================================================== +## TEST 9: Special characters in strings +## =================================================================== +echo +echo "=== Test 9: Special characters ===" + +test_special=$(mktemp) +cat >"$test_special" <<'EOF' +{ + "par": { + "backtick": "run `id`", + "dollar": "path is $PATH", + "subst": "time $(date)", + "backslash": "C:\\Users\\test", + "quotes_in_str": "She said \"hi\"", + "slash": "a/b/c", + "mixed": "a\\b\"c$d`e" + } +} +EOF +ViashParseJsonBash <"$test_special" +rm -f "$test_special" + +test_equal "special: backtick" "$par_backtick" 'run `id`' +test_equal "special: dollar" "$par_dollar" 'path is $PATH' +test_equal "special: subst" "$par_subst" 'time $(date)' +test_equal "special: backslash" "$par_backslash" 'C:\Users\test' +test_equal "special: quotes" "$par_quotes_in_str" 'She said "hi"' +test_equal "special: slash" "$par_slash" "a/b/c" +test_equal "special: mixed" "$par_mixed" 'a\b"c$d`e' + +## =================================================================== +## TEST 10: Negative numbers, scientific notation +## =================================================================== +echo +echo "=== Test 10: Numeric edge cases ===" + +ViashParseJsonBash <<< '{"par":{"neg":-7,"sci":1.5e10,"neg_sci":-2.5E-3,"big":999999999}}' + +test_equal "numeric: neg" "$par_neg" "-7" +test_equal "numeric: sci" "$par_sci" "1.5E+10" +test_equal "numeric: neg_sci" "$par_neg_sci" "-0.0025" +test_equal "numeric: big" "$par_big" "999999999" + +## =================================================================== +## TEST 11: Mixed-type arrays +## =================================================================== +echo +echo "=== Test 11: Mixed-type arrays ===" + +ViashParseJsonBash <<< '{"par":{"mix":["text",123,true,false,null,-5]}}' + +test_equal "mixed array[0]" "${par_mix[0]}" "text" +test_equal "mixed array[1]" "${par_mix[1]}" "123" +test_equal "mixed array[2]" "${par_mix[2]}" "true" +test_equal "mixed array[3]" "${par_mix[3]}" "false" +test_equal "mixed array[4]" "${par_mix[4]}" "null" +test_equal "mixed array[5]" "${par_mix[5]}" "-5" +test_equal "mixed array length" "${#par_mix[@]}" "6" + +## =================================================================== +## TEST 12: Empty and minimal JSON +## =================================================================== +echo +echo "=== Test 12: Empty/minimal JSON ===" + +ViashParseJsonBash <<< '{}' +# Should not crash - that's the test + +ViashParseJsonBash <<< '{"par":{}}' +# Should also not crash + +ViashParseJsonBash <<< '{"par":{"only":true}}' +test_equal "minimal: par_only" "$par_only" "true" + +## =================================================================== +## TEST 13: Deeply nested objects stored as JSON +## =================================================================== +echo +echo "=== Test 13: Deeply nested objects ===" + +test_deep=$(mktemp) +cat >"$test_deep" <<'EOF' +{ + "par": { + "config": { + "db": { + "host": "localhost", + "port": 5432 + }, + "cache": true + } + } +} +EOF +ViashParseJsonBash <"$test_deep" +rm -f "$test_deep" + +test_equal "deep: par_config exists" "${par_config:+set}" "set" +# Should contain the nested JSON including db and cache +if [[ "$par_config" =~ "host" ]] && [[ "$par_config" =~ "localhost" ]] && [[ "$par_config" =~ "cache" ]]; then + echo -e "${GREEN}PASS${RESET}: deep: par_config contains expected structure" + ((test_passed++)) +else + echo -e "${RED}FAIL${RESET}: deep: par_config contains expected structure" + echo " Got: $par_config" + ((test_failed++)) +fi + +## =================================================================== +## TEST 14: Error handling - invalid JSON should fail +## =================================================================== +echo +echo "=== Test 14: Error handling ===" + +# Invalid JSON: missing closing brace +set +e +stderr=$(ViashParseJsonBash <<< '{"par":{"x": "hello"' 2>&1 >/dev/null) +exit_code=$? +set -e +if [ $exit_code -ne 0 ]; then + echo -e "${GREEN}PASS${RESET}: invalid JSON exits with error" + ((test_passed++)) +else + echo -e "${RED}FAIL${RESET}: invalid JSON exits with error (got exit code 0)" + ((test_failed++)) +fi + +# Invalid JSON: trailing garbage +set +e +stderr=$(ViashParseJsonBash <<< '{"x": }' 2>&1 >/dev/null) +exit_code=$? +set -e +if [ $exit_code -ne 0 ]; then + echo -e "${GREEN}PASS${RESET}: malformed value exits with error" + ((test_passed++)) +else + echo -e "${RED}FAIL${RESET}: malformed value exits with error (got exit code 0)" + ((test_failed++)) +fi + +## =================================================================== +## TEST 15: Strings with commas, colons, braces (tricky for line parsers) +## =================================================================== +echo +echo "=== Test 15: Tricky string content ===" + +test_tricky=$(mktemp) +cat >"$test_tricky" <<'EOF' +{ + "par": { + "with_comma": "a, b, c", + "with_colon": "key: value", + "with_braces": "obj = {x: 1}", + "with_brackets": "arr = [1, 2]" + } +} +EOF +ViashParseJsonBash <"$test_tricky" +rm -f "$test_tricky" + +test_equal "tricky: comma" "$par_with_comma" "a, b, c" +test_equal "tricky: colon" "$par_with_colon" "key: value" +test_equal "tricky: braces" "$par_with_braces" "obj = {x: 1}" +test_equal "tricky: brackets" "$par_with_brackets" "arr = [1, 2]" + +# Print summary +echo +echo "==================================================" +echo "Tests completed: $((test_passed + test_failed))" +echo "Tests passed: $test_passed" +if [ $test_failed -gt 0 ]; then + echo -e "${RED}Tests failed: $test_failed${RESET}" + exit 1 +else + echo -e "${GREEN}All tests passed!${RESET}" +fi diff --git a/src/test/resources/io/viash/helpers/languages/bash/test_ViashParseJsonCompatibility.sh b/src/test/resources/io/viash/helpers/languages/bash/test_ViashParseJsonCompatibility.sh new file mode 100755 index 000000000..fbab4dfdc --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/bash/test_ViashParseJsonCompatibility.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash + +# Test suite for viash_parse_json Bash function (compatibility parser) +echo "Running viash_parse_json Bash compatibility unit tests..." + +# Load the parser +source src/main/resources/io/viash/languages/bash/ViashParseJsonCompatibility.sh + +# Colors for test output +RED='\033[0;31m' +GREEN='\033[0;32m' +RESET='\033[0m' + +test_passed=0 +test_failed=0 + +# Test helper function +test_equal() { + local description="$1" + local actual="$2" + local expected="$3" + + if [ "$actual" = "$expected" ]; then + echo -e "${GREEN}PASS${RESET}: $description" + ((test_passed++)) + return 0 + else + echo -e "${RED}FAIL${RESET}: $description" + echo " Expected: $expected" + echo " Got: $actual" + ((test_failed++)) + return 1 + fi +} + +# Create test JSON file +test_json=$(mktemp) +cat >"$test_json" <<'EOF' +{ + "par": { + "input": "file.txt", + "number": 42, + "flag": true, + "empty_value": "", + "array_simple": ["a", "b", "c"], + "array_numbers": [1, 2, 3], + "array_mixed": ["text", 123, true, null], + "nested": { + "level1": { + "nasty_val": "{nasty}", + "level2": "deep_value" + } + }, + "path_with_spaces": "/path/with spaces/file.txt", + "quotes": "She said \"hello\"", + "newlines": "line1\nline2", + "tabs": "col1\tcol2", + "string": "text", + "integer": 123, + "float": 3.14, + "bool_true": true, + "bool_false": false, + "null_value": null, + "empty_string": "", + "zero": 0, + "empty_array": [], + "empty_object": {} + }, + "meta": { + "name": "test_component", + "version": "1.0" + }, + "simple_key": "value", + "number_key": 99, + "bool_key": false +} +EOF + +# Parse the JSON +ViashParseJsonBash <"$test_json" + +echo "=== Test 1: Basic key-value pairs ===" +test_equal "par_input" "$par_input" "file.txt" +test_equal "par_number" "$par_number" "42" +test_equal "par_flag" "$par_flag" "true" +test_equal "par_empty_value" "$par_empty_value" "" +test_equal "meta_name" "$meta_name" "test_component" +test_equal "meta_version" "$meta_version" "1.0" + +echo +echo "=== Test 2: Arrays ===" +test_equal "par_array_simple[0]" "${par_array_simple[0]}" "a" +test_equal "par_array_simple[1]" "${par_array_simple[1]}" "b" +test_equal "par_array_simple[2]" "${par_array_simple[2]}" "c" +test_equal "par_array_numbers[0]" "${par_array_numbers[0]}" "1" +test_equal "par_array_numbers[1]" "${par_array_numbers[1]}" "2" +test_equal "par_array_numbers[2]" "${par_array_numbers[2]}" "3" + +echo +echo "=== Test 3: Nested objects (stored as JSON strings) ===" +# Nested objects should be stored as JSON strings +test_equal "par_nested exists" "${par_nested:+set}" "set" +# Simple check that it looks like JSON +if [[ "$par_nested" =~ "level1" ]] && [[ "$par_nested" =~ "level2" ]]; then + echo -e "${GREEN}PASS${RESET}: par_nested contains expected nested structure" + ((test_passed++)) +else + echo -e "${RED}FAIL${RESET}: par_nested contains expected nested structure" + echo " Got: $par_nested" + ((test_failed++)) +fi + +echo +echo "=== Test 4: Quoted strings ===" +test_equal "par_path_with_spaces" "$par_path_with_spaces" "/path/with spaces/file.txt" +test_equal "par_quotes" "$par_quotes" "She said \"hello\"" +# Note: In bash, \n and \t are literal strings, not escape sequences +test_equal "par_newlines" "$par_newlines" "line1\nline2" +test_equal "par_tabs" "$par_tabs" "col1\tcol2" + +echo +echo "=== Test 5: Type conversions ===" +test_equal "par_string" "$par_string" "text" +test_equal "par_integer" "$par_integer" "123" +test_equal "par_float" "$par_float" "3.14" +test_equal "par_bool_true" "$par_bool_true" "true" +test_equal "par_bool_false" "$par_bool_false" "false" +# null values should leave the variable unset +test_equal "par_null_value (unset)" "${par_null_value-UNSET}" "UNSET" + +echo +echo "=== Test 6: Root-level values ===" +test_equal "simple_key" "$simple_key" "value" +test_equal "number_key" "$number_key" "99" +test_equal "bool_key" "$bool_key" "false" + +echo +echo "=== Test 7: Edge cases ===" +test_equal "par_empty_string" "$par_empty_string" "" +test_equal "par_zero" "$par_zero" "0" +test_equal "par_empty_array length" "${#par_empty_array[@]}" "0" + +# Clean up +rm -f "$test_json" + +## =================================================================== +## TEST 8: Compact / minified JSON (no whitespace) +## =================================================================== +echo +echo "=== Test 8: Compact JSON (minified) ===" + +ViashParseJsonBash <<< '{"par":{"x":"hello","y":42,"z":true,"arr":["a","b"],"nested":{"k":"v"}},"meta":{"name":"comp"}}' + +test_equal "compact: par_x" "$par_x" "hello" +test_equal "compact: par_y" "$par_y" "42" +test_equal "compact: par_z" "$par_z" "true" +test_equal "compact: par_arr[0]" "${par_arr[0]}" "a" +test_equal "compact: par_arr[1]" "${par_arr[1]}" "b" +test_equal "compact: par_nested contains k" "$(echo "$par_nested" | grep -c '"k"')" "1" +test_equal "compact: meta_name" "$meta_name" "comp" + +## =================================================================== +## TEST 9: Special characters in strings +## =================================================================== +echo +echo "=== Test 9: Special characters ===" + +test_special=$(mktemp) +cat >"$test_special" <<'EOF' +{ + "par": { + "backtick": "run `id`", + "dollar": "path is $PATH", + "subst": "time $(date)", + "backslash": "C:\\Users\\test", + "quotes_in_str": "She said \"hi\"", + "slash": "a/b/c", + "mixed": "a\\b\"c$d`e" + } +} +EOF +ViashParseJsonBash <"$test_special" +rm -f "$test_special" + +test_equal "special: backtick" "$par_backtick" 'run `id`' +test_equal "special: dollar" "$par_dollar" 'path is $PATH' +test_equal "special: subst" "$par_subst" 'time $(date)' +test_equal "special: backslash" "$par_backslash" 'C:\Users\test' +test_equal "special: quotes" "$par_quotes_in_str" 'She said "hi"' +test_equal "special: slash" "$par_slash" "a/b/c" +test_equal "special: mixed" "$par_mixed" 'a\b"c$d`e' + +## =================================================================== +## TEST 10: Negative numbers, scientific notation +## =================================================================== +echo +echo "=== Test 10: Numeric edge cases ===" + +ViashParseJsonBash <<< '{"par":{"neg":-7,"sci":1.5e10,"neg_sci":-2.5E-3,"big":999999999}}' + +test_equal "numeric: neg" "$par_neg" "-7" +test_equal "numeric: sci" "$par_sci" "1.5e10" +test_equal "numeric: neg_sci" "$par_neg_sci" "-2.5E-3" +test_equal "numeric: big" "$par_big" "999999999" + +## =================================================================== +## TEST 11: Mixed-type arrays +## =================================================================== +echo +echo "=== Test 11: Mixed-type arrays ===" + +ViashParseJsonBash <<< '{"par":{"mix":["text",123,true,false,null,-5]}}' + +test_equal "mixed array[0]" "${par_mix[0]}" "text" +test_equal "mixed array[1]" "${par_mix[1]}" "123" +test_equal "mixed array[2]" "${par_mix[2]}" "true" +test_equal "mixed array[3]" "${par_mix[3]}" "false" +test_equal "mixed array[4]" "${par_mix[4]}" "null" +test_equal "mixed array[5]" "${par_mix[5]}" "-5" +test_equal "mixed array length" "${#par_mix[@]}" "6" + +## =================================================================== +## TEST 12: Empty and minimal JSON +## =================================================================== +echo +echo "=== Test 12: Empty/minimal JSON ===" + +ViashParseJsonBash <<< '{}' +# Should not crash - that's the test + +ViashParseJsonBash <<< '{"par":{}}' +# Should also not crash + +ViashParseJsonBash <<< '{"par":{"only":true}}' +test_equal "minimal: par_only" "$par_only" "true" + +## =================================================================== +## TEST 13: Deeply nested objects stored as JSON +## =================================================================== +echo +echo "=== Test 13: Deeply nested objects ===" + +test_deep=$(mktemp) +cat >"$test_deep" <<'EOF' +{ + "par": { + "config": { + "db": { + "host": "localhost", + "port": 5432 + }, + "cache": true + } + } +} +EOF +ViashParseJsonBash <"$test_deep" +rm -f "$test_deep" + +test_equal "deep: par_config exists" "${par_config:+set}" "set" +# Should contain the nested JSON including db and cache +if [[ "$par_config" =~ "host" ]] && [[ "$par_config" =~ "localhost" ]] && [[ "$par_config" =~ "cache" ]]; then + echo -e "${GREEN}PASS${RESET}: deep: par_config contains expected structure" + ((test_passed++)) +else + echo -e "${RED}FAIL${RESET}: deep: par_config contains expected structure" + echo " Got: $par_config" + ((test_failed++)) +fi + +## =================================================================== +## TEST 14: Error handling - invalid JSON should fail +## =================================================================== +echo +echo "=== Test 14: Error handling ===" + +# Invalid JSON: missing closing brace +set +e +stderr=$(ViashParseJsonBash <<< '{"par":{"x": "hello"' 2>&1 >/dev/null) +exit_code=$? +set -e +if [ $exit_code -ne 0 ]; then + echo -e "${GREEN}PASS${RESET}: invalid JSON exits with error" + ((test_passed++)) +else + echo -e "${RED}FAIL${RESET}: invalid JSON exits with error (got exit code 0)" + ((test_failed++)) +fi + +# Invalid JSON: trailing garbage +set +e +stderr=$(ViashParseJsonBash <<< '{"x": }' 2>&1 >/dev/null) +exit_code=$? +set -e +if [ $exit_code -ne 0 ]; then + echo -e "${GREEN}PASS${RESET}: malformed value exits with error" + ((test_passed++)) +else + echo -e "${RED}FAIL${RESET}: malformed value exits with error (got exit code 0)" + ((test_failed++)) +fi + +## =================================================================== +## TEST 15: Strings with commas, colons, braces (tricky for line parsers) +## =================================================================== +echo +echo "=== Test 15: Tricky string content ===" + +test_tricky=$(mktemp) +cat >"$test_tricky" <<'EOF' +{ + "par": { + "with_comma": "a, b, c", + "with_colon": "key: value", + "with_braces": "obj = {x: 1}", + "with_brackets": "arr = [1, 2]" + } +} +EOF +ViashParseJsonBash <"$test_tricky" +rm -f "$test_tricky" + +test_equal "tricky: comma" "$par_with_comma" "a, b, c" +test_equal "tricky: colon" "$par_with_colon" "key: value" +test_equal "tricky: braces" "$par_with_braces" "obj = {x: 1}" +test_equal "tricky: brackets" "$par_with_brackets" "arr = [1, 2]" + +# Print summary +echo +echo "==================================================" +echo "Tests completed: $((test_passed + test_failed))" +echo "Tests passed: $test_passed" +if [ $test_failed -gt 0 ]; then + echo -e "${RED}Tests failed: $test_failed${RESET}" + exit 1 +else + echo -e "${GREEN}All tests passed!${RESET}" +fi diff --git a/src/test/resources/io/viash/helpers/languages/csharp/test_ViashParseJson.csx b/src/test/resources/io/viash/helpers/languages/csharp/test_ViashParseJson.csx new file mode 100644 index 000000000..e85838a82 --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/csharp/test_ViashParseJson.csx @@ -0,0 +1,225 @@ +#!/usr/bin/env dotnet script + +/* + * Unit test for ViashJsonParser.ParseJson function + * This test verifies that the C# JSON parser correctly parses JSON content + * and returns appropriate C# data structures. + */ + +#load "../../../../../../../main/resources/io/viash/languages/csharp/ViashParseJson.csx" + +using System; +using System.Collections.Generic; + +// ANSI color codes +const string RED = "\x1b[31m"; +const string GREEN = "\x1b[32m"; +const string RESET = "\x1b[0m"; + +int testsPassed = 0; +int testsFailed = 0; + +void TestEqual(string testName, object actual, object expected) +{ + bool passed = false; + + if (actual == null && expected == null) + { + passed = true; + } + else if (actual != null && expected != null) + { + if (actual is List actualList && expected is List expectedList) + { + passed = actualList.Count == expectedList.Count; + if (passed) + { + for (int i = 0; i < actualList.Count; i++) + { + if (!actualList[i].Equals(expectedList[i])) + { + passed = false; + break; + } + } + } + } + else + { + passed = actual.Equals(expected) || actual.ToString() == expected.ToString(); + } + } + + if (passed) + { + Console.WriteLine($"PASS: {testName}"); + testsPassed++; + } + else + { + Console.WriteLine($"{RED}FAIL: {testName}{RESET}"); + Console.WriteLine($" Expected: {expected}"); + Console.WriteLine($" Got: {actual}"); + testsFailed++; + } +} + +void TestTrue(string testName, bool condition) +{ + if (condition) + { + Console.WriteLine($"PASS: {testName}"); + testsPassed++; + } + else + { + Console.WriteLine($"{RED}FAIL: {testName}{RESET}"); + testsFailed++; + } +} + +Console.WriteLine("Running viash_parse_json C# unit tests..."); +Console.WriteLine(); + +// Test JSON content +string jsonContent = @"{ + ""par"": { + ""input"": ""file.txt"", + ""number"": 42, + ""flag"": true, + ""empty_value"": """", + ""array_simple"": [""a"", ""b"", ""c""], + ""array_numbers"": [1, 2, 3], + ""array_mixed"": [""text"", 123, true], + ""nested"": { + ""level1"": { + ""level2"": ""deep_value"" + } + }, + ""path_with_spaces"": ""/path/with spaces/file.txt"", + ""quotes"": ""He said \""hello\"""", + ""newlines"": ""line1\nline2"", + ""tabs"": ""col1\tcol2"", + ""string"": ""text"", + ""integer"": 42, + ""float"": 3.14, + ""bool_true"": true, + ""bool_false"": false, + ""null_value"": null, + ""empty_string"": """", + ""zero"": 0, + ""empty_array"": [], + ""empty_object"": {} + }, + ""meta"": { + ""name"": ""test_component"", + ""version"": ""1.0.0"" + }, + ""simple_key"": ""simple_value"", + ""number_key"": 123, + ""bool_key"": true +}"; + +// Parse the JSON +var tempFile = Path.GetTempFileName(); +File.WriteAllText(tempFile, jsonContent); + +try +{ + var parsed = ViashJsonParser.ParseJson(tempFile); + var par = parsed["par"] as Dictionary; + var meta = parsed["meta"] as Dictionary; + +// Test 1: Basic key-value pairs +Console.WriteLine("=== Test 1: Basic key-value pairs ==="); +TestEqual("par.input", par["input"], "file.txt"); +TestEqual("par.number", par["number"], 42); +TestEqual("par.flag", par["flag"], true); +TestEqual("par.empty_value", par["empty_value"], ""); +TestEqual("meta.name", meta["name"], "test_component"); +TestEqual("meta.version", meta["version"], "1.0.0"); +Console.WriteLine(); + +// Test 2: Arrays +Console.WriteLine("=== Test 2: Arrays ==="); +var arraySimple = par["array_simple"] as List; +TestTrue("par.array_simple length", arraySimple.Count == 3); +TestEqual("par.array_simple[0]", arraySimple[0], "a"); +TestEqual("par.array_simple[1]", arraySimple[1], "b"); +TestEqual("par.array_simple[2]", arraySimple[2], "c"); + +var arrayNumbers = par["array_numbers"] as List; +TestTrue("par.array_numbers length", arrayNumbers.Count == 3); +TestEqual("par.array_numbers[0]", arrayNumbers[0], 1); +TestEqual("par.array_numbers[1]", arrayNumbers[1], 2); +TestEqual("par.array_numbers[2]", arrayNumbers[2], 3); + +var arrayMixed = par["array_mixed"] as List; +TestTrue("par.array_mixed length", arrayMixed.Count == 3); +Console.WriteLine(); + +// Test 3: Nested structures +Console.WriteLine("=== Test 3: Nested structures ==="); +var nested = par["nested"] as Dictionary; +var level1 = nested["level1"] as Dictionary; +TestEqual("par.nested.level1.level2", level1["level2"], "deep_value"); +Console.WriteLine(); + +// Test 4: Quoted strings +Console.WriteLine("=== Test 4: Quoted strings ==="); +TestEqual("par.path_with_spaces", par["path_with_spaces"], "/path/with spaces/file.txt"); +TestEqual("par.quotes", par["quotes"], "He said \"hello\""); +TestEqual("par.newlines", par["newlines"], "line1\nline2"); +TestEqual("par.tabs", par["tabs"], "col1\tcol2"); +Console.WriteLine(); + +// Test 5: Type conversions +Console.WriteLine("=== Test 5: Type conversions ==="); +TestTrue("par.string type", par["string"] is string); +TestTrue("par.integer type", par["integer"] is int); +TestTrue("par.float type", par["float"] is double); +TestTrue("par.bool_true type", par["bool_true"] is bool); +TestEqual("par.bool_false", par["bool_false"], false); +TestTrue("par.null_value", par["null_value"] == null); +Console.WriteLine(); + +// Test 6: Root-level values +Console.WriteLine("=== Test 6: Root-level values ==="); +TestEqual("simple_key", parsed["simple_key"], "simple_value"); +TestEqual("number_key", parsed["number_key"], 123); +TestEqual("bool_key", parsed["bool_key"], true); +Console.WriteLine(); + +// Test 7: Edge cases +Console.WriteLine("=== Test 7: Edge cases ==="); +TestEqual("par.empty_string", par["empty_string"], ""); +TestEqual("par.zero", par["zero"], 0); +var emptyArray = par["empty_array"] as List; +TestTrue("par.empty_array length", emptyArray.Count == 0); +TestTrue("par.empty_object type", par["empty_object"] is Dictionary); +Console.WriteLine(); + +// Summary +Console.WriteLine("=================================================="); +Console.WriteLine($"Tests completed: {testsPassed + testsFailed}"); +Console.WriteLine($"Tests passed: {testsPassed}"); +if (testsFailed > 0) +{ + Console.WriteLine($"{RED}Tests failed: {testsFailed}{RESET}"); + File.Delete(tempFile); + Environment.Exit(1); +} +else +{ + Console.WriteLine($"{GREEN}All tests passed!{RESET}"); + File.Delete(tempFile); + Environment.Exit(0); +} +} +finally +{ + if (File.Exists(tempFile)) + { + File.Delete(tempFile); + } +} diff --git a/src/test/resources/io/viash/helpers/languages/javascript/test_ViashParseJson.js b/src/test/resources/io/viash/helpers/languages/javascript/test_ViashParseJson.js new file mode 100644 index 000000000..3548ee099 --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/javascript/test_ViashParseJson.js @@ -0,0 +1,140 @@ +#!/usr/bin/env node + +// Test suite for viash_parse_json JavaScript function +console.log("Running viash_parse_json JavaScript unit tests...\n"); + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +// Load the parser - simple relative path from project root +const parserPath = 'src/main/resources/io/viash/languages/javascript/ViashParseJson.js'; +const { viashParseJson } = require(path.resolve(parserPath)); + +// Test helper functions +const RED = "\x1b[31m"; +const GREEN = "\x1b[32m"; +const RESET = "\x1b[0m"; + +let testPassed = 0; +let testFailed = 0; + +function testEqual(description, actual, expected) { + const actualStr = JSON.stringify(actual); + const expectedStr = JSON.stringify(expected); + + if (actualStr === expectedStr) { + console.log(`${GREEN}PASS${RESET}: ${description}`); + testPassed++; + return true; + } else { + console.log(`${RED}FAIL${RESET}: ${description}`); + console.log(` Expected: ${expectedStr}`); + console.log(` Got: ${actualStr}`); + testFailed++; + return false; + } +} + +// Create test JSON file +const testJson = path.join(os.tmpdir(), 'test_viash_' + Date.now() + '.json'); +const jsonContent = { + "par": { + "input": "file.txt", + "number": 42, + "flag": true, + "empty_value": "", + "array_simple": ["a", "b", "c"], + "array_numbers": [1, 2, 3], + "array_mixed": ["text", 123, true, null], + "nested": { + "level1": { + "level2": "deep_value" + } + }, + "path_with_spaces": "/path/with spaces/file.txt", + "quotes": "She said \"hello\"", + "newlines": "line1\nline2", + "tabs": "col1\tcol2", + "string": "text", + "integer": 123, + "float": 3.14, + "bool_true": true, + "bool_false": false, + "null_value": null, + "empty_string": "", + "zero": 0, + "empty_array": [], + "empty_object": {} + }, + "meta": { + "name": "test_component", + "version": "1.0" + }, + "simple_key": "value", + "number_key": 99, + "bool_key": false +}; + +fs.writeFileSync(testJson, JSON.stringify(jsonContent, null, 2)); + +// Set environment variable +process.env.VIASH_WORK_PARAMS = testJson; + +// Parse the JSON +const data = viashParseJson(); + +console.log("=== Test 1: Basic key-value pairs ==="); +testEqual("par.input", data.par.input, "file.txt"); +testEqual("par.number", data.par.number, 42); +testEqual("par.flag", data.par.flag, true); +testEqual("par.empty_value", data.par.empty_value, ""); +testEqual("meta.name", data.meta.name, "test_component"); +testEqual("meta.version", data.meta.version, "1.0"); + +console.log("\n=== Test 2: Arrays ==="); +testEqual("par.array_simple", data.par.array_simple, ["a", "b", "c"]); +testEqual("par.array_numbers", data.par.array_numbers, [1, 2, 3]); +testEqual("par.array_mixed.length", data.par.array_mixed.length, 4); + +console.log("\n=== Test 3: Nested structures ==="); +testEqual("par.nested.level1.level2", data.par.nested.level1.level2, "deep_value"); + +console.log("\n=== Test 4: Quoted strings ==="); +testEqual("par.path_with_spaces", data.par.path_with_spaces, "/path/with spaces/file.txt"); +testEqual("par.quotes", data.par.quotes, "She said \"hello\""); +testEqual("par.newlines", data.par.newlines, "line1\nline2"); +testEqual("par.tabs", data.par.tabs, "col1\tcol2"); + +console.log("\n=== Test 5: Type conversions ==="); +testEqual("par.string type", typeof data.par.string, "string"); +testEqual("par.integer type", typeof data.par.integer, "number"); +testEqual("par.float type", typeof data.par.float, "number"); +testEqual("par.bool_true type", typeof data.par.bool_true, "boolean"); +testEqual("par.bool_false", data.par.bool_false, false); +testEqual("par.null_value", data.par.null_value, null); + +console.log("\n=== Test 6: Root-level values ==="); +testEqual("simple_key", data.simple_key, "value"); +testEqual("number_key", data.number_key, 99); +testEqual("bool_key", data.bool_key, false); + +console.log("\n=== Test 7: Edge cases ==="); +testEqual("par.empty_string", data.par.empty_string, ""); +testEqual("par.zero", data.par.zero, 0); +testEqual("par.empty_array.length", data.par.empty_array.length, 0); +testEqual("par.empty_object type", typeof data.par.empty_object, "object"); + +// Clean up +fs.unlinkSync(testJson); + +// Print summary +console.log("\n=================================================="); +console.log(`Tests completed: ${testPassed + testFailed}`); +console.log(`Tests passed: ${testPassed}`); +if (testFailed > 0) { + console.log(`${RED}Tests failed: ${testFailed}${RESET}`); + process.exit(1); +} else { + console.log(`${GREEN}All tests passed!${RESET}`); +} diff --git a/src/test/resources/io/viash/helpers/languages/python/test_ViashParseJson.py b/src/test/resources/io/viash/helpers/languages/python/test_ViashParseJson.py new file mode 100644 index 000000000..03eaeeca4 --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/python/test_ViashParseJson.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 + +""" +Unit test for viash_parse_json function +This test verifies that the Python JSON parser correctly parses JSON content +and returns appropriate Python data structures. +""" + +import sys +import os +import tempfile +import json + +# Add the path to the JSON parser +parser_path = 'src/main/resources/io/viash/languages/python/ViashParseJson.py' +with open(parser_path, 'r') as f: + parser_code = f.read() + # Extract just the function, not the main execution part + exec('\n'.join([line for line in parser_code.split('\n') if not line.startswith('if __name__')])) + +# Colors for test output +class Colors: + RED = '\033[0;31m' + GREEN = '\033[0;32m' + YELLOW = '\033[1;33m' + NC = '\033[0m' # No Color + +# Test counters +tests_passed = 0 +tests_failed = 0 + +def run_test(test_name, expected, actual): + """Run a test and print results""" + global tests_passed, tests_failed + + print(f"Testing {test_name}... ", end="") + + if actual == expected: + print(f"{Colors.GREEN}PASS{Colors.NC}") + tests_passed += 1 + else: + print(f"{Colors.RED}FAIL{Colors.NC}") + print(f" Expected: {repr(expected)}") + print(f" Actual: {repr(actual)}") + tests_failed += 1 + +def test_with_temp_file(json_data): + """Create a temp file with JSON data and parse it""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(json_data, f) + temp_path = f.name + + try: + # Set environment variable + old_env = os.environ.get('VIASH_WORK_PARAMS') + os.environ['VIASH_WORK_PARAMS'] = temp_path + result = viash_parse_json() + if old_env is not None: + os.environ['VIASH_WORK_PARAMS'] = old_env + else: + del os.environ['VIASH_WORK_PARAMS'] + return result + finally: + os.unlink(temp_path) + +def main(): + """Run all tests""" + print(f"{Colors.YELLOW}Running viash_parse_json Python unit tests...{Colors.NC}") + print() + + # Test 1: Basic key-value pairs in sections + print("=== Test 1: Basic key-value pairs ===") + json_data = { + "par": { + "input": "/path/to/input.txt", + "number": 42, + "flag": True, + "empty_value": None + }, + "meta": { + "name": "test_component", + "version": "1.0.0" + } + } + + result = test_with_temp_file(json_data) + + # Test the results + run_test("par.input", "/path/to/input.txt", result.get('par', {}).get('input')) + run_test("par.number", 42, result.get('par', {}).get('number')) + run_test("par.flag", True, result.get('par', {}).get('flag')) + run_test("par.empty_value", None, result.get('par', {}).get('empty_value')) + run_test("meta.name", "test_component", result.get('meta', {}).get('name')) + run_test("meta.version", "1.0.0", result.get('meta', {}).get('version')) + print() + + # Test 2: Arrays + print("=== Test 2: Arrays ===") + json_data = { + "par": { + "array_simple": ["a", "b", "c"], + "array_numbers": [1, 2, 3], + "array_mixed": ["string", 42, True, None] + } + } + + result = test_with_temp_file(json_data) + + run_test("par.array_simple", ["a", "b", "c"], result.get('par', {}).get('array_simple')) + run_test("par.array_numbers", [1, 2, 3], result.get('par', {}).get('array_numbers')) + run_test("par.array_mixed", ["string", 42, True, None], result.get('par', {}).get('array_mixed')) + print() + + # Test 3: Nested structures + print("=== Test 3: Nested structures ===") + json_data = { + "par": { + "nested": { + "level1": { + "level2": "deep_value" + } + } + } + } + + result = test_with_temp_file(json_data) + + run_test("par.nested.level1.level2", "deep_value", + result.get('par', {}).get('nested', {}).get('level1', {}).get('level2')) + print() + + # Test 4: Quoted strings with special characters + print("=== Test 4: Quoted strings ===") + json_data = { + "par": { + "path_with_spaces": "/path/to/my file.txt", + "quotes": 'Text with "quotes"', + "newlines": "Line1\nLine2", + "tabs": "Column1\tColumn2" + } + } + + result = test_with_temp_file(json_data) + + run_test("par.path_with_spaces", "/path/to/my file.txt", result.get('par', {}).get('path_with_spaces')) + run_test("par.quotes", 'Text with "quotes"', result.get('par', {}).get('quotes')) + run_test("par.newlines", "Line1\nLine2", result.get('par', {}).get('newlines')) + run_test("par.tabs", "Column1\tColumn2", result.get('par', {}).get('tabs')) + print() + + # Test 5: Type conversions + print("=== Test 5: Type conversions ===") + json_data = { + "par": { + "string": "hello", + "integer": 123, + "float": 3.14, + "bool_true": True, + "bool_false": False, + "null_value": None + } + } + + result = test_with_temp_file(json_data) + + run_test("par.string type", str, type(result.get('par', {}).get('string'))) + run_test("par.integer type", int, type(result.get('par', {}).get('integer'))) + run_test("par.float type", float, type(result.get('par', {}).get('float'))) + run_test("par.bool_true type", bool, type(result.get('par', {}).get('bool_true'))) + run_test("par.bool_false", False, result.get('par', {}).get('bool_false')) + run_test("par.null_value", None, result.get('par', {}).get('null_value')) + print() + + # Test 6: Root-level values (no section) + print("=== Test 6: Root-level values ===") + json_data = { + "simple_key": "simple_value", + "number_key": 789, + "bool_key": True + } + + result = test_with_temp_file(json_data) + + run_test("simple_key", "simple_value", result.get('simple_key')) + run_test("number_key", 789, result.get('number_key')) + run_test("bool_key", True, result.get('bool_key')) + print() + + # Test 7: Edge cases + print("=== Test 7: Edge cases ===") + json_data = { + "par": { + "empty_string": "", + "zero": 0, + "empty_array": [], + "empty_object": {} + } + } + + result = test_with_temp_file(json_data) + + run_test("par.empty_string", "", result.get('par', {}).get('empty_string')) + run_test("par.zero", 0, result.get('par', {}).get('zero')) + run_test("par.empty_array", [], result.get('par', {}).get('empty_array')) + run_test("par.empty_object", {}, result.get('par', {}).get('empty_object')) + print() + + # Print summary + total_tests = tests_passed + tests_failed + print("=" * 50) + print(f"Tests completed: {total_tests}") + print(f"{Colors.GREEN}Tests passed: {tests_passed}{Colors.NC}") + if tests_failed > 0: + print(f"{Colors.RED}Tests failed: {tests_failed}{Colors.NC}") + sys.exit(1) + else: + print(f"{Colors.GREEN}All tests passed!{Colors.NC}") + +if __name__ == "__main__": + main() diff --git a/src/test/resources/io/viash/helpers/languages/r/test_ViashParseJson.R b/src/test/resources/io/viash/helpers/languages/r/test_ViashParseJson.R new file mode 100644 index 000000000..e21a53415 --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/r/test_ViashParseJson.R @@ -0,0 +1,229 @@ +#!/usr/bin/env Rscript + +# Test suite for viash_parse_json R function using jsonlite +cat("Running viash_parse_json jsonlite-only unit tests...\n\n") + +# Source the jsonlite-only parser +source("src/main/resources/io/viash/languages/r/ViashParseJson.R") + +# Test helper functions +RED <- "\033[31m" +GREEN <- "\033[32m" +RESET <- "\033[0m" + +test_passed <- 0 +test_failed <- 0 + +test_equal <- function(description, actual, expected) { + if (identical(actual, expected)) { + cat(sprintf("%sPASS%s: %s\n", GREEN, RESET, description)) + test_passed <<- test_passed + 1 + return(TRUE) + } else { + cat(sprintf("%sFAIL%s: %s\n", RED, RESET, description)) + cat(" Expected:", deparse(expected), "\n") + cat(" Got:", deparse(actual), "\n") + test_failed <<- test_failed + 1 + return(FALSE) + } +} + +# Check that jsonlite is available +if (!requireNamespace("jsonlite", quietly = TRUE)) { + cat("SKIP: jsonlite is not installed, skipping jsonlite-only tests.\n") + quit(status = 0) +} + +## =================================================================== +## TEST 1: Basic key-value pairs +## =================================================================== +cat("=== Test 1: Basic key-value pairs ===\n") + +test_json <- tempfile(fileext = ".json") +writeLines('{ + "par": { + "input": "file.txt", + "number": 42, + "flag": true, + "empty_value": "", + "array_simple": ["a", "b", "c"], + "array_numbers": [1, 2, 3], + "nested": { + "level1": { + "level2": "deep_value" + } + }, + "path_with_spaces": "/path/with spaces/file.txt", + "quotes": "She said \\"hello\\"", + "newlines": "line1\\nline2", + "tabs": "col1\\tcol2", + "string": "text", + "integer": 123, + "float": 3.14, + "bool_true": true, + "bool_false": false, + "null_value": null, + "empty_string": "", + "zero": 0, + "empty_array": [], + "empty_object": {} + }, + "meta": { + "name": "test_component", + "version": "1.0" + }, + "simple_key": "value", + "number_key": 99, + "bool_key": false +}', test_json) + +Sys.setenv(VIASH_WORK_PARAMS = test_json) +data <- viash_parse_json() +unlink(test_json) + +test_equal("par$input", data$par$input, "file.txt") +test_equal("par$number", data$par$number, 42L) +test_equal("par$flag", data$par$flag, TRUE) +test_equal("par$empty_value", data$par$empty_value, "") +test_equal("meta$name", data$meta$name, "test_component") +test_equal("meta$version", data$meta$version, "1.0") + +## =================================================================== +## TEST 2: Arrays +## =================================================================== +cat("\n=== Test 2: Arrays ===\n") +test_equal("par$array_simple", data$par$array_simple, c("a", "b", "c")) +test_equal("par$array_numbers", data$par$array_numbers, c(1L, 2L, 3L)) + +## =================================================================== +## TEST 3: Nested structures +## =================================================================== +cat("\n=== Test 3: Nested structures ===\n") +test_equal("par$nested$level1$level2", data$par$nested$level1$level2, "deep_value") + +## =================================================================== +## TEST 4: Quoted strings +## =================================================================== +cat("\n=== Test 4: Quoted strings ===\n") +test_equal("par$path_with_spaces", data$par$path_with_spaces, "/path/with spaces/file.txt") +test_equal("par$quotes", data$par$quotes, "She said \"hello\"") +test_equal("par$newlines", data$par$newlines, "line1\nline2") +test_equal("par$tabs", data$par$tabs, "col1\tcol2") + +## =================================================================== +## TEST 5: Type conversions +## =================================================================== +cat("\n=== Test 5: Type conversions ===\n") +test_equal("par$string type", is.character(data$par$string), TRUE) +test_equal("par$integer type", is.numeric(data$par$integer), TRUE) +test_equal("par$float type", is.numeric(data$par$float), TRUE) +test_equal("par$bool_true type", is.logical(data$par$bool_true), TRUE) +test_equal("par$bool_false", data$par$bool_false, FALSE) +test_equal("par$null_value", data$par$null_value, NULL) + +## =================================================================== +## TEST 6: Root-level values +## =================================================================== +cat("\n=== Test 6: Root-level values ===\n") +test_equal("simple_key", data$simple_key, "value") +test_equal("number_key", data$number_key, 99L) +test_equal("bool_key", data$bool_key, FALSE) + +## =================================================================== +## TEST 7: Edge cases +## =================================================================== +cat("\n=== Test 7: Edge cases ===\n") +test_equal("par$empty_string", data$par$empty_string, "") +test_equal("par$zero", data$par$zero, 0L) +test_equal("par$empty_array length", length(data$par$empty_array), 0L) +test_equal("par$empty_object type", is.list(data$par$empty_object), TRUE) + +## =================================================================== +## TEST 8: Special characters +## =================================================================== +cat("\n=== Test 8: Special characters ===\n") + +special_json <- tempfile(fileext = ".json") +writeLines('{ + "par": { + "backtick": "run `id`", + "dollar": "path is $PATH", + "subst": "time $(date)", + "backslash": "C:\\\\Users\\\\test", + "quotes_in_str": "She said \\"hi\\"", + "slash": "a/b/c", + "mixed": "a\\\\b\\"c$d`e" + } +}', special_json) +Sys.setenv(VIASH_WORK_PARAMS = special_json) +data <- viash_parse_json() +unlink(special_json) + +test_equal("special: backtick", data$par$backtick, "run `id`") +test_equal("special: dollar", data$par$dollar, "path is $PATH") +test_equal("special: subst", data$par$subst, "time $(date)") +test_equal("special: backslash", data$par$backslash, "C:\\Users\\test") +test_equal("special: quotes", data$par$quotes_in_str, 'She said "hi"') +test_equal("special: slash", data$par$slash, "a/b/c") +test_equal("special: mixed", data$par$mixed, 'a\\b"c$d`e') + +## =================================================================== +## TEST 9: Numeric edge cases +## =================================================================== +cat("\n=== Test 9: Numeric edge cases ===\n") + +num_json <- tempfile(fileext = ".json") +writeLines('{"par":{"neg":-7,"sci":1.5e10,"neg_sci":-2.5E-3,"big":999999999}}', + num_json) +Sys.setenv(VIASH_WORK_PARAMS = num_json) +data <- viash_parse_json() +unlink(num_json) + +test_equal("numeric: neg", data$par$neg, -7L) +test_equal("numeric: sci", data$par$sci, 1.5e10) +test_equal("numeric: neg_sci", data$par$neg_sci, -2.5E-3) +test_equal("numeric: big", data$par$big, 999999999L) + +## =================================================================== +## TEST 10: Error handling - missing file +## =================================================================== +cat("\n=== Test 10: Error handling ===\n") + +Sys.setenv(VIASH_WORK_PARAMS = "/nonexistent/path.json") +result <- tryCatch(viash_parse_json(), error = function(e) "ERROR") +test_equal("missing file errors", result, "ERROR") + +## =================================================================== +## TEST 11: Big integer handling (bigint_as_char = TRUE) +## =================================================================== +cat("\n=== Test 11: Big integer handling ===\n") + +bigint_json <- tempfile(fileext = ".json") +writeLines('{ + "par": { + "small_int": 42, + "big_int": 9007199254740993, + "negative_big": -9007199254740993 + } +}', bigint_json) +Sys.setenv(VIASH_WORK_PARAMS = bigint_json) +data <- viash_parse_json() +unlink(bigint_json) + +test_equal("small_int is integer", data$par$small_int, 42L) +# Big integers (> 2^53) should be preserved as character strings +test_equal("big_int is character", is.character(data$par$big_int), TRUE) +test_equal("big_int value preserved", data$par$big_int, "9007199254740993") +test_equal("negative_big is character", is.character(data$par$negative_big), TRUE) +test_equal("negative_big value preserved", data$par$negative_big, "-9007199254740993") + +# Print summary +cat("\n==================================================\n") +cat(sprintf("Tests completed: %d\n", test_passed + test_failed)) +cat(sprintf("Tests passed: %d\n", test_passed)) +if (test_failed > 0) { + cat(sprintf("%sTests failed: %d%s\n", RED, test_failed, RESET)) + quit(status = 1) +} else { + cat(sprintf("%sAll tests passed!%s\n", GREEN, RESET)) +} diff --git a/src/test/resources/io/viash/helpers/languages/r/test_ViashParseJsonHybrid.R b/src/test/resources/io/viash/helpers/languages/r/test_ViashParseJsonHybrid.R new file mode 100644 index 000000000..60ac7e6db --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/r/test_ViashParseJsonHybrid.R @@ -0,0 +1,385 @@ +#!/usr/bin/env Rscript + +# Test suite for viash_parse_json R function in hybrid mode +# (jsonlite preferred, custom parser fallback) +cat("Running viash_parse_json hybrid mode unit tests...\n\n") + +# Source the hybrid parser (self-contained: includes both jsonlite and custom parser) +source("src/main/resources/io/viash/languages/r/ViashParseJsonHybrid.R") + +# Test helper functions +RED <- "\033[31m" +GREEN <- "\033[32m" +RESET <- "\033[0m" + +test_passed <- 0 +test_failed <- 0 + +test_equal <- function(description, actual, expected) { + if (identical(actual, expected)) { + cat(sprintf("%sPASS%s: %s\n", GREEN, RESET, description)) + test_passed <<- test_passed + 1 + return(TRUE) + } else { + cat(sprintf("%sFAIL%s: %s\n", RED, RESET, description)) + cat(" Expected:", deparse(expected), "\n") + cat(" Got:", deparse(actual), "\n") + test_failed <<- test_failed + 1 + return(FALSE) + } +} + +## =================================================================== +## TEST 1: Basic key-value pairs +## =================================================================== +cat("=== Test 1: Basic key-value pairs ===\n") + +test_json <- tempfile(fileext = ".json") +writeLines('{ + "par": { + "input": "file.txt", + "number": 42, + "flag": true, + "empty_value": "", + "array_simple": ["a", "b", "c"], + "array_numbers": [1, 2, 3], + "array_mixed": ["text", 123, true, null], + "nested": { + "level1": { + "level2": "deep_value" + } + }, + "path_with_spaces": "/path/with spaces/file.txt", + "quotes": "She said \\"hello\\"", + "newlines": "line1\\nline2", + "tabs": "col1\\tcol2", + "string": "text", + "integer": 123, + "float": 3.14, + "bool_true": true, + "bool_false": false, + "null_value": null, + "empty_string": "", + "zero": 0, + "empty_array": [], + "empty_object": {} + }, + "meta": { + "name": "test_component", + "version": "1.0" + }, + "simple_key": "value", + "number_key": 99, + "bool_key": false +}', test_json) + +Sys.setenv(VIASH_WORK_PARAMS = test_json) +data <- viash_parse_json() +unlink(test_json) + +test_equal("par$input", data$par$input, "file.txt") +test_equal("par$number", data$par$number, 42L) +test_equal("par$flag", data$par$flag, TRUE) +test_equal("par$empty_value", data$par$empty_value, "") +test_equal("meta$name", data$meta$name, "test_component") +test_equal("meta$version", data$meta$version, "1.0") + +## =================================================================== +## TEST 2: Arrays +## =================================================================== +cat("\n=== Test 2: Arrays ===\n") +test_equal("par$array_simple", data$par$array_simple, c("a", "b", "c")) +test_equal("par$array_numbers", data$par$array_numbers, c(1L, 2L, 3L)) +test_equal("par$array_mixed length", length(data$par$array_mixed), 4L) + +## =================================================================== +## TEST 3: Nested structures +## =================================================================== +cat("\n=== Test 3: Nested structures ===\n") +test_equal("par$nested$level1$level2", data$par$nested$level1$level2, "deep_value") + +## =================================================================== +## TEST 4: Quoted strings +## =================================================================== +cat("\n=== Test 4: Quoted strings ===\n") +test_equal("par$path_with_spaces", data$par$path_with_spaces, "/path/with spaces/file.txt") +test_equal("par$quotes", data$par$quotes, "She said \"hello\"") +test_equal("par$newlines", data$par$newlines, "line1\nline2") +test_equal("par$tabs", data$par$tabs, "col1\tcol2") + +## =================================================================== +## TEST 5: Type conversions +## =================================================================== +cat("\n=== Test 5: Type conversions ===\n") +test_equal("par$string type", is.character(data$par$string), TRUE) +test_equal("par$integer type", is.numeric(data$par$integer), TRUE) +test_equal("par$float type", is.numeric(data$par$float), TRUE) +test_equal("par$bool_true type", is.logical(data$par$bool_true), TRUE) +test_equal("par$bool_false", data$par$bool_false, FALSE) +test_equal("par$null_value", data$par$null_value, NULL) + +## =================================================================== +## TEST 6: Root-level values +## =================================================================== +cat("\n=== Test 6: Root-level values ===\n") +test_equal("simple_key", data$simple_key, "value") +test_equal("number_key", data$number_key, 99L) +test_equal("bool_key", data$bool_key, FALSE) + +## =================================================================== +## TEST 7: Edge cases +## =================================================================== +cat("\n=== Test 7: Edge cases ===\n") +test_equal("par$empty_string", data$par$empty_string, "") +test_equal("par$zero", data$par$zero, 0L) +test_equal("par$empty_array length", length(data$par$empty_array), 0L) +test_equal("par$empty_object type", is.list(data$par$empty_object), TRUE) + +## =================================================================== +## TEST 8: Compact / minified JSON (no whitespace) +## =================================================================== +cat("\n=== Test 8: Compact JSON (minified) ===\n") + +compact_json <- tempfile(fileext = ".json") +writeLines('{"par":{"x":"hello","y":42,"z":true,"arr":["a","b"],"nested":{"k":"v"}},"meta":{"name":"comp"}}', + compact_json) +Sys.setenv(VIASH_WORK_PARAMS = compact_json) +data <- viash_parse_json() +unlink(compact_json) + +test_equal("compact: par$x", data$par$x, "hello") +test_equal("compact: par$y", data$par$y, 42L) +test_equal("compact: par$z", data$par$z, TRUE) +test_equal("compact: par$arr", data$par$arr, c("a", "b")) +test_equal("compact: par$nested$k", data$par$nested$k, "v") +test_equal("compact: meta$name", data$meta$name, "comp") + +## =================================================================== +## TEST 9: Special characters in strings +## =================================================================== +cat("\n=== Test 9: Special characters ===\n") + +special_json <- tempfile(fileext = ".json") +writeLines('{ + "par": { + "backtick": "run `id`", + "dollar": "path is $PATH", + "subst": "time $(date)", + "backslash": "C:\\\\Users\\\\test", + "quotes_in_str": "She said \\"hi\\"", + "slash": "a/b/c", + "mixed": "a\\\\b\\"c$d`e" + } +}', special_json) +Sys.setenv(VIASH_WORK_PARAMS = special_json) +data <- viash_parse_json() +unlink(special_json) + +test_equal("special: backtick", data$par$backtick, "run `id`") +test_equal("special: dollar", data$par$dollar, "path is $PATH") +test_equal("special: subst", data$par$subst, "time $(date)") +test_equal("special: backslash", data$par$backslash, "C:\\Users\\test") +test_equal("special: quotes", data$par$quotes_in_str, 'She said "hi"') +test_equal("special: slash", data$par$slash, "a/b/c") +test_equal("special: mixed", data$par$mixed, 'a\\b"c$d`e') + +## =================================================================== +## TEST 10: Numeric edge cases +## =================================================================== +cat("\n=== Test 10: Numeric edge cases ===\n") + +num_json <- tempfile(fileext = ".json") +writeLines('{"par":{"neg":-7,"sci":1.5e10,"neg_sci":-2.5E-3,"big":999999999}}', + num_json) +Sys.setenv(VIASH_WORK_PARAMS = num_json) +data <- viash_parse_json() +unlink(num_json) + +test_equal("numeric: neg", data$par$neg, -7L) +test_equal("numeric: sci", data$par$sci, 1.5e10) +test_equal("numeric: neg_sci", data$par$neg_sci, -2.5E-3) +test_equal("numeric: big", data$par$big, 999999999L) + +## =================================================================== +## TEST 11: Mixed-type arrays +## =================================================================== +cat("\n=== Test 11: Mixed-type arrays ===\n") + +mix_json <- tempfile(fileext = ".json") +writeLines('{"par":{"mix":["text",123,true,false,null,-5]}}', mix_json) +Sys.setenv(VIASH_WORK_PARAMS = mix_json) +data <- viash_parse_json() +unlink(mix_json) + +# jsonlite with simplifyVector converts mixed-type arrays to character vectors, +# while the custom parser preserves native types in a list. +# Both behaviors are acceptable; test for whichever parser was used. +if (is.character(data$par$mix)) { + # jsonlite path: mixed arrays become character vectors (NULLs become NA) + test_equal("mixed array length (jsonlite)", length(data$par$mix), 6L) + test_equal("mixed array[1] (jsonlite)", data$par$mix[[1]], "text") + test_equal("mixed array[2] (jsonlite)", data$par$mix[[2]], "123") + test_equal("mixed array[3] (jsonlite)", data$par$mix[[3]], "TRUE") + test_equal("mixed array[4] (jsonlite)", data$par$mix[[4]], "FALSE") + test_equal("mixed array[5] is NA (jsonlite)", is.na(data$par$mix[[5]]), TRUE) + test_equal("mixed array[6] (jsonlite)", data$par$mix[[6]], "-5") +} else { + # Custom parser path: mixed arrays are kept as lists with native types + test_equal("mixed array length (custom)", length(data$par$mix), 6L) + test_equal("mixed array[[1]] (custom)", data$par$mix[[1]], "text") + test_equal("mixed array[[2]] (custom)", data$par$mix[[2]], 123L) + test_equal("mixed array[[3]] (custom)", data$par$mix[[3]], TRUE) + test_equal("mixed array[[4]] (custom)", data$par$mix[[4]], FALSE) + test_equal("mixed array[[5]] (custom)", is.null(data$par$mix[[5]]), TRUE) + test_equal("mixed array[[6]] (custom)", data$par$mix[[6]], -5L) +} + +## =================================================================== +## TEST 12: Empty and minimal JSON +## =================================================================== +cat("\n=== Test 12: Empty/minimal JSON ===\n") + +empty_json <- tempfile(fileext = ".json") +writeLines('{}', empty_json) +Sys.setenv(VIASH_WORK_PARAMS = empty_json) +data <- viash_parse_json() +unlink(empty_json) +test_equal("empty object", is.list(data), TRUE) +test_equal("empty object length", length(data), 0L) + +empty_par_json <- tempfile(fileext = ".json") +writeLines('{"par":{}}', empty_par_json) +Sys.setenv(VIASH_WORK_PARAMS = empty_par_json) +data <- viash_parse_json() +unlink(empty_par_json) +test_equal("empty par", length(data$par), 0L) + +## =================================================================== +## TEST 13: Deeply nested objects +## =================================================================== +cat("\n=== Test 13: Deeply nested objects ===\n") + +deep_json <- tempfile(fileext = ".json") +writeLines('{ + "par": { + "config": { + "db": { + "host": "localhost", + "port": 5432 + }, + "cache": true + } + } +}', deep_json) +Sys.setenv(VIASH_WORK_PARAMS = deep_json) +data <- viash_parse_json() +unlink(deep_json) + +test_equal("deep: config$db$host", data$par$config$db$host, "localhost") +test_equal("deep: config$db$port", data$par$config$db$port, 5432L) +test_equal("deep: config$cache", data$par$config$cache, TRUE) + +## =================================================================== +## TEST 14: Error handling +## =================================================================== +cat("\n=== Test 14: Error handling ===\n") + +# Invalid JSON: missing closing brace +bad_json <- tempfile(fileext = ".json") +writeLines('{"par":{"x": "hello"', bad_json) +Sys.setenv(VIASH_WORK_PARAMS = bad_json) +result <- tryCatch(viash_parse_json(), error = function(e) "ERROR") +unlink(bad_json) +test_equal("invalid JSON errors", result, "ERROR") + +# Invalid JSON: trailing garbage +bad_json2 <- tempfile(fileext = ".json") +writeLines('{"x": }', bad_json2) +Sys.setenv(VIASH_WORK_PARAMS = bad_json2) +result2 <- tryCatch(viash_parse_json(), error = function(e) "ERROR") +unlink(bad_json2) +test_equal("malformed value errors", result2, "ERROR") + +## =================================================================== +## TEST 15: Tricky string content (commas, colons, braces) +## =================================================================== +cat("\n=== Test 15: Tricky string content ===\n") + +tricky_json <- tempfile(fileext = ".json") +writeLines('{ + "par": { + "with_comma": "a, b, c", + "with_colon": "key: value", + "with_braces": "obj = {x: 1}", + "with_brackets": "arr = [1, 2]" + } +}', tricky_json) +Sys.setenv(VIASH_WORK_PARAMS = tricky_json) +data <- viash_parse_json() +unlink(tricky_json) + +test_equal("tricky: comma", data$par$with_comma, "a, b, c") +test_equal("tricky: colon", data$par$with_colon, "key: value") +test_equal("tricky: braces", data$par$with_braces, "obj = {x: 1}") +test_equal("tricky: brackets", data$par$with_brackets, "arr = [1, 2]") + +## =================================================================== +## TEST 16: Custom parser fallback via .viash_json_parse directly +## =================================================================== +cat("\n=== Test 16: Custom parser fallback ===\n") + +# Test the custom parser path by calling .viash_json_parse directly +test_json_fb <- tempfile(fileext = ".json") +writeLines('{"par":{"x":"hello","y":42,"flag":true}}', test_json_fb) + +json_text <- paste(readLines(test_json_fb, warn = FALSE), collapse = "\n") +data2 <- .viash_json_parse(json_text) +unlink(test_json_fb) + +test_equal("fallback: par$x", data2$par$x, "hello") +test_equal("fallback: par$y", data2$par$y, 42L) +test_equal("fallback: par$flag", data2$par$flag, TRUE) + +## =================================================================== +## TEST 17: Fallback complex parsing +## =================================================================== +cat("\n=== Test 17: Fallback complex parsing ===\n") + +complex_json <- tempfile(fileext = ".json") +writeLines('{ + "par": { + "config": { + "db": { + "host": "localhost", + "port": 5432 + }, + "cache": true + }, + "array_numbers": [1, 2, 3], + "empty_array": [], + "null_value": null + } +}', complex_json) + +# Test the custom parser path directly +json_text <- paste(readLines(complex_json, warn = FALSE), collapse = "\n") +data <- .viash_json_parse(json_text) +unlink(complex_json) + +test_equal("fallback: config$db$host", data$par$config$db$host, "localhost") +test_equal("fallback: config$db$port", data$par$config$db$port, 5432L) +test_equal("fallback: config$cache", data$par$config$cache, TRUE) +test_equal("fallback: array_numbers", data$par$array_numbers, c(1L, 2L, 3L)) +test_equal("fallback: empty_array", length(data$par$empty_array), 0L) +test_equal("fallback: null_value", data$par$null_value, NULL) + +# Print summary +cat("\n==================================================\n") +cat(sprintf("Tests completed: %d\n", test_passed + test_failed)) +cat(sprintf("Tests passed: %d\n", test_passed)) +if (test_failed > 0) { + cat(sprintf("%sTests failed: %d%s\n", RED, test_failed, RESET)) + quit(status = 1) +} else { + cat(sprintf("%sAll tests passed!%s\n", GREEN, RESET)) +} diff --git a/src/test/resources/io/viash/helpers/languages/scala/.gitignore b/src/test/resources/io/viash/helpers/languages/scala/.gitignore new file mode 100644 index 000000000..edd910422 --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/scala/.gitignore @@ -0,0 +1,2 @@ +.bsp +.scala-build \ No newline at end of file diff --git a/src/test/resources/io/viash/helpers/languages/scala/test_ViashParseJson.scala b/src/test/resources/io/viash/helpers/languages/scala/test_ViashParseJson.scala new file mode 100755 index 000000000..c21e81680 --- /dev/null +++ b/src/test/resources/io/viash/helpers/languages/scala/test_ViashParseJson.scala @@ -0,0 +1,222 @@ +#!/usr/bin/env scala + +//> using file ../../../../../../../main/resources/io/viash/languages/scala/ViashParseJson.scala + +/* + * Unit test for ViashJsonParser.parseJson function + * This test verifies that the Scala JSON parser correctly parses JSON content + * and returns appropriate Scala data structures. + */ + +// Colors for test output +object Colors { + val RED = "\u001b[0;31m" + val GREEN = "\u001b[0;32m" + val YELLOW = "\u001b[1;33m" + val NC = "\u001b[0m" // No Color +} + +// Test counters +var testsPass = 0 +var testsFail = 0 + +def runTest(testName: String, expected: Any, actual: Any): Unit = { + print(s"Testing $testName... ") + + val isEqual = (expected, actual) match { + case (None, None) => true + case (Some(e), Some(a)) => e == a + case (e: Double, a: Int) => e == a.toDouble + case (e: Int, a: Double) => e.toDouble == a + case (e, a) => e == a + } + + if (isEqual) { + println(s"${Colors.GREEN}PASS${Colors.NC}") + testsPass += 1 + } else { + println(s"${Colors.RED}FAIL${Colors.NC}") + println(s" Expected: $expected (${expected.getClass.getSimpleName})") + println(s" Actual: $actual (${actual.getClass.getSimpleName})") + testsFail += 1 + } +} + +def testArray(testName: String, expectedArray: List[Any], actualArray: List[Any]): Unit = { + print(s"Testing $testName... ") + + val isEqual = expectedArray == actualArray + + if (isEqual) { + println(s"${Colors.GREEN}PASS${Colors.NC}") + testsPass += 1 + } else { + println(s"${Colors.RED}FAIL${Colors.NC}") + println(s" Expected array: [${expectedArray.mkString(", ")}]") + println(s" Actual array: [${actualArray.mkString(", ")}]") + testsFail += 1 + } +} + +def testType(testName: String, expectedType: String, actual: Any): Unit = { + print(s"Testing $testName type... ") + + val actualType = actual match { + case _: String => "String" + case _: Int => "Int" + case _: Double => "Double" + case _: Boolean => "Boolean" + case _: List[_] => "List" + case _: Map[_, _] => "Map" + case null => "Null" + case _ => actual.getClass.getSimpleName + } + + val isEqual = expectedType == actualType + + if (isEqual) { + println(s"${Colors.GREEN}PASS${Colors.NC}") + testsPass += 1 + } else { + println(s"${Colors.RED}FAIL${Colors.NC}") + println(s" Expected type: $expectedType") + println(s" Actual type: $actualType") + testsFail += 1 + } +} + +@main def main(): Unit = { + println("Running ViashJsonParser Scala unit tests...") + println() + + // Test JSON content + val jsonContent = """{ + "par": { + "input": "file.txt", + "number": 42, + "flag": true, + "empty_value": "", + "array_simple": ["a", "b", "c"], + "array_numbers": [1, 2, 3], + "array_mixed": ["text", 123, true], + "nested": { + "level1": { + "level2": "deep_value" + } + }, + "path_with_spaces": "/path/with spaces/file.txt", + "quotes": "He said \"hello\"", + "newlines": "line1\nline2", + "tabs": "col1\tcol2", + "string": "text", + "integer": 42, + "float": 3.14, + "bool_true": true, + "bool_false": false, + "null_value": null, + "empty_string": "", + "zero": 0, + "empty_array": [], + "empty_object": {} + }, + "meta": { + "name": "test_component", + "version": "1.0.0" + }, + "simple_key": "simple_value", + "number_key": 123, + "bool_key": true +}""" + + // Parse the JSON + val tempFile = java.nio.file.Files.createTempFile("viash-test", ".json") + try { + java.nio.file.Files.write(tempFile, jsonContent.getBytes("UTF-8")) + val parsed = ViashJsonParser.parseJson(Some(tempFile.toString)) + val par = parsed("par").asInstanceOf[Map[String, Any]] + val meta = parsed("meta").asInstanceOf[Map[String, Any]] + + // Test 1: Basic key-value pairs + println("=== Test 1: Basic key-value pairs ===") + runTest("par.input", "file.txt", par("input")) + runTest("par.number", 42, par("number")) + runTest("par.flag", true, par("flag")) + runTest("par.empty_value", "", par("empty_value")) + runTest("meta.name", "test_component", meta("name")) + runTest("meta.version", "1.0.0", meta("version")) + println() + + // Test 2: Arrays + println("=== Test 2: Arrays ===") + val arraySimple = par("array_simple").asInstanceOf[List[Any]] + runTest("par.array_simple length", 3, arraySimple.length) + runTest("par.array_simple[0]", "a", arraySimple(0)) + runTest("par.array_simple[1]", "b", arraySimple(1)) + runTest("par.array_simple[2]", "c", arraySimple(2)) + + val arrayNumbers = par("array_numbers").asInstanceOf[List[Any]] + runTest("par.array_numbers length", 3, arrayNumbers.length) + runTest("par.array_numbers[0]", 1, arrayNumbers(0)) + runTest("par.array_numbers[1]", 2, arrayNumbers(1)) + runTest("par.array_numbers[2]", 3, arrayNumbers(2)) + + val arrayMixed = par("array_mixed").asInstanceOf[List[Any]] + runTest("par.array_mixed length", 3, arrayMixed.length) + println() + + // Test 3: Nested structures + println("=== Test 3: Nested structures ===") + val nested = par("nested").asInstanceOf[Map[String, Any]] + val level1 = nested("level1").asInstanceOf[Map[String, Any]] + runTest("par.nested.level1.level2", "deep_value", level1("level2")) + println() + + // Test 4: Quoted strings + println("=== Test 4: Quoted strings ===") + runTest("par.path_with_spaces", "/path/with spaces/file.txt", par("path_with_spaces")) + runTest("par.quotes", "He said \"hello\"", par("quotes")) + runTest("par.newlines", "line1\nline2", par("newlines")) + runTest("par.tabs", "col1\tcol2", par("tabs")) + println() + + // Test 5: Type conversions + println("=== Test 5: Type conversions ===") + testType("par.string", "String", par("string")) + testType("par.integer", "Int", par("integer")) + testType("par.float", "Double", par("float")) + testType("par.bool_true", "Boolean", par("bool_true")) + runTest("par.bool_false", false, par("bool_false")) + runTest("par.null_value", null, par("null_value")) + println() + + // Test 6: Root-level values + println("=== Test 6: Root-level values ===") + runTest("simple_key", "simple_value", parsed("simple_key")) + runTest("number_key", 123, parsed("number_key")) + runTest("bool_key", true, parsed("bool_key")) + println() + + // Test 7: Edge cases + println("=== Test 7: Edge cases ===") + runTest("par.empty_string", "", par("empty_string")) + runTest("par.zero", 0, par("zero")) + val emptyArray = par("empty_array").asInstanceOf[List[Any]] + runTest("par.empty_array length", 0, emptyArray.length) + testType("par.empty_object", "Map", par("empty_object")) + println() + + // Summary + println("==================================================") + println(s"Tests completed: ${testsPass + testsFail}") + println(s"Tests passed: $testsPass") + if (testsFail > 0) { + println(s"${Colors.RED}Tests failed: $testsFail${Colors.NC}") + sys.exit(1) + } else { + println(s"${Colors.GREEN}All tests passed!${Colors.NC}") + sys.exit(0) + } + } finally { + java.nio.file.Files.deleteIfExists(tempFile) + } +} diff --git a/src/test/resources/test_escaping/code.sh b/src/test/resources/test_escaping/code.sh index 334df0fd4..d1864c927 100755 --- a/src/test/resources/test_escaping/code.sh +++ b/src/test/resources/test_escaping/code.sh @@ -31,6 +31,21 @@ function output { fi } +# Helper function to join array elements with semicolon +function join_by_semicolon { + local varname=$1 + # Check if variable is an array + if declare -p "$varname" 2>/dev/null | grep -q '^declare -a'; then + local arr_name="${varname}[@]" + local arr=("${!arr_name}") + local IFS=";" + echo "${arr[*]}" + else + # Not an array, just echo the value + echo "${!varname}" + fi +} + log "INFO: Parsed input arguments." if [ -z "$par_output" ]; then @@ -55,5 +70,5 @@ INPUT=`head -1 "$par_input"` output "head of input: |$INPUT|" RESOURCE=`head -1 "$meta_resources_dir/resource1.txt"` output "head of resource1: |$RESOURCE|" -output "multiple: |$par_multiple|" -output "multiple_pos: |$par_multiple_pos|" +output "multiple: |$(join_by_semicolon par_multiple)|" +output "multiple_pos: |$(join_by_semicolon par_multiple_pos)|" diff --git a/src/test/resources/test_escaping/config.vsh.yaml b/src/test/resources/test_escaping/config.vsh.yaml index 8c082af71..4231138b5 100755 --- a/src/test/resources/test_escaping/config.vsh.yaml +++ b/src/test/resources/test_escaping/config.vsh.yaml @@ -31,6 +31,7 @@ arguments: example: Test {test_detect} example value resources: - type: bash_script + use_jq: true path: ./code.sh engines: - type: native diff --git a/src/test/resources/test_languages/bash/code.sh b/src/test/resources/test_languages/bash/code.sh index 94bcf262b..ed29f2bd5 100755 --- a/src/test/resources/test_languages/bash/code.sh +++ b/src/test/resources/test_languages/bash/code.sh @@ -19,16 +19,16 @@ set -e function log { if [ -z "$par_log" ]; then - echo $@ + echo "$*" else - echo $@ >> $par_log + echo "$*" >> $par_log fi } function output { if [ -z "$par_output" ]; then - echo $@ + echo "$*" else - echo $@ >> $par_output + echo "$*" >> $par_output fi } @@ -56,8 +56,14 @@ INPUT=`head -1 "$par_input"` output "head of input: |$INPUT|" RESOURCE=`head -1 "$meta_resources_dir/resource1.txt"` output "head of resource1: |$RESOURCE|" -output "multiple: |$par_multiple|" -output "multiple_pos: |$par_multiple_pos|" + +# Join array elements with semicolons +# Note: We need to set IFS before expansion, then restore it +_old_IFS="$IFS" +IFS=';' +output "multiple: |${par_multiple[*]}|" +output "multiple_pos: |${par_multiple_pos[*]}|" +IFS="$_old_IFS" output "meta_name: |$meta_name|" output "meta_resources_dir: |$meta_resources_dir|" @@ -72,4 +78,4 @@ output "meta_memory_kib: |$meta_memory_kib|" output "meta_memory_mib: |$meta_memory_mib|" output "meta_memory_gib: |$meta_memory_gib|" output "meta_memory_tib: |$meta_memory_tib|" -output "meta_memory_pib: |$meta_memory_pib|" \ No newline at end of file +output "meta_memory_pib: |$meta_memory_pib|" diff --git a/src/test/resources/test_languages/bash/config.vsh.yaml b/src/test/resources/test_languages/bash/config.vsh.yaml index 95e7563b2..e20dc008e 100755 --- a/src/test/resources/test_languages/bash/config.vsh.yaml +++ b/src/test/resources/test_languages/bash/config.vsh.yaml @@ -2,11 +2,16 @@ __merge__: [., ../common.yaml, ../common-runners.yaml] name: test_languages_bash resources: - type: bash_script + use_jq: true path: ./code.sh engines: - type: native - type: docker image: "bash:3.2" + setup: + - type: apk + packages: + - jq - type: docker image: "bash:3.2" id: "throwawayimage" @@ -14,4 +19,5 @@ engines: setup: - type: apk packages: + - jq - fortune diff --git a/src/test/resources/test_languages/common.yaml b/src/test/resources/test_languages/common.yaml index 739270ef6..23fe44e0b 100644 --- a/src/test/resources/test_languages/common.yaml +++ b/src/test/resources/test_languages/common.yaml @@ -68,6 +68,7 @@ resources: - path: ../resource1.txt test_resources: - type: bash_script + use_jq: true path: ../test.sh - path: ../resource2.txt info: diff --git a/src/test/resources/test_languages/csharp/config.vsh.yaml b/src/test/resources/test_languages/csharp/config.vsh.yaml index de2e2278e..c162abef8 100644 --- a/src/test/resources/test_languages/csharp/config.vsh.yaml +++ b/src/test/resources/test_languages/csharp/config.vsh.yaml @@ -9,4 +9,4 @@ engines: image: ghcr.io/data-intuitive/dotnet-script:1.3.1 setup: - type: apk - packages: [ bash ] + packages: [ bash, jq ] diff --git a/src/test/resources/test_languages/csharp/script.csx b/src/test/resources/test_languages/csharp/script.csx index 9694a9bc8..fcabd0410 100644 --- a/src/test/resources/test_languages/csharp/script.csx +++ b/src/test/resources/test_languages/csharp/script.csx @@ -72,7 +72,7 @@ try var array = p.GetValue(par) as Array; if (array.Length == 0) - Output($"{p.Name}: |empty array|"); + Output($"{p.Name}: ||"); else if (array is bool[]) { var array2 = (array as bool[]).Select(x => x.ToString().ToLower()); @@ -116,7 +116,7 @@ try { Output($"head of input: |{input.ReadLine()}|"); } - using(StreamReader input = new StreamReader("resource1.txt")) + using(StreamReader input = new StreamReader($"{meta.resources_dir}/resource1.txt")) { Output($"head of resource1: |{input.ReadLine()}|"); } @@ -126,15 +126,20 @@ try foreach (PropertyInfo p in pi) { - if (p.PropertyType.IsArray) + var value = p.GetValue(meta); + if (value == null) + { + Output($"meta_{p.Name}: ||"); + } + else if (p.PropertyType.IsArray) { - object[] array = (object[])p.GetValue(meta); + object[] array = (object[])value; Output($"meta_{p.Name}: |{string.Join(", ", array)}|"); } else { - Output($"meta_{p.Name}: |{p.GetValue(meta)}|"); + Output($"meta_{p.Name}: |{value}|"); } } } diff --git a/src/test/resources/test_languages/executable/config.vsh.yaml b/src/test/resources/test_languages/executable/config.vsh.yaml index 6dfe4f417..ba0ff1d00 100644 --- a/src/test/resources/test_languages/executable/config.vsh.yaml +++ b/src/test/resources/test_languages/executable/config.vsh.yaml @@ -1,4 +1,4 @@ -name: testexecutable +name: test_languages_scala description: | List buckets and objects with mc. arguments: @@ -13,10 +13,16 @@ resources: - path: ../resource1.txt test_resources: - type: bash_script + use_jq: true path: test.sh - path: ../resource2.txt -__merge__: [../common-runners.yaml] +runners: + - type: executable engines: - type: native - type: docker - image: "bash" + image: "bash:3.2" + setup: + - type: apk + packages: + - jq diff --git a/src/test/resources/test_languages/executable/test.sh b/src/test/resources/test_languages/executable/test.sh index 30a54d0e1..419106abe 100755 --- a/src/test/resources/test_languages/executable/test.sh +++ b/src/test/resources/test_languages/executable/test.sh @@ -2,10 +2,10 @@ set -ex echo ">>> Checking whether output is correct" -./testexecutable . > output.txt 2>&1 +"$meta_executable" . > output.txt 2>&1 [[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 -grep -q 'testexecutable' output.txt +grep -q 'test_languages_scala' output.txt grep -q 'resource1.txt' output.txt grep -q 'resource2.txt' output.txt diff --git a/src/test/resources/test_languages/js/code.js b/src/test/resources/test_languages/js/code.js index 09252c501..50b83eb2c 100644 --- a/src/test/resources/test_languages/js/code.js +++ b/src/test/resources/test_languages/js/code.js @@ -21,32 +21,28 @@ let meta = { const fs = require('fs') let logFun; -if (typeof par['log'] === 'undefined') { +if (!par['log']) { logFun = function(out) { - console.log("INFO:" + out); + console.log("INFO:" + out); } } else { logFun = function(out) { - fs.appendFile(par['log'], "INFO:" + out + "\n", function (err) { - if (err) throw err; - }); + fs.appendFileSync(par['log'], "INFO:" + out + "\n"); } } let outFun; -if (typeof par['output'] === 'undefined') { +if (!par['output']) { outFun = console.log } else { outFun = function(out) { - fs.appendFile(par['output'], out + "\n", function (err) { - if (err) throw err; - }); + fs.appendFileSync(par['output'], out + "\n"); } } // process parameters logFun('Parsed input arguments.') -if (typeof par['output'] === 'undefined') { +if (!par['output']) { logFun('Printing output to console') } else { logFun('Writing output to file') @@ -54,7 +50,7 @@ if (typeof par['output'] === 'undefined') { for (const key in par) { if (Array.isArray(par[key]) && par[key].length == 0) - outFun(`${key}: |empty array|`) + outFun(`${key}: ||`) else if (Array.isArray(par[key])) outFun(`${key}: |${par[key].join(';')}|`) else if (par[key] == undefined) @@ -63,12 +59,10 @@ for (const key in par) { outFun(`${key}: |${par[key]}|`) } -fs.readFile(par['input'], 'utf8', function(err, data){ - outFun(`head of input: |${data.split('\n')[0]}|`) -}) -fs.readFile(meta['resources_dir'] + '/resource1.txt', 'utf8', function(err, data){ - outFun(`head of resource1: |${data.split('\n')[0]}|`) -}) +let inputData = fs.readFileSync(par['input'], 'utf8'); +outFun(`head of input: |${inputData.split('\n')[0]}|`) +let resourceData = fs.readFileSync(meta['resources_dir'] + '/resource1.txt', 'utf8'); +outFun(`head of resource1: |${resourceData.split('\n')[0]}|`) for (const key in meta) { if (meta[key] == undefined || String(meta[key]) == 'NaN') diff --git a/src/test/resources/test_languages/js/config.vsh.yaml b/src/test/resources/test_languages/js/config.vsh.yaml index 90b46fef5..7331eb760 100644 --- a/src/test/resources/test_languages/js/config.vsh.yaml +++ b/src/test/resources/test_languages/js/config.vsh.yaml @@ -6,7 +6,10 @@ resources: engines: - type: native - type: docker - image: node:15-buster + image: node:24-bookworm setup: + - type: apt + packages: + - jq - type: javascript npm: [ plot ] diff --git a/src/test/resources/test_languages/python/code.py b/src/test/resources/test_languages/python/code.py index e7d0f3238..a6f94feb7 100644 --- a/src/test/resources/test_languages/python/code.py +++ b/src/test/resources/test_languages/python/code.py @@ -57,7 +57,10 @@ def echo(s): echo(f"{key}: |{value}|") for key, value in meta.items(): - echo(f"meta_{key}: |{value}|") + if value is None: + echo(f"meta_{key}: ||") + else: + echo(f"meta_{key}: |{value}|") with open(par['input'], 'r') as infile: input = infile.readline().strip() diff --git a/src/test/resources/test_languages/python/config.vsh.yaml b/src/test/resources/test_languages/python/config.vsh.yaml index 3f7d24283..a4b564af3 100644 --- a/src/test/resources/test_languages/python/config.vsh.yaml +++ b/src/test/resources/test_languages/python/config.vsh.yaml @@ -8,6 +8,9 @@ engines: - type: docker image: python setup: + - type: apt + packages: + - jq - type: docker build_args: - TESTING_FOO=bar diff --git a/src/test/resources/test_languages/r/script.vsh.R b/src/test/resources/test_languages/r/script.vsh.R index 67304dbd8..ab57c56e1 100644 --- a/src/test/resources/test_languages/r/script.vsh.R +++ b/src/test/resources/test_languages/r/script.vsh.R @@ -3,8 +3,11 @@ #' engines: #' - type: native #' - type: docker -#' image: rocker/tidyverse:3.6 +#' image: rocker/tidyverse:4.5 #' setup: +#' - type: apt +#' packages: +#' - jq #' - type: r #' cran: optparse #' github: tidyverse/glue@main @@ -46,15 +49,19 @@ str <- sapply(names(par), function(n) { }) write_fun(par$output, str) -con = file(par[["input"]], "r") -str = paste0("head of input: |", readLines(con, n = 1), "|\n") +con <- file(par[["input"]], "r") +str <- paste0("head of input: |", readLines(con, n = 1), "|\n") write_fun(par$output, str) -con = file("resource1.txt", "r") -str = paste0("head of resource1: |", readLines(con, n = 1), "|\n") +con <- file(paste0(meta$resources_dir, "/resource1.txt"), "r") +str <- paste0("head of resource1: |", readLines(con, n = 1), "|\n") write_fun(par$output, str) str <- sapply(names(meta), function(n) { - paste0("meta_", n, ": |", paste(meta[[n]], collapse = ";"), "|\n") + if (is.null(meta[[n]])) { + paste0("meta_", n, ": ||\n") + } else { + paste0("meta_", n, ": |", paste(meta[[n]], collapse = ";"), "|\n") + } }) write_fun(par$output, str) diff --git a/src/test/resources/test_languages/scala/config.vsh.yaml b/src/test/resources/test_languages/scala/config.vsh.yaml index 1da5787af..6ff9d5540 100644 --- a/src/test/resources/test_languages/scala/config.vsh.yaml +++ b/src/test/resources/test_languages/scala/config.vsh.yaml @@ -7,3 +7,7 @@ engines: - type: native - type: docker image: hseeberger/scala-sbt:eclipse-temurin-11.0.14.1_1.6.2_2.13.8 + setup: + - type: apt + packages: + - jq diff --git a/src/test/resources/test_languages/scala/script.scala b/src/test/resources/test_languages/scala/script.scala index 86f383d2b..c25842965 100644 --- a/src/test/resources/test_languages/scala/script.scala +++ b/src/test/resources/test_languages/scala/script.scala @@ -49,7 +49,7 @@ try { val input = Source.fromFile(par.input).getLines().toArray outputFun(s"head of input: |${input(0)}|") - val resource1 = Source.fromFile("resource1.txt").getLines().toArray + val resource1 = Source.fromFile(s"${meta.resources_dir}/resource1.txt").getLines().toArray outputFun(s"head of resource1: |${resource1(0)}|") } finally { diff --git a/src/test/resources/test_languages/test.sh b/src/test/resources/test_languages/test.sh index 26150d0a8..6d4163386 100755 --- a/src/test/resources/test_languages/test.sh +++ b/src/test/resources/test_languages/test.sh @@ -1,5 +1,28 @@ #!/usr/bin/env bash -set -ex +set -e + +## VIASH START +## VIASH END + +# Helper function to check output with verbose error messages +check_output() { + local pattern="$1" + local file="$2" + if ! grep -q "$pattern" "$file"; then + echo "FAILED: Pattern not found: $pattern" + echo "Actual content of relevant lines:" + # Try to show relevant lines by extracting the key name + local key=$(echo "$pattern" | sed 's/[:|].*//; s/.*|//') + grep "$key" "$file" || echo "(no lines matching '$key' found)" + echo "---" + return 1 + fi +} + +# Set up temporary directory and environment variables for the test +export VIASH_KEEP_WORK_DIR=silent +export VIASH_TEMP=$meta_temp_dir/temp +mkdir -p $VIASH_TEMP echo ">>> Checking whether expected resources exist" [[ ! -f "$meta_executable" ]] && echo "executable could not be found!" && exit 1 @@ -17,37 +40,37 @@ echo ">>> Checking whether output is correct" --long_number 112589990684262400 [[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 -grep -q 'input: |resource1.txt|' output.txt -grep -q 'real_number: |10.5|' output.txt -grep -q 'whole_number: |10|' output.txt -grep -q 'long_number: |112589990684262400|' output.txt -grep -q 's: |a string with spaces|' output.txt -grep -q 'truth: |true|' output.txt -grep -q 'falsehood: |false|' output.txt -grep -q 'reality: |true|' output.txt -grep -q 'output: |.*/output.txt|' output.txt -grep -q 'log: |.*/log.txt|' output.txt -grep -q 'optional: |foo|' output.txt -grep -q 'optional_with_default: |bar|' output.txt -grep -q 'multiple: |one;two|' output.txt -grep -q 'multiple_pos: |a;b;c;d;e;f|' output.txt -grep -q 'meta_name: |test_languages_.*|' output.txt -grep -q 'meta_resources_dir: |..*|' output.txt -grep -q 'meta_cpus: |2|' output.txt -grep -q 'meta_memory_b: |2000000000|' output.txt -grep -q 'meta_memory_kb: |2000000|' output.txt -grep -q 'meta_memory_mb: |2000|' output.txt -grep -q 'meta_memory_gb: |2|' output.txt -grep -q 'meta_memory_tb: |1|' output.txt -grep -q 'meta_memory_pb: |1|' output.txt -grep -q 'meta_memory_kib: |1953125|' output.txt -grep -q 'meta_memory_mib: |1908|' output.txt -grep -q 'meta_memory_gib: |2|' output.txt -grep -q 'meta_memory_tib: |1|' output.txt -grep -q 'meta_memory_pib: |1|' output.txt - -grep -q 'head of input: |if you can read this,|' output.txt -grep -q 'head of resource1: |if you can read this,|' output.txt +check_output 'input: |resource1.txt|' output.txt +check_output 'real_number: |10.5|' output.txt +check_output 'whole_number: |10|' output.txt +check_output 'long_number: |112589990684262400|' output.txt +check_output 's: |a string with spaces|' output.txt +check_output 'truth: |true|' output.txt +check_output 'falsehood: |false|' output.txt +check_output 'reality: |true|' output.txt +check_output 'output: |.*/output.txt|' output.txt +check_output 'log: |.*/log.txt|' output.txt +check_output 'optional: |foo|' output.txt +check_output 'optional_with_default: |bar|' output.txt +check_output 'multiple: |one;two|' output.txt +check_output 'multiple_pos: |a;b;c;d;e;f|' output.txt +check_output 'meta_name: |test_languages_.*|' output.txt +check_output 'meta_resources_dir: |..*|' output.txt +check_output 'meta_cpus: |2|' output.txt +check_output 'meta_memory_b: |2000000000|' output.txt +check_output 'meta_memory_kb: |2000000|' output.txt +check_output 'meta_memory_mb: |2000|' output.txt +check_output 'meta_memory_gb: |2|' output.txt +check_output 'meta_memory_tb: |1|' output.txt +check_output 'meta_memory_pb: |1|' output.txt +check_output 'meta_memory_kib: |1953125|' output.txt +check_output 'meta_memory_mib: |1908|' output.txt +check_output 'meta_memory_gib: |2|' output.txt +check_output 'meta_memory_tib: |1|' output.txt +check_output 'meta_memory_pib: |1|' output.txt + +check_output 'head of input: |if you can read this,|' output.txt +check_output 'head of resource1: |if you can read this,|' output.txt [[ ! -f log.txt ]] && echo "Log file could not be found!" && exit 1 grep -q 'Parsed input arguments.' log.txt @@ -63,38 +86,38 @@ echo ">>> Checking whether output is correct with minimal parameters" > output2.txt [[ ! -f output2.txt ]] && echo "Output file could not be found!" && exit 1 -grep -q 'input: |resource2.txt|' output2.txt -grep -q 'real_number: |123.456|' output2.txt -grep -q 'whole_number: |789|' output2.txt -grep -q 'long_number: ||' output2.txt -grep -q "s: |a \\\\ b \\\$ c \` d \" e ' f \\\\n g # h @ i { j } k \"\"\" l ''' m todo_add_back_DOLLAR_VIASH_TEMP n : o ; p|" output2.txt -grep -q 'truth: |false|' output2.txt -grep -q 'falsehood: |true|' output2.txt -grep -q 'reality: ||' output2.txt -grep -q 'output: ||' output2.txt -grep -q 'log: ||' output2.txt -grep -q 'optional: ||' output2.txt -grep -q 'optional_with_default: |The default value.|' output2.txt -grep -q 'multiple: ||' output2.txt -grep -q 'multiple_pos: ||' output2.txt - -grep -q 'meta_name: |test_languages_.*|' output2.txt -grep -q 'meta_resources_dir: |..*|' output2.txt -grep -q 'meta_cpus: |666|' output2.txt -grep -q 'meta_memory_b: |100000000000000000|' output2.txt -grep -q 'meta_memory_kb: |100000000000000|' output2.txt -grep -q 'meta_memory_mb: |100000000000|' output2.txt -grep -q 'meta_memory_gb: |100000000|' output2.txt -grep -q 'meta_memory_tb: |100000|' output2.txt -grep -q 'meta_memory_pb: |100|' output2.txt -grep -q 'meta_memory_kib: |97656250000000|' output2.txt -grep -q 'meta_memory_mib: |95367431641|' output2.txt -grep -q 'meta_memory_gib: |93132258|' output2.txt -grep -q 'meta_memory_tib: |90950|' output2.txt -grep -q 'meta_memory_pib: |89|' output2.txt - -grep -q 'head of input: |this file is only for testing|' output2.txt -grep -q 'head of resource1: |if you can read this,|' output2.txt +check_output 'input: |resource2.txt|' output2.txt +check_output 'real_number: |123.456|' output2.txt +check_output 'whole_number: |789|' output2.txt +check_output 'long_number: ||' output2.txt +check_output "s: |a \\\\ b \\\$ c \` d \" e ' f \\\\n g # h @ i { j } k \"\"\" l ''' m todo_add_back_DOLLAR_VIASH_TEMP n : o ; p|" output2.txt +check_output 'truth: |false|' output2.txt +check_output 'falsehood: |true|' output2.txt +check_output 'reality: ||' output2.txt +check_output 'output: ||' output2.txt +check_output 'log: ||' output2.txt +check_output 'optional: ||' output2.txt +check_output 'optional_with_default: |The default value.|' output2.txt +check_output 'multiple: ||' output2.txt +check_output 'multiple_pos: ||' output2.txt + +check_output 'meta_name: |test_languages_.*|' output2.txt +check_output 'meta_resources_dir: |..*|' output2.txt +check_output 'meta_cpus: |666|' output2.txt +check_output 'meta_memory_b: |100000000000000000|' output2.txt +check_output 'meta_memory_kb: |100000000000000|' output2.txt +check_output 'meta_memory_mb: |100000000000|' output2.txt +check_output 'meta_memory_gb: |100000000|' output2.txt +check_output 'meta_memory_tb: |100000|' output2.txt +check_output 'meta_memory_pb: |100|' output2.txt +check_output 'meta_memory_kib: |97656250000000|' output2.txt +check_output 'meta_memory_mib: |95367431641|' output2.txt +check_output 'meta_memory_gib: |93132258|' output2.txt +check_output 'meta_memory_tib: |90950|' output2.txt +check_output 'meta_memory_pib: |89|' output2.txt + +check_output 'head of input: |this file is only for testing|' output2.txt +check_output 'head of resource1: |if you can read this,|' output2.txt echo ">>> Checking whether output is correct with minimal parameters, but with 1024-base memory" @@ -107,40 +130,50 @@ echo ">>> Checking whether output is correct with minimal parameters, but with 1 ---memory 100PiB \ > output2.txt -grep -q 'meta_memory_b: |112589990684262400|' output2.txt -grep -q 'meta_memory_kb: |112589990684263|' output2.txt -grep -q 'meta_memory_mb: |112589990685|' output2.txt -grep -q 'meta_memory_gb: |112589991|' output2.txt -grep -q 'meta_memory_tb: |112590|' output2.txt -grep -q 'meta_memory_pb: |113|' output2.txt -grep -q 'meta_memory_kib: |109951162777600|' output2.txt -grep -q 'meta_memory_mib: |107374182400|' output2.txt -grep -q 'meta_memory_gib: |104857600|' output2.txt -grep -q 'meta_memory_tib: |102400|' output2.txt -grep -q 'meta_memory_pib: |100|' output2.txt - -if [[ $meta_name == "bash" || $meta_name == "js" ]]; then -# This currently only works fully on bash and javascript - - echo ">>> Try to unset defaults" - "$meta_executable" \ - "resource2.txt" \ - --real_number 123.456 \ - --whole_number=789 \ - -s "my\$weird#string\"\"\"" \ - ---cpus "" \ - ---memory "" \ - > output4.txt - - [[ ! -f output4.txt ]] && echo "Output file could not be found!" && exit 1 - grep -q 'meta_cpus: ||' output4.txt - grep -q 'meta_memory_b: ||' output4.txt - grep -q 'meta_memory_kb: ||' output4.txt - grep -q 'meta_memory_mb: ||' output4.txt - grep -q 'meta_memory_gb: ||' output4.txt - grep -q 'meta_memory_tb: ||' output4.txt - grep -q 'meta_memory_pb: ||' output4.txt - -fi +check_output 'meta_memory_b: |112589990684262400|' output2.txt +check_output 'meta_memory_kb: |112589990684263|' output2.txt +check_output 'meta_memory_mb: |112589990685|' output2.txt +check_output 'meta_memory_gb: |112589991|' output2.txt +check_output 'meta_memory_tb: |112590|' output2.txt +check_output 'meta_memory_pb: |113|' output2.txt +check_output 'meta_memory_kib: |109951162777600|' output2.txt +check_output 'meta_memory_mib: |107374182400|' output2.txt +check_output 'meta_memory_gib: |104857600|' output2.txt +check_output 'meta_memory_tib: |102400|' output2.txt +check_output 'meta_memory_pib: |100|' output2.txt + +# Test unsetting defaults using UNDEFINED (issue #375) +# With JSON-based parameter passing, this now works in all languages +echo ">>> Try to unset defaults" +"$meta_executable" \ + "resource2.txt" \ + --real_number 123.456 \ + --whole_number=789 \ + -s "my\$weird#string\"\"\"" \ + ---cpus UNDEFINED \ + ---memory UNDEFINED \ + > output4.txt + +[[ ! -f output4.txt ]] && echo "Output file could not be found!" && exit 1 +check_output 'meta_cpus: ||' output4.txt +check_output 'meta_memory_b: ||' output4.txt +check_output 'meta_memory_kb: ||' output4.txt +check_output 'meta_memory_mb: ||' output4.txt +check_output 'meta_memory_gb: ||' output4.txt +check_output 'meta_memory_tb: ||' output4.txt +check_output 'meta_memory_pb: ||' output4.txt + +# Test backslash-quote escaping (issue #821) +# The sequence \' was previously breaking Python syntax +echo ">>> Test backslash-quote escaping" +"$meta_executable" \ + "resource2.txt" \ + --real_number 123.456 \ + --whole_number=789 \ + -s "test\\'value" \ + > output5.txt + +[[ ! -f output5.txt ]] && echo "Output file could not be found!" && exit 1 +check_output "s: |test\\\\'value|" output5.txt echo ">>> Test finished successfully" diff --git a/src/test/resources/testbash/auxiliary_requirements/config_requirements.vsh.yaml b/src/test/resources/testbash/auxiliary_requirements/config_requirements.vsh.yaml index 6b5a00efc..022c25f7b 100755 --- a/src/test/resources/testbash/auxiliary_requirements/config_requirements.vsh.yaml +++ b/src/test/resources/testbash/auxiliary_requirements/config_requirements.vsh.yaml @@ -14,11 +14,17 @@ arguments: resources: - type: bash_script + use_jq: true path: ./check_requirements.sh test_resources: - type: bash_script + use_jq: true path: ./check_requirements_test.sh engines: - type: docker image: "bash:3.2" + setup: + - type: apk + packages: + - jq diff --git a/src/test/resources/testbash/auxiliary_requirements/parameter_check.vsh.yaml b/src/test/resources/testbash/auxiliary_requirements/parameter_check.vsh.yaml index bc3099258..da43c705c 100755 --- a/src/test/resources/testbash/auxiliary_requirements/parameter_check.vsh.yaml +++ b/src/test/resources/testbash/auxiliary_requirements/parameter_check.vsh.yaml @@ -162,6 +162,7 @@ arguments: resources: - type: bash_script + use_jq: true path: ./parameter_check.sh info: foo: bar diff --git a/src/test/resources/testbash/auxiliary_requirements/parameter_check_loop.vsh.yaml b/src/test/resources/testbash/auxiliary_requirements/parameter_check_loop.vsh.yaml index 7de81ba79..7a33baddb 100644 --- a/src/test/resources/testbash/auxiliary_requirements/parameter_check_loop.vsh.yaml +++ b/src/test/resources/testbash/auxiliary_requirements/parameter_check_loop.vsh.yaml @@ -18,6 +18,7 @@ arguments: required: true resources: - type: bash_script + use_jq: true text: | while IFS= read -r line do diff --git a/src/test/resources/testbash/auxiliary_resource/code.sh b/src/test/resources/testbash/auxiliary_resource/code.sh index 334df0fd4..d1864c927 100755 --- a/src/test/resources/testbash/auxiliary_resource/code.sh +++ b/src/test/resources/testbash/auxiliary_resource/code.sh @@ -31,6 +31,21 @@ function output { fi } +# Helper function to join array elements with semicolon +function join_by_semicolon { + local varname=$1 + # Check if variable is an array + if declare -p "$varname" 2>/dev/null | grep -q '^declare -a'; then + local arr_name="${varname}[@]" + local arr=("${!arr_name}") + local IFS=";" + echo "${arr[*]}" + else + # Not an array, just echo the value + echo "${!varname}" + fi +} + log "INFO: Parsed input arguments." if [ -z "$par_output" ]; then @@ -55,5 +70,5 @@ INPUT=`head -1 "$par_input"` output "head of input: |$INPUT|" RESOURCE=`head -1 "$meta_resources_dir/resource1.txt"` output "head of resource1: |$RESOURCE|" -output "multiple: |$par_multiple|" -output "multiple_pos: |$par_multiple_pos|" +output "multiple: |$(join_by_semicolon par_multiple)|" +output "multiple_pos: |$(join_by_semicolon par_multiple_pos)|" diff --git a/src/test/resources/testbash/auxiliary_resource/config_resource_test.vsh.yaml b/src/test/resources/testbash/auxiliary_resource/config_resource_test.vsh.yaml index 3036b8dc0..ddbfe4f2f 100755 --- a/src/test/resources/testbash/auxiliary_resource/config_resource_test.vsh.yaml +++ b/src/test/resources/testbash/auxiliary_resource/config_resource_test.vsh.yaml @@ -3,8 +3,10 @@ description: | Test various ways of specifying resources and check they ended up being in the right place. resources: - type: bash_script + use_jq: true path: ./check_bash_version.sh - type: bash_script + use_jq: true path: ./code.sh - path: resource1.txt - path: ./resource2.txt @@ -38,3 +40,7 @@ engines: - type: native - type: docker image: "bash:3.2" + setup: + - type: apk + packages: + - jq diff --git a/src/test/resources/testbash/auxiliary_tag/config_bash_tag.vsh.yaml b/src/test/resources/testbash/auxiliary_tag/config_bash_tag.vsh.yaml index 638f9bcc6..8146720c4 100755 --- a/src/test/resources/testbash/auxiliary_tag/config_bash_tag.vsh.yaml +++ b/src/test/resources/testbash/auxiliary_tag/config_bash_tag.vsh.yaml @@ -3,6 +3,7 @@ description: | Prints out version of bash. resources: - type: bash_script + use_jq: true path: ./check_bash_version.sh arguments: - name: "--optional" @@ -12,13 +13,24 @@ engines: - id: "testtag1" type: docker image: "bash:5.0" + setup: + - type: apk + packages: + - jq - id: "testtag2" type: docker image: "bash:3.2" + setup: + - type: apk + packages: + - jq - id: "testtargetimage1" type: docker image: "bash:5.0" setup: + - type: apk + packages: + - jq - type: docker run: [ ls ] - id: "testtargetimage2" @@ -28,6 +40,9 @@ engines: target_registry: foo.io target_tag: 0.0.1 setup: + - type: apk + packages: + - jq - type: docker run: [ ls ] - id: "testtargetimage3" @@ -36,6 +51,9 @@ engines: target_image: bar target_tag: 0.0.2 setup: + - type: apk + packages: + - jq - type: docker run: [ ls ] diff --git a/src/test/resources/testbash/check_computational_requirements.vsh.yaml b/src/test/resources/testbash/check_computational_requirements.vsh.yaml index a21668ab4..32bc6d8d8 100755 --- a/src/test/resources/testbash/check_computational_requirements.vsh.yaml +++ b/src/test/resources/testbash/check_computational_requirements.vsh.yaml @@ -5,6 +5,7 @@ description: | Check setting of computational requirements resources: - type: bash_script + use_jq: true text: | if [ -n "$meta_cpus" ]; then echo "cpus: $meta_cpus" @@ -19,14 +20,15 @@ resources: fi test_resources: - type: bash_script + use_jq: true path: "test_script.sh" text: | - if [ -n "$VIASH_META_CPUS" ] && [ -n "$VIASH_META_MEMORY" ]; then - VIASH_META_CPUS=$VIASH_META_CPUS VIASH_META_MEMORY=$VIASH_META_MEMORY "$meta_executable" - elif [ -z "$VIASH_META_CPUS" ] && [ -n "$VIASH_META_MEMORY" ]; then - VIASH_META_MEMORY=$VIASH_META_MEMORY "$meta_executable" - elif [ -n "$VIASH_META_CPUS" ] && [ -z "$VIASH_META_MEMORY" ]; then - VIASH_META_CPUS=$VIASH_META_CPUS "$meta_executable" + if [ -n "$meta_cpus" ] && [ -n "$meta_memory_mb" ]; then + VIASH_META_CPUS=$meta_cpus VIASH_META_MEMORY=${meta_memory_mb}MB "$meta_executable" + elif [ -z "$meta_cpus" ] && [ -n "$meta_memory_mb" ]; then + VIASH_META_MEMORY=${meta_memory_mb}MB "$meta_executable" + elif [ -n "$meta_cpus" ] && [ -z "$meta_memory_mb" ]; then + VIASH_META_CPUS=$meta_cpus "$meta_executable" else "$meta_executable" fi diff --git a/src/test/resources/testbash/code.sh b/src/test/resources/testbash/code.sh index e45a31077..2e18f932a 100755 --- a/src/test/resources/testbash/code.sh +++ b/src/test/resources/testbash/code.sh @@ -33,6 +33,21 @@ function output { fi } +# Helper function to join array elements with semicolon +function join_by_semicolon { + local varname=$1 + # Check if variable is an array + if declare -p "$varname" 2>/dev/null | grep -q '^declare -a'; then + local arr_name="${varname}[@]" + local arr=("${!arr_name}") + local IFS=";" + echo "${arr[*]}" + else + # Not an array, just echo the value + echo "${!varname}" + fi +} + log "INFO: Parsed input arguments." if [ -z "$par_output" ]; then @@ -42,7 +57,7 @@ else fi output "input: |$par_input|" -output "real_number: |$par_real_number|" +output "real_number: |$(join_by_semicolon par_real_number)|" output "whole_number: |$par_whole_number|" output "long_number: |$par_long_number|" output "s: |$par_s|" @@ -53,8 +68,8 @@ output "output: |$par_output|" output "log: |$par_log|" output "optional: |$par_optional|" output "optional_with_default: |$par_optional_with_default|" -output "multiple: |$par_multiple|" -output "multiple_pos: |$par_multiple_pos|" +output "multiple: |$(join_by_semicolon par_multiple)|" +output "multiple_pos: |$(join_by_semicolon par_multiple_pos)|" output "multiple_output: |$par_multiple_output|" output "meta_resources_dir: |$meta_resources_dir|" diff --git a/src/test/resources/testbash/config.vsh.yaml b/src/test/resources/testbash/config.vsh.yaml index a55f679cd..d88feaf76 100755 --- a/src/test/resources/testbash/config.vsh.yaml +++ b/src/test/resources/testbash/config.vsh.yaml @@ -78,13 +78,16 @@ argument_groups: direction: output resources: - type: bash_script + use_jq: true path: ./code.sh - path: resource1.txt - path: https://raw.githubusercontent.com/scala/scala/fff4ec3539ac58f56fdc8f1382c365f32a9fd25a/NOTICE test_resources: - type: bash_script + use_jq: true path: tests/check_outputs.sh - type: bash_script + use_jq: true path: tests/fail.sh - path: resource2.txt info: @@ -101,6 +104,10 @@ engines: id: native - type: docker image: "bash:3.2" + setup: + - type: apk + packages: + - jq - type: docker id: throwawayimage image: "bash:3.2" @@ -108,4 +115,5 @@ engines: setup: - type: apk packages: + - jq - fortune diff --git a/src/test/resources/testbash/docker_options/code.sh b/src/test/resources/testbash/docker_options/code.sh index 9cf1c2e1d..7ff4297e9 100755 --- a/src/test/resources/testbash/docker_options/code.sh +++ b/src/test/resources/testbash/docker_options/code.sh @@ -32,6 +32,21 @@ function output { fi } +# Helper function to join array elements with semicolon +function join_by_semicolon { + local varname=$1 + # Check if variable is an array + if declare -p "$varname" 2>/dev/null | grep -q '^declare -a'; then + local arr_name="${varname}[@]" + local arr=("${!arr_name}") + local IFS=";" + echo "${arr[*]}" + else + # Not an array, just echo the value + echo "${!varname}" + fi +} + log "INFO: Parsed input arguments." if [ -z "$par_output" ]; then @@ -56,5 +71,5 @@ INPUT=`head -1 "$par_input"` output "head of input: |$INPUT|" RESOURCE=`head -1 "$meta_resources_dir/resource1.txt"` output "head of resource1: |$RESOURCE|" -output "multiple: |$par_multiple|" -output "multiple_pos: |$par_multiple_pos|" +output "multiple: |$(join_by_semicolon par_multiple)|" +output "multiple_pos: |$(join_by_semicolon par_multiple_pos)|" diff --git a/src/test/resources/testbash/tests/check_outputs.sh b/src/test/resources/testbash/tests/check_outputs.sh index c7484b0d8..b297507c0 100755 --- a/src/test/resources/testbash/tests/check_outputs.sh +++ b/src/test/resources/testbash/tests/check_outputs.sh @@ -1,6 +1,21 @@ #!/usr/bin/env bash set -ex +# Helper function to check output with verbose error messages +check_output() { + local pattern="$1" + local file="$2" + if ! grep -q "$pattern" "$file"; then + echo "FAILED: Pattern not found: $pattern" + echo "Actual content of relevant lines:" + # Try to show relevant lines by extracting the key name + local key=$(echo "$pattern" | sed 's/[:|].*//; s/.*|//') + grep "$key" "$file" || echo "(no lines matching '$key' found)" + echo "---" + return 1 + fi +} + echo ">>> Checking whether expected resources exist" [[ ! -f "$meta_executable" ]] && echo "executable could not be found!" && exit 1 [[ ! -f "$meta_resources_dir/.config.vsh.yaml" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 @@ -18,20 +33,20 @@ echo ">>> Checking whether output is correct" --multiple_output './output_*.txt' [[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 -grep -q 'input: |NOTICE|' output.txt -grep -q 'real_number: |10.5|' output.txt -grep -q 'whole_number: |10|' output.txt -grep -q 'long_number: |112589990684262400|' output.txt -grep -q 's: |a string with spaces|' output.txt -grep -q 'truth: |true|' output.txt -grep -q 'falsehood: |false|' output.txt -grep -q 'reality: |true|' output.txt -grep -q 'output: |.*/output.txt|' output.txt -grep -q 'log: |.*/log.txt|' output.txt -grep -q 'optional: |foo|' output.txt -grep -q 'optional_with_default: |bar|' output.txt -grep -q 'multiple: |one;two|' output.txt -grep -q 'multiple_pos: |a;b;c;d;e;f|' output.txt -grep -q 'multiple_output: |.*/output_\*.txt|' output.txt +check_output 'input: |NOTICE|' output.txt +check_output 'real_number: |10.5|' output.txt +check_output 'whole_number: |10|' output.txt +check_output 'long_number: |112589990684262400|' output.txt +check_output 's: |a string with spaces|' output.txt +check_output 'truth: |true|' output.txt +check_output 'falsehood: |false|' output.txt +check_output 'reality: |true|' output.txt +check_output 'output: |.*/output.txt|' output.txt +check_output 'log: |.*/log.txt|' output.txt +check_output 'optional: |foo|' output.txt +check_output 'optional_with_default: |bar|' output.txt +check_output 'multiple: |one;two|' output.txt +check_output 'multiple_pos: |a;b;c;d;e;f|' output.txt +check_output 'multiple_output: |.*/output_\*.txt|' output.txt echo ">>> Test finished successfully" diff --git a/src/test/resources/testbash/tests/fail.sh b/src/test/resources/testbash/tests/fail.sh index f6857a070..11b57417e 100755 --- a/src/test/resources/testbash/tests/fail.sh +++ b/src/test/resources/testbash/tests/fail.sh @@ -2,7 +2,7 @@ set -x echo ">>> This test should always fail but that is to be expected" -./testbash "missingresource" --real_number abc --whole_number abc -s "a string with spaces" --truth \ +"$meta_executable" "missingresource" --real_number abc --whole_number abc -s "a string with spaces" --truth \ --output ./output_fail.txt --log ./log.txt \ --optional foo --optional_with_default bar diff --git a/src/test/resources/testnextflowvdsl3/src/integer_as_double/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/integer_as_double/config.vsh.yaml index b18a81bc2..317cb699a 100644 --- a/src/test/resources/testnextflowvdsl3/src/integer_as_double/config.vsh.yaml +++ b/src/test/resources/testnextflowvdsl3/src/integer_as_double/config.vsh.yaml @@ -16,11 +16,16 @@ arguments: example: 10.5 resources: - type: bash_script + use_jq: true path: script.sh engines: - type: native - type: docker image: nextflow/bash:latest + setup: + - type: apk + packages: + - jq runners: - type: executable - type: nextflow diff --git a/src/test/resources/testnextflowvdsl3/src/multiple_output/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/multiple_output/config.vsh.yaml index a552fa0d0..255614e47 100644 --- a/src/test/resources/testnextflowvdsl3/src/multiple_output/config.vsh.yaml +++ b/src/test/resources/testnextflowvdsl3/src/multiple_output/config.vsh.yaml @@ -15,11 +15,16 @@ arguments: example: output_*.txt resources: - type: bash_script + use_jq: true path: script.sh engines: - type: native - type: docker image: nextflow/bash:latest + setup: + - type: apk + packages: + - jq runners: - type: executable - type: nextflow diff --git a/src/test/resources/testnextflowvdsl3/src/multiple_output/script.sh b/src/test/resources/testnextflowvdsl3/src/multiple_output/script.sh index c9ffcea7a..778d83f6e 100644 --- a/src/test/resources/testnextflowvdsl3/src/multiple_output/script.sh +++ b/src/test/resources/testnextflowvdsl3/src/multiple_output/script.sh @@ -7,10 +7,8 @@ par_output='output_*.txt' output_i=0 -if [ ! -z "$par_input" ]; then - IFS=";" - for var in $par_input; do - unset IFS +if [ ${#par_input[@]} -gt 0 ]; then + for var in "${par_input[@]}"; do output=$(echo "$par_output" | sed "s/\*/$output_i/g") cp "$var" "$output" output_i=$((output_i+1)) diff --git a/src/test/resources/testnextflowvdsl3/src/step1/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/step1/config.vsh.yaml index 38085ad82..619063ce5 100644 --- a/src/test/resources/testnextflowvdsl3/src/step1/config.vsh.yaml +++ b/src/test/resources/testnextflowvdsl3/src/step1/config.vsh.yaml @@ -12,10 +12,15 @@ arguments: example: output.txt resources: - type: bash_script + use_jq: true path: script.sh engines: - type: native - type: docker image: nextflow/bash:latest + setup: + - type: apk + packages: + - jq runners: - type: nextflow diff --git a/src/test/resources/testnextflowvdsl3/src/step1/script.sh b/src/test/resources/testnextflowvdsl3/src/step1/script.sh index 0bb5fa28f..40995c331 100644 --- a/src/test/resources/testnextflowvdsl3/src/step1/script.sh +++ b/src/test/resources/testnextflowvdsl3/src/step1/script.sh @@ -2,10 +2,8 @@ [ -f "$par_output" ] && rm "$par_output" -if [ ! -z "$par_input" ]; then - IFS=";" - for var in $par_input; do - unset IFS +if [ ${#par_input[@]} -gt 0 ]; then + for var in "${par_input[@]}"; do cat "$var" >> "$par_output" done fi diff --git a/src/test/resources/testnextflowvdsl3/src/step2/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/step2/config.vsh.yaml index d49c22587..58e668077 100644 --- a/src/test/resources/testnextflowvdsl3/src/step2/config.vsh.yaml +++ b/src/test/resources/testnextflowvdsl3/src/step2/config.vsh.yaml @@ -54,10 +54,15 @@ arguments: \n\r\t\s\ resources: - type: bash_script + use_jq: true path: script.sh engines: - type: native - type: docker image: nextflow/bash:latest + setup: + - type: apk + packages: + - jq runners: - type: nextflow diff --git a/src/test/resources/testnextflowvdsl3/src/step3/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/step3/config.vsh.yaml index d037fb923..1781be94b 100644 --- a/src/test/resources/testnextflowvdsl3/src/step3/config.vsh.yaml +++ b/src/test/resources/testnextflowvdsl3/src/step3/config.vsh.yaml @@ -12,10 +12,15 @@ arguments: example: output.txt resources: - type: bash_script + use_jq: true path: script.sh engines: - type: native - type: docker image: nextflow/bash:latest + setup: + - type: apk + packages: + - jq runners: - type: nextflow diff --git a/src/test/resources/testnextflowvdsl3/src/step3/script.sh b/src/test/resources/testnextflowvdsl3/src/step3/script.sh index b5641a18e..da14768f0 100644 --- a/src/test/resources/testnextflowvdsl3/src/step3/script.sh +++ b/src/test/resources/testnextflowvdsl3/src/step3/script.sh @@ -7,10 +7,8 @@ function clean_up { trap clean_up EXIT -if [ ! -z "$par_input" ]; then - IFS=";" - for var in $par_input; do - unset IFS +if [ ${#par_input[@]} -gt 0 ]; then + for var in "${par_input[@]}"; do cat "$var" >> "$tmpfile" done fi diff --git a/src/test/resources/testns/src/ns_add/config.vsh.yaml b/src/test/resources/testns/src/ns_add/config.vsh.yaml index 8b6435550..c37549f0a 100755 --- a/src/test/resources/testns/src/ns_add/config.vsh.yaml +++ b/src/test/resources/testns/src/ns_add/config.vsh.yaml @@ -21,6 +21,7 @@ resources: path: code.py test_resources: - type: bash_script + use_jq: true path: test.sh engines: - type: native diff --git a/src/test/resources/testns/src/ns_disabled/config.vsh.yaml b/src/test/resources/testns/src/ns_disabled/config.vsh.yaml index 9efcf3421..ccf3ae4e1 100755 --- a/src/test/resources/testns/src/ns_disabled/config.vsh.yaml +++ b/src/test/resources/testns/src/ns_disabled/config.vsh.yaml @@ -22,6 +22,7 @@ resources: path: code.py test_resources: - type: bash_script + use_jq: true path: test.sh engines: - type: native diff --git a/src/test/resources/testns/src/ns_divide/config.vsh.yaml b/src/test/resources/testns/src/ns_divide/config.vsh.yaml index 25b21e92d..27152d71d 100755 --- a/src/test/resources/testns/src/ns_divide/config.vsh.yaml +++ b/src/test/resources/testns/src/ns_divide/config.vsh.yaml @@ -21,8 +21,10 @@ resources: path: code.py test_resources: - type: bash_script + use_jq: true path: test.sh - type: bash_script + use_jq: true path: test_div0.sh engines: - type: native diff --git a/src/test/resources/testns/src/ns_multiply/config.vsh.yaml b/src/test/resources/testns/src/ns_multiply/config.vsh.yaml index bf72e3e40..f5acb0e01 100755 --- a/src/test/resources/testns/src/ns_multiply/config.vsh.yaml +++ b/src/test/resources/testns/src/ns_multiply/config.vsh.yaml @@ -21,6 +21,7 @@ resources: path: code.py test_resources: - type: bash_script + use_jq: true path: test.sh engines: - type: native diff --git a/src/test/resources/testns/src/ns_subtract/config.vsh.yaml b/src/test/resources/testns/src/ns_subtract/config.vsh.yaml index 2e45c8dfd..391adb3c4 100755 --- a/src/test/resources/testns/src/ns_subtract/config.vsh.yaml +++ b/src/test/resources/testns/src/ns_subtract/config.vsh.yaml @@ -21,6 +21,7 @@ resources: path: code.py test_resources: - type: bash_script + use_jq: true path: test.sh engines: - type: native diff --git a/src/test/resources/verification/check_config/config.vsh.yaml b/src/test/resources/verification/check_config/config.vsh.yaml index dc14e577f..fc9b82b54 100644 --- a/src/test/resources/verification/check_config/config.vsh.yaml +++ b/src/test/resources/verification/check_config/config.vsh.yaml @@ -12,11 +12,15 @@ arguments: required: true resources: - type: bash_script + use_jq: true text: ajv validate -s $par_schema $par_data engines: - type: docker image: node:20 setup: + - type: apt + packages: + - jq - type: javascript # npm: ajv-cli npm: "@jirutka/ajv-cli@6.0.0-beta.5" diff --git a/src/test/scala/io/viash/TestingAllComponentsSuite.scala b/src/test/scala/io/viash/TestingAllComponentsSuite.scala index 44a9053c5..d9867a052 100644 --- a/src/test/scala/io/viash/TestingAllComponentsSuite.scala +++ b/src/test/scala/io/viash/TestingAllComponentsSuite.scala @@ -2,10 +2,12 @@ package io.viash import io.viash.config.Config import org.scalatest.funsuite.AnyFunSuite -import io.viash.helpers.Logger +import io.viash.helpers.{Logger, Exec, IO} import org.scalatest.ParallelTestExecution import io.viash.lenses.ConfigLenses import io.viash.config.{decodeConfig, encodeConfig} +import java.nio.file.{Files, Paths} +import io.circe.parser._ class TestingAllComponentsSuite extends AnyFunSuite with ParallelTestExecution { Logger.UseColorOverride.value = Some(false) @@ -90,5 +92,79 @@ class TestingAllComponentsSuite extends AnyFunSuite with ParallelTestExecution { // check if equal assert(strippedConf2 == conf2) } + + // Test that VIASH_KEEP_WORK_DIR preserves the work directory and params.json is valid + // Skip executable type as it doesn't have a script/language + if (name != "executable") { + test(s"Testing $name params.json generation with VIASH_KEEP_WORK_DIR", DockerTest) { + val tempDir = IO.makeTemp("viash_test_json_") + val executable = tempDir.resolve(s"test_languages_$name") + + try { + // Build the component with docker engine + TestHelper.testMain("build", "--engine", "docker", "-o", tempDir.toString, config) + + // Run with VIASH_KEEP_WORK_DIR set + val result = Exec.runCatchPath( + List( + executable.toString, + getClass.getResource("/testbash/resource1.txt").getPath, + "--whole_number", "42", + "--real_number", "3.14", + "-s", "test_string", + "--multiple", "a", "--multiple", "b" + ), + cwd = None, + extraEnv = Seq("VIASH_KEEP_WORK_DIR" -> "1") + ) + + assert(result.exitValue == 0, s"Component execution failed:\n${result.output}") + + // Extract work directory path from output + val workDirPattern = """Keeping work directory at '([^']+)'""".r + val workDirMatch = workDirPattern.findFirstMatchIn(result.output) + assert(workDirMatch.isDefined, s"Could not find work directory path in output:\n${result.output}") + + val workDir = Paths.get(workDirMatch.get.group(1)) + val paramsJson = workDir.resolve("params.json") + + // Verify params.json exists + assert(Files.exists(paramsJson), s"params.json not found at $paramsJson") + + // Read and parse JSON + val jsonContent = new String(Files.readAllBytes(paramsJson)) + val json = parse(jsonContent) + assert(json.isRight, s"Invalid JSON:\n$jsonContent") + + // Verify JSON structure + val jsonObj = json.toOption.get.asObject.get + assert(jsonObj.contains("par"), "JSON should contain 'par' section") + assert(jsonObj.contains("meta"), "JSON should contain 'meta' section") + assert(jsonObj.contains("dep"), "JSON should contain 'dep' section") + + // Verify par section has expected values + val par = jsonObj("par").get.asObject.get + assert(par("whole_number").get.asNumber.get.toInt.contains(42), "whole_number should be 42") + val realNum = par("real_number").get.asNumber.get.toDouble + assert(math.abs(realNum - 3.14) < 0.001, s"real_number should be 3.14, got $realNum") + assert(par("s").get.asString.contains("test_string"), "s should be 'test_string'") + + // Verify array parameter + val multiple = par("multiple").get.asArray.get + assert(multiple.length == 2, "multiple should have 2 elements") + assert(multiple(0).asString.contains("a"), "first element should be 'a'") + assert(multiple(1).asString.contains("b"), "second element should be 'b'") + + // Verify meta section + val meta = jsonObj("meta").get.asObject.get + assert(meta("name").get.asString.contains(s"test_languages_$name"), s"name should be test_languages_$name") + + // Clean up work directory + IO.deleteRecursively(workDir) + } finally { + IO.deleteRecursively(tempDir) + } + } + } } } \ No newline at end of file diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala index 8b708c5e3..3ddeef84f 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala @@ -39,7 +39,7 @@ class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll w assert(amount > 0) assert(amount < 4) - val engineMod = s""".engines := [{"type": "docker", "image": "bash:3.2", "id": "$dockerId"}]""" + val engineMod = s""".engines := [{"type": "docker", "image": "bash:3.2", "id": "$dockerId", "setup": [{"type": "apk", "packages": ["jq"]}]}]""" val modsWithEngine = mods :+ engineMod val localConfigFile = configDeriver.derive(modsWithEngine, dockerId) val localConfig = Config.read(localConfigFile) diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala index 7d7a4d4e6..de98648a0 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala @@ -25,6 +25,9 @@ abstract class AbstractMainBuildAuxiliaryDockerRequirements extends FixtureAnyFu protected val image = "bash:3.2" protected val dockerTag = "viash_requirements_testbench" + // jq setup requirement to be prepended to Docker engine setup. + // Subclasses should override this for non-Alpine images. + protected val jqSetup: String = """{ "type": "apk", "packages": ["jq"] }""" case class FixtureParam() @@ -45,7 +48,13 @@ abstract class AbstractMainBuildAuxiliaryDockerRequirements extends FixtureAnyFu } def deriveEngineConfig(setup: Option[String], test_setup: Option[String], name: String): String = { - val setupStr = setup.map(s => s""", "setup": $s""").getOrElse("") + val allSetupItems = setup match { + case Some(s) => + val inner = s.trim.stripPrefix("[").stripSuffix("]").trim + if (inner.nonEmpty) s"$jqSetup, $inner" else jqSetup + case None => jqSetup + } + val setupStr = s""", "setup": [$allSetupItems]""" val testSetupStr = test_setup.map(s => s""", "test_setup": $s""").getOrElse("") configDeriver.derive( @@ -142,6 +151,7 @@ class MainBuildAuxiliaryDockerRequirementsApk extends AbstractMainBuildAuxiliary class MainBuildAuxiliaryDockerRequirementsApt extends AbstractMainBuildAuxiliaryDockerRequirements { override val dockerTag = "viash_requirements_testbench_apt" override val image = "debian:bullseye-slim" + override protected val jqSetup: String = """{ "type": "apt", "packages": ["jq"] }""" test("setup; check base image for apt still does not contain the cowsay package", DockerTest) { f => val newConfigFilePath = deriveEngineConfig(None, None, "apt_base") @@ -219,6 +229,7 @@ class MainBuildAuxiliaryDockerRequirementsApt extends AbstractMainBuildAuxiliary class MainBuildAuxiliaryDockerRequirementsYum extends AbstractMainBuildAuxiliaryDockerRequirements{ override val dockerTag = "viash_requirements_testbench_yum" override val image = "fedora:38" + override protected val jqSetup: String = """{ "type": "yum", "packages": ["jq"] }""" test("setup; check base image for yum still does not contain the which package", DockerTest) { f => val newConfigFilePath = deriveEngineConfig(None, None, "yum_base") @@ -296,6 +307,7 @@ class MainBuildAuxiliaryDockerRequirementsYum extends AbstractMainBuildAuxiliary class MainBuildAuxiliaryDockerRequirementsRuby extends AbstractMainBuildAuxiliaryDockerRequirements{ override val dockerTag = "viash_requirements_testbench_ruby" override val image = "ruby:slim-bullseye" + override protected val jqSetup: String = """{ "type": "apt", "packages": ["jq"] }""" test("setup; check base image for yum still does not contain the which package", DockerTest) { f => val newConfigFilePath = deriveEngineConfig(None, None, "ruby_base") @@ -373,6 +385,7 @@ class MainBuildAuxiliaryDockerRequirementsRuby extends AbstractMainBuildAuxiliar class MainBuildAuxiliaryDockerRequirementsR extends AbstractMainBuildAuxiliaryDockerRequirements{ override val dockerTag = "viash_requirements_testbench_r" override val image = "r-base:4.3.1" + override protected val jqSetup: String = """{ "type": "apt", "packages": ["jq"] }""" test("setup; check base image for r still does not contain the glue package", DockerTest) { f => val newConfigFilePath = deriveEngineConfig(None, None, "r_base") @@ -484,6 +497,7 @@ class MainBuildAuxiliaryDockerRequirementsR extends AbstractMainBuildAuxiliaryDo class MainBuildAuxiliaryDockerRequirementsRBioc extends AbstractMainBuildAuxiliaryDockerRequirements{ override val dockerTag = "viash_requirements_testbench_rbioc" override val image = "r-base:4.3.1" + override protected val jqSetup: String = """{ "type": "apt", "packages": ["jq"] }""" test("setup; check base image for r-bioc still does not contain the BiocGenerics package", DockerTest) { f => val newConfigFilePath = deriveEngineConfig(None, None, "rbioc_base") diff --git a/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala b/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala index 934c651f5..f06990388 100644 --- a/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala +++ b/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala @@ -27,7 +27,7 @@ class StringArgumentTest extends AnyFunSuite with BeforeAndAfterAll { assert(arg.`type` == "string") assert(arg.par == "par_foo") - assert(arg.VIASH_PAR == "VIASH_PAR_FOO") + assert(arg.VIASH_PAR == "VIASH_PAR_foo") assert(arg.flags == "--") assert(arg.plainName == "foo") diff --git a/src/test/scala/io/viash/config/dependencies/DependencyOfDependencyTest.scala b/src/test/scala/io/viash/config/dependencies/DependencyOfDependencyTest.scala index 9173f4446..a7844e037 100644 --- a/src/test/scala/io/viash/config/dependencies/DependencyOfDependencyTest.scala +++ b/src/test/scala/io/viash/config/dependencies/DependencyOfDependencyTest.scala @@ -40,7 +40,7 @@ class DependencyOfDependencyTest extends AnyFunSuite with BeforeAndAfterAll { // Short wrapper for creating a bash script containing some text and using it as a single resource def textBashScript(text: String): List[BashScript] = - List(BashScript(text = Some(text), dest = Some("./script.sh"))) + List(BashScript(text = Some(text), dest = Some("./script.sh"), use_jq = Some(true))) test("Prepare package 1") { val workingDir = createViashSubFolder(temporaryFolder, "pack1") diff --git a/src/test/scala/io/viash/config/dependencies/Dependency.scala b/src/test/scala/io/viash/config/dependencies/DependencyTest.scala similarity index 96% rename from src/test/scala/io/viash/config/dependencies/Dependency.scala rename to src/test/scala/io/viash/config/dependencies/DependencyTest.scala index fb2a827b7..567e14b0d 100644 --- a/src/test/scala/io/viash/config/dependencies/Dependency.scala +++ b/src/test/scala/io/viash/config/dependencies/DependencyTest.scala @@ -36,7 +36,7 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll { // Short wrapper for creating a bash script containing some text and using it as a single resource def textBashScript(text: String): List[BashScript] = - List(BashScript(text = Some(text), dest = Some("./script.sh"))) + List(BashScript(text = Some(text), dest = Some("./script.sh"), use_jq = Some(true))) test("Create a remote dependency with repository defined as sugar syntax") { val dep = Dependency("dep1", repository = Left("vsh://hendrik/dependency_test@main_build")) @@ -89,7 +89,7 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll { assert(executable.canExecute) val outputText = IO.read(outputPath.toUri()) - assert(outputText.contains("VIASH_DEP_DEP1="), "check the dependency is set in the output script") + assert(outputText.contains("VIASH_DEP_dep1="), "check the dependency is set in the output script") // check output when running val out = Exec.runCatch( @@ -144,7 +144,7 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll { assert(executable.canExecute) val outputText = IO.read(outputPath.toUri()) - assert(outputText.contains("VIASH_DEP_DEP1="), "check the dependency is set in the output script") + assert(outputText.contains("VIASH_DEP_dep1="), "check the dependency is set in the output script") // check output when running val out = Exec.runCatch( @@ -199,7 +199,7 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll { assert(executable.canExecute) val outputText = IO.read(outputPath.toUri()) - assert(outputText.contains("VIASH_DEP_DEP1="), "check the dependency is set in the output script") + assert(outputText.contains("VIASH_DEP_dep1="), "check the dependency is set in the output script") // check output when running val out = Exec.runCatch( @@ -238,7 +238,7 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll { assert(executable.canExecute) val outputText = IO.read(outputPath.toUri()) - assert(outputText.contains("VIASH_DEP_VIASH_HUB_DEP="), "check the dependency is set in the output script") + assert(outputText.contains("VIASH_DEP_viash_hub_dep="), "check the dependency is set in the output script") // check output when running val out = Exec.runCatch( @@ -278,7 +278,7 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll { assert(executable.canExecute) val outputText = IO.read(outputPath.toUri()) - assert(outputText.contains("VIASH_DEP_VIASH_HUB_DEP="), "check the dependency is set in the output script") + assert(outputText.contains("VIASH_DEP_viash_hub_dep="), "check the dependency is set in the output script") // check output when running val out = Exec.runCatch( @@ -325,7 +325,7 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll { assert(executable.canExecute) val outputText = IO.read(outputPath.toUri()) - assert(outputText.contains("VIASH_DEP_VIASH_HUB_DEP="), "check the dependency is set in the output script") + assert(outputText.contains("VIASH_DEP_viash_hub_dep="), "check the dependency is set in the output script") // check output when running val out = Exec.runCatch( @@ -364,7 +364,7 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll { assert(executable.canExecute) val outputText = IO.read(outputPath.toUri()) - assert(outputText.contains("VIASH_DEP_VIASH_HUB_TEST_TREE="), "check the dependency is set in the output script") + assert(outputText.contains("VIASH_DEP_viash_hub_test_tree="), "check the dependency is set in the output script") // check output when running val out = Exec.runCatch( diff --git a/src/test/scala/io/viash/config/dependencies/Repository.scala b/src/test/scala/io/viash/config/dependencies/RepositoryTest.scala similarity index 100% rename from src/test/scala/io/viash/config/dependencies/Repository.scala rename to src/test/scala/io/viash/config/dependencies/RepositoryTest.scala diff --git a/src/test/scala/io/viash/e2e/build/NativeSuite.scala b/src/test/scala/io/viash/e2e/build/NativeSuite.scala index f3eb24065..ea61d7de8 100644 --- a/src/test/scala/io/viash/e2e/build/NativeSuite.scala +++ b/src/test/scala/io/viash/e2e/build/NativeSuite.scala @@ -163,7 +163,7 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { ) assert(out.exitValue == 1) - assert(out.output.contains("[error] Bad arguments for option '--whole_number': '789' & '123' - you should provide exactly one argument for this option.")) + assert(out.output.contains("[error] Pass only one argument to argument '--whole_number'. Found: '789' & '123'")) } test("Repeated flag arguments are not allowed") { @@ -179,7 +179,7 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { ) ) assert(out.exitValue == 1) - assert(out.output.contains("[error] Bad arguments for option '--falsehood': 'false' & '' - you should provide exactly one argument for this option.")) + assert(out.output.contains("[error] Pass only one argument to argument '--falsehood'. Found: 'false' & 'false'")) } test("Repeated arguments with --multiple defined are allowed") { @@ -199,6 +199,92 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { assert(out.output.contains("multiple: |foo;bar|")) } + test("Build and run with use_jq set to false") { + val useJqFalseDir = Paths.get(tempFolStr, "use_jq_false").toString + val newConfigFilePath = configDeriver.derive( + """.resources[.type == "bash_script"].use_jq := false""", + "use_jq_false" + ) + TestHelper.testMain( + "build", + "--engine", "native", + "--runner", "executable", + "-o", useJqFalseDir, + newConfigFilePath + ) + + val useJqFalseExecutable = Paths.get(useJqFalseDir, config.name).toFile + assert(useJqFalseExecutable.exists) + assert(useJqFalseExecutable.canExecute) + + val output = Paths.get(useJqFalseDir, "output_jqf.txt").toFile + Exec.run( + Seq( + useJqFalseExecutable.toString, + useJqFalseExecutable.toString, + "--real_number", "10.5", + "--whole_number=10", + "-s", "a string with a few spaces", + "--truth", + "--output", output.toString, + "--multiple", "foo", + "--multiple=bar" + ) + ) + + assert(output.exists()) + val outputSrc = Source.fromFile(output) + try { + val outputLines = outputSrc.mkString + assert(outputLines.contains("""multiple: |foo;bar|""")) + } finally { + outputSrc.close() + } + } + + test("Build and run with use_jq unset") { + val useJqUnsetDir = Paths.get(tempFolStr, "use_jq_unset").toString + val newConfigFilePath = configDeriver.derive( + List(""".resources[.type == "bash_script"].use_jq := null""", """.test_resources[.type == "bash_script"].use_jq := null"""), + "use_jq_unset" + ) + TestHelper.testMain( + "build", + "--engine", "native", + "--runner", "executable", + "-o", useJqUnsetDir, + newConfigFilePath + ) + + val useJqUnsetExecutable = Paths.get(useJqUnsetDir, config.name).toFile + assert(useJqUnsetExecutable.exists) + assert(useJqUnsetExecutable.canExecute) + + val output = Paths.get(useJqUnsetDir, "output_jqu.txt").toFile + Exec.run( + Seq( + useJqUnsetExecutable.toString, + useJqUnsetExecutable.toString, + "--real_number", "10.5", + "--whole_number=10", + "-s", "a string with a few spaces", + "--truth", + "--output", output.toString, + "--multiple", "foo", + "--multiple=bar" + ) + ) + + assert(output.exists()) + val outputSrc = Source.fromFile(output) + try { + val outputLines = outputSrc.mkString + assert(outputLines.contains("""multiple: |foo;bar|""")) + } finally { + outputSrc.close() + } + } + test("when --runner is omitted, the system should run as native") { val newConfigFilePath = configDeriver.derive("""del(.runners)""", "no_runner") val testText = TestHelper.testMain( diff --git a/src/test/scala/io/viash/e2e/config_inject/MainConfigInjectSuite.scala b/src/test/scala/io/viash/e2e/config_inject/MainConfigInjectSuite.scala index 2de41466a..477067273 100644 --- a/src/test/scala/io/viash/e2e/config_inject/MainConfigInjectSuite.scala +++ b/src/test/scala/io/viash/e2e/config_inject/MainConfigInjectSuite.scala @@ -23,7 +23,7 @@ class MainConfigInjectSuite extends AnyFunSuite with BeforeAndAfterAll { ("python", "config.vsh.yaml", "#", "'input': r'input.txt'"), // ("r", "script.vsh.R", "//", "input = \"input.txt\""), // TODO add back when `viash config inject` works for inline configs or add separate config/script combo for R ("js", "config.vsh.yaml", "//", "'input': String.raw`input.txt`"), - ("scala", "config.vsh.yaml", "//", "Some(\"\"\"input.txt\"\"\"),"), + ("scala", "config.vsh.yaml", "//", "\"\"\"input.txt\"\"\""), ("csharp", "config.vsh.yaml", "//", "input = @\"input.txt\""), ) @@ -33,8 +33,6 @@ class MainConfigInjectSuite extends AnyFunSuite with BeforeAndAfterAll { } for ((name, file, comment, expectedInputString) <- tests) { - println(s"$name $file $expectedInputString") - test(s"config inject works for $name") { // check source file exists val configFile = destPath.resolve(s"$name/$file") @@ -48,6 +46,7 @@ class MainConfigInjectSuite extends AnyFunSuite with BeforeAndAfterAll { // inject script TestHelper.testMain( "config", "inject", + "--force", configFile.toString(), ) @@ -60,25 +59,13 @@ class MainConfigInjectSuite extends AnyFunSuite with BeforeAndAfterAll { // run inject script a second time. The result should be the same as running it once (no changes that stack). TestHelper.testMain( "config", "inject", + "--force", configFile.toString(), ) val code2 = Source.fromFile(scriptFile.toString()).getLines().mkString("\n") - // these lines change each time config inject is run, clear the random number - // meta_resources_dir='/tmp/viash_inject_test_languages15183647856847957861' - // meta_executable='/tmp/viash_inject_test_languages15183647856847957861/test_languages' - // meta_config='/tmp/viash_inject_test_languages15183647856847957861/.config.vsh.yaml' - - // assume number is a Long, so between 1 and 20 decimal characters - val code_replacements = code.replaceAll(s"inject_test_languages_$name\\d*", "") - val code2_replacements = code2.replaceAll(s"inject_test_languages_$name\\d*", "") - - assert(code.length - code_replacements.length <= 126 + (name.length+1)*3, "Stripping the paths should not cause very big differences") - assert(code.length - code_replacements.length >= 66 + (name.length+1)*3, "Stripping the paths should not cause very big differences, but at least some") - assert(code2.length - code2_replacements.length <= 126 + (name.length+1)*3, "Stripping the paths should not cause very big differences") - assert(code2.length - code2_replacements.length >= 66 + (name.length+1)*3, "Stripping the paths should not cause very big differences, but at least some") - - assert(code_replacements.length == code2_replacements.length, "Running config inject multiple times should not result in substantial code differences. Only the placeholder folder is different.") + // Running config inject multiple times should produce identical results + assert(code == code2, "Running config inject multiple times should produce identical code.") } } diff --git a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala index 2c33a0771..6e8fc55a2 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala @@ -68,7 +68,7 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall } test("Check setup strategy", DockerTest) { - val newConfigFilePath = configDeriver.derive(""".engines[.type == "docker" && !has(.id) ].setup := [{ type: "docker", run: "echo 'Hello world!'" }]""", "cache_config") + val newConfigFilePath = configDeriver.derive(""".engines[.type == "docker" && !has(.id) ].setup := [{ type: "apk", packages: ["jq"] }, { type: "docker", run: "echo 'Hello world!'" }]""", "cache_config") // first run to create cache entries val testOutput = TestHelper.testMain( "test", diff --git a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala index ff8a0ca46..bb24c13a8 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala @@ -38,6 +38,38 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { checkTempDirAndRemove(testOutput.stdout, false) } + test("Check test output with use_jq set to false") { + val newConfigFilePath = configDeriver.derive( + """.resources[.type == "bash_script"].use_jq := false""", + "use_jq_false" + ) + val testOutput = TestHelper.testMain( + "test", + "--engine", "native", + "--runner", "executable", + newConfigFilePath + ) + + assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!")) + checkTempDirAndRemove(testOutput.stdout, false) + } + + test("Check test output with use_jq unset") { + val newConfigFilePath = configDeriver.derive( + List(""".resources[.type == "bash_script"].use_jq := null""", """.test_resources[.type == "bash_script"].use_jq := null"""), + "use_jq_unset" + ) + val testOutput = TestHelper.testMain( + "test", + "--engine", "native", + "--runner", "executable", + newConfigFilePath + ) + + assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!")) + checkTempDirAndRemove(testOutput.stdout, false) + } + test("Check standard test output with trailing arguments") { val testOutput = TestHelper.testMain( "test", diff --git a/src/test/scala/io/viash/helpers/BashUtilsTest.scala b/src/test/scala/io/viash/helpers/BashUtilsTest.scala new file mode 100644 index 000000000..75c5dc1c9 --- /dev/null +++ b/src/test/scala/io/viash/helpers/BashUtilsTest.scala @@ -0,0 +1,48 @@ +package io.viash.helpers + +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import java.nio.file.{Files, Paths, Path} + +/** + * Test suite for bash utility functions. + * Runs all *.test.sh files in src/test/resources/io/viash/helpers/bashutils/ + */ +class BashUtilsTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + + private val projectRoot = Paths.get(System.getProperty("user.dir")) + private val bashUtilsTestDir = projectRoot.resolve("src/test/resources/io/viash/helpers/bashutils") + + /** Helper method to run a bash test script and assert it passes */ + private def runBashTest(scriptName: String, testName: String): Unit = { + val testScript = bashUtilsTestDir.resolve(scriptName) + + val result = Exec.runCatchPath( + List("bash", testScript.toString), + cwd = Some(projectRoot) + ) + + assert(result.exitValue == 0, s"$testName test failed:\n${result.output}") + } + + // Define test cases as (test name, script filename) + private val testCases = Seq( + ("ViashParseArgumentValue", "ViashParseArgumentValue.test.sh"), + ("ViashCleanupRegistry", "ViashCleanupRegistry.test.sh"), + ("ViashLogging", "ViashLogging.test.sh"), + ("ViashQuote", "ViashQuote.test.sh"), + ("ViashRenderJson", "ViashRenderJson.test.sh"), + ("ViashAbsolutePath", "ViashAbsolutePath.test.sh"), + ("ViashRemoveFlags", "ViashRemoveFlags.test.sh"), + ("ViashSourceDir and ViashFindTargetDir", "ViashSourceDir.test.sh"), + ("ViashDockerAutodetectMount", "ViashDockerAutodetectMount.test.sh") + ) + + // Register all tests + testCases.foreach { case (testName, scriptName) => + test(testName) { + runBashTest(scriptName, testName) + } + } +} diff --git a/src/test/scala/io/viash/helpers/JsonParserTest.scala b/src/test/scala/io/viash/helpers/JsonParserTest.scala new file mode 100644 index 000000000..0c7568669 --- /dev/null +++ b/src/test/scala/io/viash/helpers/JsonParserTest.scala @@ -0,0 +1,219 @@ +package io.viash.helpers + +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import java.nio.file.{Files, Paths, Path, StandardCopyOption} + +/** + * Test suite for language-specific JSON parsers. + * Tests the ViashParseJson functions for Bash, Python, R, JavaScript, C#, and Scala. + */ +class JsonParserTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + + private val projectRoot = Paths.get(System.getProperty("user.dir")) + + // Helper to set up temp dir with correct structure for all language parsers + private def setupTempDirWithParser( + language: String, + parserFileName: String + ): (Path, Path) = { + setupTempDirWithParserFiles( + language, + s"test_ViashParseJson.$parserFileName", + List(s"ViashParseJson.$parserFileName") + ) + } + + // Helper to set up temp dir with explicit test and parser file names + private def setupTempDirWithParserFiles( + language: String, + testFileName: String, + parserFileNames: List[String] + ): (Path, Path) = { + val tempDir = Files.createTempDirectory("viash_json_parser_test_") + + val testScriptSrc = projectRoot.resolve(s"src/test/resources/io/viash/helpers/languages/$language/$testFileName") + val testSubPath = s"src/test/resources/io/viash/helpers/languages/$language/$testFileName" + + // Create directory structure for test file + val testDir = tempDir.resolve(testSubPath).getParent + Files.createDirectories(testDir) + val testScript = tempDir.resolve(testSubPath) + Files.copy(testScriptSrc, testScript, StandardCopyOption.REPLACE_EXISTING) + + // Create directory structure for parser file(s) + for (parserFileName <- parserFileNames) { + val parserScriptSrc = projectRoot.resolve(s"src/main/resources/io/viash/languages/$language/$parserFileName") + val parserSubPath = s"src/main/resources/io/viash/languages/$language/$parserFileName" + val parserDir = tempDir.resolve(parserSubPath).getParent + Files.createDirectories(parserDir) + val parserScript = tempDir.resolve(parserSubPath) + Files.copy(parserScriptSrc, parserScript, StandardCopyOption.REPLACE_EXISTING) + } + + (tempDir, testScript) + } + + private def cleanupTempDir(tempDir: Path): Unit = { + import scala.jdk.CollectionConverters._ + Files.walk(tempDir).iterator().asScala.toList.reverse.foreach(Files.delete) + } + + test("Bash JSON parser") { + // Check if jq is available + val jqCheck = Exec.runCatch(List("jq", "--version")) + assume(jqCheck.exitValue == 0, "jq not available, skipping jq-based Bash test") + + val (tempDir, testScript) = setupTempDirWithParserFiles( + "bash", + "test_ViashParseJson.sh", + List("ViashParseJson.sh") + ) + + try { + val result = Exec.runCatchPath( + List("bash", testScript.toString), + cwd = Some(tempDir) + ) + + assert(result.exitValue == 0, s"Bash JSON parser (jq) test failed:\n${result.output}") + } finally { + cleanupTempDir(tempDir) + } + } + + test("Bash JSON parser (compatibility)") { + val (tempDir, testScript) = setupTempDirWithParserFiles( + "bash", + "test_ViashParseJsonCompatibility.sh", + List("ViashParseJsonCompatibility.sh") + ) + + try { + val result = Exec.runCatchPath( + List("bash", testScript.toString), + cwd = Some(tempDir) + ) + + assert(result.exitValue == 0, s"Bash JSON parser (compatibility) test failed:\n${result.output}") + } finally { + cleanupTempDir(tempDir) + } + } + + test("Python JSON parser") { + val (tempDir, testScript) = setupTempDirWithParser("python", "py") + + try { + val result = Exec.runCatchPath( + List("python3", testScript.toString), + cwd = Some(tempDir) + ) + + assert(result.exitValue == 0, s"Python JSON parser test failed:\n${result.output}") + } finally { + cleanupTempDir(tempDir) + } + } + + test("R JSON parser (hybrid)") { + val (tempDir, testScript) = setupTempDirWithParserFiles( + "r", + "test_ViashParseJsonHybrid.R", + List("ViashParseJsonHybrid.R") + ) + + try { + val result = Exec.runCatchPath( + List("Rscript", testScript.toString), + cwd = Some(tempDir) + ) + + assert(result.exitValue == 0, s"R JSON parser (hybrid) test failed:\n${result.output}") + } finally { + cleanupTempDir(tempDir) + } + } + + test("R JSON parser (jsonlite)") { + // Check if jsonlite is available + val jsonliteCheck = Exec.runCatch(List("Rscript", "-e", "library(jsonlite)")) + assume(jsonliteCheck.exitValue == 0, "jsonlite not available, skipping jsonlite-only R test") + + val (tempDir, testScript) = setupTempDirWithParserFiles( + "r", + "test_ViashParseJson.R", + List("ViashParseJson.R") + ) + + try { + val result = Exec.runCatchPath( + List("Rscript", testScript.toString), + cwd = Some(tempDir) + ) + + assert(result.exitValue == 0, s"R JSON parser (jsonlite) test failed:\n${result.output}") + } finally { + cleanupTempDir(tempDir) + } + } + + test("JavaScript JSON parser") { + val (tempDir, testScript) = setupTempDirWithParser("javascript", "js") + + try { + val result = Exec.runCatchPath( + List("node", testScript.toString), + cwd = Some(tempDir) + ) + + assert(result.exitValue == 0, s"JavaScript JSON parser test failed:\n${result.output}") + } finally { + cleanupTempDir(tempDir) + } + } + + test("C# JSON parser") { + // Check if dotnet-script is available (either as 'dotnet script' or 'dotnet-script') + val dotnetScriptCheck = Exec.runCatch(List("dotnet-script", "--version")) + val dotnetCheck = Exec.runCatch(List("dotnet", "script", "--version")) + val useDotnetScript = dotnetScriptCheck.exitValue == 0 + val useDotnetCmd = dotnetCheck.exitValue == 0 + assume(useDotnetScript || useDotnetCmd, "dotnet script not available, skipping C# test") + + val (tempDir, testScript) = setupTempDirWithParser("csharp", "csx") + + try { + val cmd = if (useDotnetScript) List("dotnet-script", testScript.toString) else List("dotnet", "script", testScript.toString) + val result = Exec.runCatchPath( + cmd, + cwd = Some(tempDir) + ) + + assert(result.exitValue == 0, s"C# JSON parser test failed:\n${result.output}") + } finally { + cleanupTempDir(tempDir) + } + } + + test("Scala JSON parser") { + // Check if scala is available and can actually compile + // Scala 2.x has known issues with Java 17+ due to missing javax.tools classes + val scalaCheck = Exec.runCatch(List("scala", "-e", "println(\"hello\")")) + assume(scalaCheck.exitValue == 0, s"scala not available or incompatible with current JDK, skipping Scala test. Output: ${scalaCheck.output}") + + val (tempDir, testScript) = setupTempDirWithParser("scala", "scala") + + try { + val result = Exec.runCatchPath( + List("scala", testScript.toString), + cwd = Some(tempDir) + ) + + assert(result.exitValue == 0, s"Scala JSON parser test failed:\n${result.output}") + } finally { + cleanupTempDir(tempDir) + } + } +}