CLI test for mass EVM assembly import/export

Co-authored-by: Kamil Śliwak <[email protected]>
Co-authored-by: r0qs <[email protected]>

Author: 3 people

Diff for: .circleci/config.yml

@@ -1123,7 +1123,7 @@ jobs:
 
   t_osx_cli:
     <<: *base_osx
-    parallelism: 7 # Should match number of tests in .circleci/cli.sh
+    parallelism: 8 # Should match number of tests in .circleci/parallel_cli_tests.py
     steps:
       - checkout
       - install_dependencies_osx
@@ -1218,7 +1218,7 @@ jobs:
 
   t_ubu_cli: &t_ubu_cli
     <<: *base_ubuntu2204_small
-    parallelism: 7 # Should match number of tests in .circleci/cli.sh
+    parallelism: 8 # Should match number of tests in .circleci/parallel_cli_tests.py
     steps:
       - cmdline_tests
 
@@ -1237,7 +1237,7 @@ jobs:
   t_ubu_asan_cli:
     # Runs slightly faster on medium but we only run it nightly so efficiency matters more.
     <<: *base_ubuntu2204
-    parallelism: 7 # Should match number of tests in .circleci/cli.sh
+    parallelism: 8 # Should match number of tests in .circleci/parallel_cli_tests.py
     environment:
       <<: *base_ubuntu2204_env
       ASAN_OPTIONS: check_initialization_order=true:detect_stack_use_after_return=true:strict_init_order=true:strict_string_checks=true:detect_invalid_pointer_pairs=2
@@ -1286,7 +1286,7 @@ jobs:
 
   t_ubu_ubsan_clang_cli:
     <<: *base_ubuntu2204_clang
-    parallelism: 7 # Should match number of tests in .circleci/cli.sh
+    parallelism: 8 # Should match number of tests in .circleci/parallel_cli_tests.py
     steps:
       - cmdline_tests
 

Diff for: .circleci/parallel_cli_tests.py

@@ -8,6 +8,7 @@
 # TODO: We should switch to time-based splitting but that requires JUnit XML report support in cmdlineTests.sh.
 tests_to_run_in_parallel = [
     '~ast_import_export',                   # ~7 min
+    '~evmasm_import_export',                # ~5 min
     '~ast_export_with_stop_after_parsing',  # ~4 min
     '~soljson_via_fuzzer',                  # ~3 min
     '~via_ir_equivalence',                  # ~1 min

Diff for: scripts/ASTImportTest.sh

@@ -23,33 +23,48 @@
 # ast import/export tests:
 #   - first exporting a .sol file to JSON, then loading it into the compiler
 #     and exporting it again. The second JSON should be identical to the first.
+#
+# evm-assembly import/export tests:
+#   - first a .sol file will be compiled and the EVM Assembly will be exported
+#     to JSON format using --asm-json command-line option.
+#     The EVM Assembly JSON output is then imported with --import-asm-json
+#     and compiled again. The binary generated initially and after the import
+#     should be identical.
 
 set -euo pipefail
 
 READLINK=readlink
 if [[ "$OSTYPE" == "darwin"* ]]; then
     READLINK=greadlink
 fi
+EXPR="expr"
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    EXPR="gexpr"
+fi
+
 REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..)
 SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build}
 SOLC="${SOLIDITY_BUILD_DIR}/solc/solc"
 SPLITSOURCES="${REPO_ROOT}/scripts/splitSources.py"
 
 # shellcheck source=scripts/common.sh
 source "${REPO_ROOT}/scripts/common.sh"
+# shellcheck source=scripts/common_cmdline.sh
+source "${REPO_ROOT}/scripts/common_cmdline.sh"
 
 function print_usage
 {
-    echo "Usage: ${0} ast [--exit-on-error|--help]."
+    echo "Usage: ${0} ast|evm-assembly [--exit-on-error|--help]."
 }
 
 function print_used_commands
 {
     local test_directory="$1"
     local export_command="$2"
     local import_command="$3"
-    printError "You can find the files used for this test here: ${test_directory}"
-    printError "Used commands for test:"
+    echo
+    printError "You can find the files used for this test in ${test_directory}"
+    printError "Commands used in the test:"
     printError "# export"
     echo "$ ${export_command}" >&2
     printError "# import"
@@ -81,6 +96,7 @@ for PARAM in "$@"
 do
     case "$PARAM" in
         ast) check_import_test_type_unset ; IMPORT_TEST_TYPE="ast" ;;
+        evm-assembly) check_import_test_type_unset ; IMPORT_TEST_TYPE="evm-assembly" ;;
         --help) print_usage ; exit 0 ;;
         --exit-on-error) EXIT_ON_ERROR=1 ;;
         *) fail "Unknown option '$PARAM'. Aborting. $(print_usage)" ;;
@@ -89,6 +105,7 @@ done
 
 SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests"
 ASTJSONTESTS_DIR="${REPO_ROOT}/test/libsolidity/ASTJSON"
+SEMANTICTESTS_DIR="${REPO_ROOT}/test/libsolidity/semanticTests"
 
 FAILED=0
 UNCOMPILABLE=0
@@ -153,6 +170,92 @@ function test_ast_import_export_equivalence
     TESTED=$((TESTED + 1))
 }
 
+function run_solc
+{
+    local parameters=( "${@}" )
+
+    if ! "${SOLC}" "${parameters[@]}" > /dev/null 2> solc_stderr
+    then
+        printError "ERROR: ${parameters[*]}"
+        printError "${PWD}"
+        # FIXME: EXIT_ON_ERROR seems to be ignored here and in some other places.
+        # We just exit unconditionally instead.
+        fail "$(cat solc_stderr)"
+    fi
+    rm solc_stderr
+}
+
+function run_solc_store_stdout
+{
+    local output_file=$1
+    local parameters=( "${@:2}" )
+
+    if ! "${SOLC}" "${parameters[@]}" > "${output_file}" 2> "${output_file}.error"
+    then
+        printError "ERROR: ${parameters[*]}"
+        printError "${PWD}"
+        fail "$(cat "${output_file}.error")"
+    fi
+    rm "${output_file}.error"
+}
+
+function test_evmjson_import_export_equivalence
+{
+    local sol_file="$1"
+    local input_files=( "${@:2}" )
+
+    # Generate bytecode and EVM assembly JSON through normal complication
+    mkdir -p export/
+    local export_options=(--bin --asm-json "${input_files[@]}" --output-dir export/)
+    run_solc "${export_options[@]}"
+
+    # NOTE: If there is no bytecode, the compiler produces a JSON file that contains just 'null'.
+    # This is not accepted by assembly import though so we must skip such contracts.
+    echo -n null > null
+    find export/ -name '*.json' -exec cmp --quiet --bytes=4 {} null \; -delete
+
+    find export/ -name '*.bin' -size 0 -delete
+
+    for asm_json_file in export/*.json
+    do
+        mv "${asm_json_file}" "${asm_json_file/_evm/}"
+    done
+
+    # Import EVM assembly JSON
+    mkdir -p import/
+    for asm_json_file in export/*.json
+    do
+        local bin_file_from_asm_import
+        bin_file_from_asm_import="import/$(basename "${asm_json_file}" .json).bin"
+
+        local import_options=(--bin --import-asm-json "${asm_json_file}")
+        run_solc_store_stdout "${bin_file_from_asm_import}" "${import_options[@]}"
+
+        stripCLIDecorations < "$bin_file_from_asm_import" > tmpfile
+        mv tmpfile "$bin_file_from_asm_import"
+    done
+
+    # Compare bytecode from compilation with bytecode from import
+    for bin_file in export/*.bin
+    do
+        local bin_file_from_asm_import=${bin_file/export/import}
+        if ! diff --strip-trailing-cr --ignore-all-space "${bin_file}" "${bin_file_from_asm_import}" > diff_error
+        then
+            printError "ERROR: Bytecode from compilation (${bin_file}) differs from bytecode from EVM asm import (${bin_file_from_asm_import}):"
+            printError "    $(cat diff_error)"
+            if (( EXIT_ON_ERROR == 1 ))
+            then
+                # NOTE: The import_options we print here refers to the wrong file (the last one
+                # processed by the previous loop) - but it's still a good starting point for debugging ;)
+                print_used_commands "${PWD}" "${SOLC} ${export_options[*]}" "${SOLC} ${import_options[*]}"
+                return 1
+            fi
+            FAILED=$((FAILED + 1))
+        fi
+    done
+    TESTED=$((TESTED + 1))
+}
+
 # function tests whether exporting and importing again is equivalent.
 # Results are recorded by incrementing the FAILED or UNCOMPILABLE global variable.
 # Also, in case of a mismatch a diff is printed
@@ -168,6 +271,7 @@ function test_import_export_equivalence {
 
     case "$IMPORT_TEST_TYPE" in
         ast) compile_test="--ast-compact-json" ;;
+        evm-assembly) compile_test="--bin" ;;
         *) assertFail "Unknown import test type '${IMPORT_TEST_TYPE}'. Aborting." ;;
     esac
 
@@ -181,6 +285,7 @@ function test_import_export_equivalence {
     then
         case "$IMPORT_TEST_TYPE" in
             ast) test_ast_import_export_equivalence "${sol_file}" "${input_files[@]}" ;;
+            evm-assembly) test_evmjson_import_export_equivalence "${sol_file}" "${input_files[@]}" ;;
             *) assertFail "Unknown import test type '${IMPORT_TEST_TYPE}'. Aborting." ;;
         esac
     else
@@ -191,16 +296,30 @@ function test_import_export_equivalence {
         # and print some details about the corresponding solc invocation.
         if (( solc_return_code == 2 ))
         then
-            fail "\n\nERROR: Uncaught Exception while executing '$SOLC ${compile_test} ${input_files[*]}':\n${output}\n"
+            # For the evm-assembly import/export tests, this script uses only the old code generator.
+            # Some semantic tests can only be compiled with --via-ir (some need to be additionally
+            # compiled with --optimize). The tests that are meant to be compiled with --via-ir are
+            # throwing an UnimplementedFeatureError exception, e.g.:
+            # "Copying of type struct C.S memory[] memory to storage not yet supported",
+            # "Copying nested calldata dynamic arrays to storage is not implemented in the old code generator".
+            # We will just ignore these kind of exceptions for now. However, any other exception
+            # will be treated as a fatal error and the script execution will be terminated with an error.
+            if [[ "${output}" != *"UnimplementedFeatureError"* ]]
+            then
+                fail "\n\nERROR: Uncaught exception while executing '${SOLC} ${compile_test} ${input_files[*]}':\n${output}\n"
+            fi
         fi
     fi
 }
 
 command_available "$SOLC" --version
 command_available jq --version
+command_available "$EXPR" --version
+command_available "$READLINK" --version
 
 case "$IMPORT_TEST_TYPE" in
     ast) TEST_DIRS=("${SYNTAXTESTS_DIR}" "${ASTJSONTESTS_DIR}") ;;
+    evm-assembly) TEST_DIRS=("${SEMANTICTESTS_DIR}") ;;
     *) assertFail "Import test type not defined. $(print_usage)" ;;
 esac
 

Diff for: test/cmdlineTests/~evmasm_import_export/test.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# shellcheck source=scripts/common.sh
+source "${REPO_ROOT}/scripts/common.sh"
+
+SOLTMPDIR=$(mktemp -d -t "cmdline-test-evmasm-import-export-XXXXXX")
+cd "$SOLTMPDIR"
+if ! "$REPO_ROOT/scripts/ASTImportTest.sh" evm-assembly
+then
+    rm -r "$SOLTMPDIR"
+    fail
+fi
+rm -r "$SOLTMPDIR"

Diff for: test/libsolidity/semanticTests/expressions/module_from_ternary_expression.sol

@@ -1,13 +1,13 @@
 ==== Source: A ====
-contract C {
+contract D {
 }
 ==== Source: B ====
 import "A" as M;
 
 contract C {
     function f() public pure returns (bool) {
         bool flag;
-        ((flag = true) ? M : M).C;
+        ((flag = true) ? M : M).D;
         return flag;
     }
 }

Diff for: test/libsolidity/semanticTests/various/many_subassemblies.sol

@@ -0,0 +1,35 @@
+contract C0 {}
+contract C1 {}
+contract C2 {}
+contract C3 {}
+contract C4 {}
+contract C5 {}
+contract C6 {}
+contract C7 {}
+contract C8 {}
+contract C9 {}
+contract C10 {}
+
+contract D {
+    function run() public {
+        // This is primarily meant to test assembly import via --import-asm-json.
+        // The exported JSON will fail the reimport unless the subassembly indices are parsed
+        // correctly - as hex numbers.
+        new C0();
+        new C1();
+        new C2();
+        new C3();
+        new C4();
+        new C5();
+        new C6();
+        new C7();
+        new C8();
+        new C9();
+        new C10();
+    }
+}
+// ----
+// run() ->
+// gas irOptimized: 381615
+// gas legacy: 392719
+// gas legacyOptimized: 392719