testall

#!/bin/sh
#
#  Copyright 2021 Northern.tech AS
#
#  This file is part of CFEngine 3 - written and maintained by Northern.tech AS.
#
#  This program is free software; you can redistribute it and/or modify it
#  under the terms of the GNU General Public License as published by the
#  Free Software Foundation; version 3.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA
#
# To the extent this program is licensed as part of the Enterprise
# versions of CFEngine, the applicable Commercial Open Source License
# (COSL) may apply to this file if you as a licensee so wish it. See
# included file COSL.txt.
#

#
# Detect and replace non-POSIX shell
#
try_exec() {
    type "$1" > /dev/null 2>&1 && exec "$@"
}

broken_posix_shell()
{
    unset foo
    local foo=1
    test "$foo" != "1"
}

if broken_posix_shell >/dev/null 2>&1
then
    try_exec /usr/xpg4/bin/sh "$0" "$@"
    echo "No compatible shell script interpreter found."
    echo "Please find a POSIX shell for your system."
    exit 42
fi

#
# Explicitly use POSIX tools if needed
#
if [ -f /usr/xpg4/bin/grep ]
then
    PATH=/usr/xpg4/bin:$PATH
    export PATH
fi

#
# Unset environment variables which might break running acceptance tests
#
GREP_OPTIONS=
export GREP_OPTIONS

#
# Defaults (overridden by command-line arguments)
#
LOG=test.log
INOTIFYWATCH_LOG=inotifywatch.log
INOTIFYWAIT_LOG=inotifywait.log
INOTIFYWATCH_OPTS="-r -e modify -e attrib -e moved_to -e create -e delete -e delete_self"
INOTIFYWAIT_OPTS="-r -m -e modify -e attrib -e moved_to -e create -e delete -e delete_self"
SUMMARY=summary.log
XML=test.xml
WHOAMI=/c/Windows/System32/whoami.exe
BASE_WORKDIR=${BASE_WORKDIR:-$(pwd)/workdir}
QUIET=

# Default test types to run if --tests= is not passed

# DEFAULT ON
COMMON_TESTS=${COMMON_TESTS:-1}       ; export COMMON_TESTS
TIMED_TESTS=${TIMED_TESTS:-1}         ; export TIMED_TESTS
SLOW_TESTS=${SLOW_TESTS:-1}           ; export SLOW_TESTS
ERROREXIT_TESTS=${ERROREXIT_TESTS:-1} ; export ERROREXIT_TESTS
ERRORLOG_TESTS=${ERRORLOG_TESTS:-1}   ; export ERRORLOG_TESTS
SERIAL_TESTS=${SERIAL_TESTS:-1}       ; export SERIAL_TESTS
NETWORK_TESTS=${NETWORK_TESTS:-1}     ; export NETWORK_TESTS
LIBXML2_TESTS=${LIBXML2_TESTS:-1}     ; export LIBXML2_TESTS
LIBCURL_TESTS=${LIBCURL_TESTS:-1}     ; export LIBCURL_TESTS
# DEFAULT OFF
UNSAFE_TESTS=${UNSAFE_TESTS:-0}       ; export UNSAFE_TESTS
STAGING_TESTS=${STAGING_TESTS:-0}     ; export STAGING_TESTS


BINDIR=${BINDIR:-}                  ; export BINDIR
NO_CLEAN=${NO_CLEAN:-0}             ; export NO_CLEAN
BASECLASSES=${BASECLASSES:-AUTO}    ; export BASECLASSES
EXTRACLASSES=${EXTRACLASSES:-DEBUG} ; export EXTRACLASSES
AGENT=${AGENT:-}                    ; export AGENT
CF_PROMISES=${CF_PROMISES:-}        ; export CF_PROMISES
CF_SERVERD=${CF_SERVERD:-}          ; export CF_SERVERD
CF_EXECD=${CF_EXECD:-}              ; export CF_EXECD
CF_KEY=${CF_KEY:-}                  ; export CF_KEY
CF_SECRET=${CF_SECRET:-}            ; export CF_SECRET
CF_NET=${CF_NET:-}                  ; export CF_NET
CF_CHECK=${CF_CHECK:-}              ; export CF_CHECK
CF_RUNAGENT=${CF_RUNAGENT:-}        ; export CF_RUNAGENT
RPMVERCMP=${RPMVERCMP:-}            ; export RPMVERCMP
DIFF=${DIFF:-}                      ; export DIFF
LIBTOOL=${LIBTOOL:-}                ; export LIBTOOL
INCLUDE_IN_WORKDIR=${INCLUDE_IN_WORKDIR:-} ; export INCLUDE_IN_WORKDIR

VALGRIND_OPTS="${VALGRIND_OPTS:---leak-check=full --show-reachable=yes --suppressions=valgrind-suppressions}"
export VALGRIND_OPTS

export MAKEFLAGS
export GAINROOT

# Use TEST_INDEX for indexing a poor man's array. Basically the subscript is
# just appended to the variable name.
TEST_INDEX=0
TEST_TIMED_INDEX=0
#TESTS_TIMED_<index>=0
#TESTS_TIMEOUT_<index>=0
#TESTS_PASSES_<index>=0
TESTS_COUNT=0
TESTS_NORMAL_COUNT=0
TESTS_TIMED_COUNT=0
TESTS_TIMED_REMAINING=0

case "$OSTYPE" in
    msys)
        if "$WHOAMI" -priv | grep SeTakeOwnershipPrivilege > /dev/null
        then
            # Don't use elevate if we already have the privileges. It is slower and
            # pops up a flashing window for every single test.
            DEFAULT_GAINROOT=
        else
            DEFAULT_GAINROOT="`dirname $0`/tool_wrappers/elevate.sh"
        fi
        # Use hardlinks on Windows. Using symbolic links will work, but Msys creates a
        # real copy, which eats disk space very quickly when you multiply with the number
        # of tests.
        LN_CMD="ln -f"
        ;;
    *)
        DEFAULT_GAINROOT=fakeroot
        LN_CMD="ln -sf"
        ;;
esac

GAINROOT=${GAINROOT:-$DEFAULT_GAINROOT}

PASSED_TESTS=0
FAILED_TESTS=0
SUPPRESSED_FAILURES=0
SOFT_FAILURES=0
SKIPPED_TESTS=0
FLAKEY_FAILURES=0

#
# Many older platforms don't support date +%s, so check for compatibility
# and find Perl for the unix_seconds() routine below. (Mantis #1254)
#
HAVE_DATE_PCT_S=
date +%s | grep %s >/dev/null 2>&1
if [ $? -ne 0 ]
then
    HAVE_DATE_PCT_S=1
fi
PERL=`command -v perl 2>/dev/null`

# color!
if [ "${CFENGINE_COLOR}" = "1" ]
then
    COLOR_SUCCESS="\\033[1;32m"
    COLOR_FAILURE="\\033[1;31m"
    COLOR_WARNING="\\033[1;33m"
    COLOR_NORMAL="\\033[0;39m"
else
    COLOR_SUCCESS=
    COLOR_FAILURE=
    COLOR_WARNING=
    COLOR_NORMAL=
fi

#
# Obtain UNIX time(), using date +%s, Perl, or POSIX-compatible approach.
#
unix_seconds() {
    if [ "$HAVE_DATE_PCT_S" ]
    then
        date +%s
        return 0
    fi

    if [ "$PERL" ]
    then
        $PERL -e 'print time() . "\n"' 2>/dev/null
        if [ $? -eq 0 ]
        then
            return 0
        fi
    fi

    # Last resort if Perl fails - the extended cpio interchange format has
    # the file modification timestamp in columns 48-59, in octal.
    : > $BASE_WORKDIR/x
    echo "ibase=8;$(pax -wx cpio $BASE_WORKDIR/$$.seconds | cut -c 48-59)" | bc 2>/dev/null
    rm $BASE_WORKDIR/x
}

usage() {
    echo "testall [-h|--help] [-q|--quiet] [--gainroot=<command>] [--agent=<agent>] [--cfpromises=<cf-promises>] [--cfserverd=<cf-serverd>] [--cfexecd=<cf-execd>] [--cfkey=<cf-key>] [--cfkeycrypt=<cf-secret>] [--cfnet=<cf-net>] [--cfcheck=<cf-check>] [--bindir=<bindir>] [--tests=...] [--gdb] [--printlog] [<test> <test>...]"
    echo
    echo "If no test is given, all standard tests are run:"
    echo "  Tests with names of form <file>.cf are expected to run successfully"
    echo "  Tests with names of form <file>.x.cf are expected to exit with error"
    echo "Set ${COLOR_SUCCESS}CFENGINE_COLOR=1${COLOR_NORMAL} to get ANSI color markers where appropriate."
    echo
    echo "If arguments are given, those are executed as tests"
    echo
    echo " -h"
    echo " --help    prints usage"
    echo " -q"
    echo " --quiet   makes script much quieter"
    echo " --gainroot=<command>  forces use of command to gain root privileges,"
    echo "           otherwise fakeroot is used.  Use --gainroot=env to make this"
    echo "           option a no-op (e.g. you're running inside fakeroot already)"
    echo " --agent   provides a way to specify non-default cf-agent location,"
    echo "           and defaults to $DEFAGENT."
    echo " --baseclasses  provides a way to override the default cf-agent classes,"
    echo "           and defaults to ${BASECLASSES}.  Also can use --bc"
    echo " --extraclasses  provides a way to append to the default cf-agent classes,"
    echo "           and defaults to ${EXTRACLASSES}.  Also can use --ec"
    echo " --cfpromises  provides a way to specify non-default cf-promises location,"
    echo "           and defaults to $DEFCF_PROMISES."
    echo " --cfserverd  provides a way to specify non-default cf-serverd location,"
    echo "           and defaults to $DEFCF_SERVERD."
    echo " --cfexecd provides a way to specify non-default cf-execd location,"
    echo "           and defaults to $DEFCF_EXECD."
    echo " --cfkey   provides a way to specify non-default cf-key location,"
    echo "           and defaults to $DEFCF_KEY."
    echo " --cfkeycrypt  provides a way to specify non-default cf-secret location,"
    echo "           and defaults to $DEFCF_SECRET."
    echo " --cfnet   provides a way to specify non-default cf-net location,"
    echo "           and defaults to $DEFCF_NET."
    echo " --cfcheck provides a way to specify non-default cf-check location,"
    echo "           and defaults to $DEFCF_CHECK."
    echo " --cfrunagent  provides a way to specify non-default cf-runagent location,"
    echo "           and defaults to $DEFCF_RUNAGENT."
    echo " --rpmvercmp  provides a way to specify non-default rpmvercmp location,"
    echo "           and defaults to $DEFRPMVERCMP."
    echo " --bindir  specifies the directory containing all the binaries."
    echo "           Mutually exclusive with --agent and --cf* arguments."
    echo " --libtool specify non-default libtool location (only needed for --gdb)."
    echo "               defaults to $DEFLIBTOOL."
    echo " --include Include the file or directory given in the workdir before the test"
    echo "           starts. This option may be given several times."
    echo " --tests=common,network,serial,timed,slow,errorexit,errorlog,libxml2,libcurl"
    echo "           This is the default value, you can also add from the following:"
    echo "           unsafe,staging"
    echo "           NOTE: 'staging' tests are not expected to pass"
    echo "           WARN: 'unsafe' tests modify the system they are running on and can cause DAMAGE!"
    echo "                          If you use this option you should also use --gainroot=sudo."
    echo " --printlog   print the full test.log output immediately.  Override with $PRINTLOG"
    echo " --gdb          Run test under GDB"
    echo " --valgrind     Run test under Valgrind"
    echo " --callgrind    Run test under valgrind --tool=callgrind"
    echo " --inotifywatch Run tests and log filesystem statistics"
    echo " --inotifywait  Run tests and log filesystem events"
    echo " --no-clean does not clean workdir after test finishes"
    echo "            (by default it gets cleaned only if test passed)."
    echo " --base-workdir Specify a local directory for all working files"
    echo " -j[n]"
    echo " -jobs=[n] Run tests in parallel, works like make -j option. Note that some"
    echo "           tests will always run one by one."
    echo " --verbose  Run tests with verbose logging"
    echo " --debug    Run tests with debug logging"
    echo " --no-no-lock  The --no-lock option is set by default. However it can be"
    echo "               useful to debug with locks"
}

workdir() {
    echo "$BASE_WORKDIR/$(echo "$1" | sed 's,[./],_,g')"
}

# Example:    fgrepvar 'word'  VARNAME
# Silent fgrep for variables: search for "word" in $VARNAME
fgrepvar() {
    eval echo \$$2  |  fgrep "$1"  >  /dev/null
}

# Same but instead of word search for a regex
grepvar() {
    eval echo \$$2  |  grep "$1"  >  /dev/null
}

# Takes the following arguments:
# 1. Agent - The agent to execute.
# 2. Test - The test to execute.
# 3. Pass number - [Optional] The current pass number. Used by timed tests.
# 4. Timeout variable - [Optional] The name of the variable to put the next
#        timeout into. Used by timed tests.
runtest() {
    # Clear/Reset local variables
    unset AGENT TEST TEST_TYPE EXTRATEXT SKIP SKIPREASON RESULT RESULT_MSG TEST_START_TIME FLATNAME WORKDIR OUTFILE TEST_DESCRIPTION TEST_STORY TEST_COVERS

    AGENT="$1"
    TEST="$2"
    PASS_NUM="$3"
    NEXT_TIMEOUT_VAR="$4"
    # With STAY_IN_WORKDIR we may be running alongside others, and need to print everything at once
    # on one line, so that we don't risk mixing lines together.
    if [ -z "$QUIET" -a -z "$STAY_IN_WORKDIR" ]
    then
        printf "$TEST "
    fi

    case "$TEST" in
        *.x.cf)       TEST_TYPE=errorexit &&
                      EXTRATEXT='(should exit with non-zero, but not crash)' ;;
        *.error.cf)   TEST_TYPE=errorlog &&
                      EXTRATEXT='(should log error(s), but exit with 0)' ;;
        */staging/*)  TEST_TYPE=staging   ;;
        */unsafe/*)   TEST_TYPE=unsafe    ;;
        */network/*)  TEST_TYPE=network   ;;
        */timed/*)    TEST_TYPE=timed     ;;
        */slow/*)     TEST_TYPE=slow      ;;
        *[/_]serial[/_.]*)  TEST_TYPE=serial  ;;
        */11_xml_edits/*)   TEST_TYPE=libxml2 ;;
        */01_vars/02_functions/network/url_get.cf)    TEST_TYPE=libcurl ;;
        *)            TEST_TYPE=common    ;;
    esac

    SKIP=0                          # by default the test is not skipped
    SKIPREASON=
    if [ $ERROREXIT_TESTS = 0  -a  $TEST_TYPE = errorexit ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}'errorexit' tests are disabled${COLOR_NORMAL}"
    elif [ $ERRORLOG_TESTS = 0  -a  $TEST_TYPE = errorlog ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}'errorlog' tests are disabled${COLOR_NORMAL}"
    elif [ $STAGING_TESTS = 0  -a  $TEST_TYPE = staging ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}Staging tests are disabled${COLOR_NORMAL}"
    elif [ $UNSAFE_TESTS = 0  -a  $TEST_TYPE = unsafe ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}Unsafe tests are disabled${COLOR_NORMAL}"
    elif [ $NETWORK_TESTS = 0  -a  $TEST_TYPE = network ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}Network-dependent tests are disabled${COLOR_NORMAL}"
    elif [ $TIMED_TESTS = 0  -a  $TEST_TYPE = timed ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}Timed tests are disabled${COLOR_NORMAL}"
    elif [ $SLOW_TESTS = 0  -a  $TEST_TYPE = slow ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}Slow tests are disabled${COLOR_NORMAL}"
    elif [ $SERIAL_TESTS = 0  -a  $TEST_TYPE = serial ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}Serial tests are disabled${COLOR_NORMAL}"
    elif [ $LIBXML2_TESTS = 0  -a  $TEST_TYPE = libxml2 ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}XML file editing tests are disabled${COLOR_NORMAL}"
    elif [ $LIBCURL_TESTS = 0  -a  $TEST_TYPE = libcurl ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}libcurl tests are disabled${COLOR_NORMAL}"
    elif [ $COMMON_TESTS = 0  -a  $TEST_TYPE = common ]
    then
        SKIP=1
        SKIPREASON="${COLOR_WARNING}Common tests are disabled${COLOR_NORMAL}"
    fi

    TEST_START_TIME=$(unix_seconds)

    # Create workdir
    WORKDIR="$(workdir "$TEST")"
    OUTFILE="$WORKDIR/output.log"

    if [ -z "$PASS_NUM" ] || [ "$PASS_NUM" -eq 1 ]
    then
        # Don't reset workdir if this is a subsequent pass.
        $GAINROOT rm -rf "$WORKDIR"
        mkdir -p "$WORKDIR/bin" "$WORKDIR/tmp"
        chmod ugo+rwxt "$WORKDIR/tmp"
    fi

    # For unknown reason on the second pass, this file is root-owned, so we need GAINROOT here
    $GAINROOT rm -f "$OUTFILE"

    # Make sure these files are owned by the script, for some reason
    # each agent execution or each pass, messes this up.
    $GAINROOT touch        "$OUTFILE" "$WORKDIR/$LOG" "$WORKDIR/$SUMMARY" "$WORKDIR/$XML"
    $GAINROOT chown $USER  "$OUTFILE" "$WORKDIR/$LOG" "$WORKDIR/$SUMMARY" "$WORKDIR/$XML"

    if [ -n "$NEXT_TIMEOUT_VAR" ]
    then
        eval $NEXT_TIMEOUT_VAR=
    fi

    if [ "$SKIP" = 1 ]                                 # Skip
    then
        TEST_END_TIME=$TEST_START_TIME
        RESULT=Skip
        RESULT_MSG="${COLOR_WARNING}Skipped ($SKIPREASON)${COLOR_NORMAL}"

    else                                               # Do not skip
        # Prepare workdir

        # Don't copy into workdir if this is a subsequent pass.
        if [ -z "$PASS_NUM" ] || [ "$PASS_NUM" -eq 1 ]
        then
            if [ -n "$BINDIR" ]
            then
                # Copy everything, because Windows depends on DLLs.
                $LN_CMD "$BINDIR"/* "$WORKDIR/bin"
            else
                $LN_CMD "$AGENT" "$WORKDIR/bin"
                $LN_CMD "$CF_PROMISES" "$WORKDIR/bin"
                $LN_CMD "$CF_SERVERD" "$WORKDIR/bin"
                $LN_CMD "$CF_EXECD" "$WORKDIR/bin"
                $LN_CMD "$CF_KEY" "$WORKDIR/bin"
                $LN_CMD "$CF_SECRET" "$WORKDIR/bin"
                $LN_CMD "$CF_NET" "$WORKDIR/bin"
                $LN_CMD "$CF_CHECK" "$WORKDIR/bin"
                $LN_CMD "$CF_RUNAGENT" "$WORKDIR/bin"
                if [ "$NEED_RPMVERCMP" = "yes" ]
                then
                    $LN_CMD "$RPMVERCMP" "$WORKDIR/bin"
                fi
                $LN_CMD "$DIFF" "$WORKDIR/bin"
            fi
            for inc in $INCLUDE_IN_WORKDIR
            do (
                # Copy directory structure, but make links inside. This allows tests to
                # add additional files to the directory.
                base=$(basename $inc)
                mkdir -p "$WORKDIR/$base"
                cd $inc || exit 2
                for dir in $(find . -type d)
                do
                    mkdir -p "$WORKDIR/$base/$dir"
                done
                for file in $(find . \! -type d)
                do
                    $LN_CMD "$inc/$file" "$WORKDIR/$base/$file"
                done
            )
            done
        fi
        if uname | grep MINGW > /dev/null
        then
            PLATFORM_WORKDIR="$(echo $WORKDIR | sed -e 's%^/\([a-zA-Z]\)/%\1:/%' | sed -e 's%/%\\%g')"
            DS="\\"
        else
            PLATFORM_WORKDIR="$WORKDIR"
            DS="/"
        fi

        ( echo ----------------------------------------------------------------------
          echo "$TEST" "$EXTRATEXT"
          echo ----------------------------------------------------------------------
        ) >> "$WORKDIR/$LOG"

        echo "#!/bin/sh
CFENGINE_TEST_OVERRIDE_WORKDIR=\"$PLATFORM_WORKDIR\"
TEMP=\"$PLATFORM_WORKDIR${DS}tmp\"
CFENGINE_TEST_OVERRIDE_EXTENSION_LIBRARY_DIR=\"$CFENGINE_TEST_OVERRIDE_EXTENSION_LIBRARY_DIR\"
export CFENGINE_TEST_OVERRIDE_WORKDIR TEMP CFENGINE_TEST_OVERRIDE_EXTENSION_LIBRARY_DIR

" > "$WORKDIR/runtest"

        if [ "$GDB" = 1 ]
        then
            if grep libtool < "$AGENT" > /dev/null
            then
                printf "\"$LIBTOOL\" --mode=execute " >> "$WORKDIR/runtest"
            fi
            printf "gdb -silent --args " >> "$WORKDIR/runtest"
        fi

        if [ -n "$USE_VALGRIND"  -a  $TEST_TYPE = errorexit ]
        then
            if grep libtool < "$AGENT" > /dev/null
            then
                printf "\"$LIBTOOL\" --mode=execute " >> "$WORKDIR/runtest"
            fi
            printf "valgrind ${VALGRIND_OPTS} \"$AGENT\" $VERBOSE $DEBUG -${NO_LOCK}lf \"$TEST\" -D ${PASS_NUM:+test_pass_$PASS_NUM,}${BASECLASSES},${EXTRACLASSES} 2>&1\n" >> "$WORKDIR/runtest"
        elif [ x"$PRELOAD_ASAN" != x ]
        then
            printf "LD_PRELOAD=$PRELOAD_ASAN \"$AGENT\" $VERBOSE $DEBUG -${NO_LOCK}lf \"$TEST\" -D ${PASS_NUM:+test_pass_$PASS_NUM,}${BASECLASSES},${EXTRACLASSES}\n" >> "$WORKDIR/runtest"
        else
            printf "\"$AGENT\" $VERBOSE $DEBUG -${NO_LOCK}lf \"$TEST\" -D ${PASS_NUM:+test_pass_$PASS_NUM,}${BASECLASSES},${EXTRACLASSES}\n" >> "$WORKDIR/runtest"
        fi

        chmod +x "$WORKDIR/runtest"

        if [ "$GDB" = 1 ]
        then
            $GAINROOT "$WORKDIR/runtest"
        else
            eval $GAINROOT "$WORKDIR/runtest" >>$OUTFILE 2>&1
        fi

        RETVAL=$?
        TEST_END_TIME=$(unix_seconds)

        cat $OUTFILE >> "$WORKDIR/$LOG"
        echo >> "$WORKDIR/$LOG"
        echo "Return code is $RETVAL." >> "$WORKDIR/$LOG"

        # Try to collect test metadata if any
        if egrep "R: test description: " $OUTFILE > /dev/null
        then
            TEST_DESCRIPTION="$(egrep "R: test description" $OUTFILE | sed -e "s,.*test description: \([A-Za-z0-9_]*\),\1,")"
        fi
        if egrep -e "R: test story_id: " $OUTFILE > /dev/null
        then
            TEST_STORY="$(egrep "R: test story_id" $OUTFILE | sed -e "s,.*test story_id: \([0-9][0-9]*\),\1,")"
        fi
        if egrep -e "R: test covers: " $OUTFILE > /dev/null
        then
            TEST_COVERS="$(egrep "R: test covers" $OUTFILE | sed -e "s,.*test covers: \([A-Za-z0-9_]*\),\1,")"
        fi

        if [ $TEST_TYPE = errorexit ]
        then
            if [ $RETVAL -gt 0 ] && [ $RETVAL -lt 128 ] && egrep "error:|aborted on defined class" $OUTFILE > /dev/null
            then
                RESULT=Pass
                RESULT_MSG="${COLOR_SUCCESS}Pass${COLOR_NORMAL}"
            elif [ $RETVAL -ge 128 ]
            then
                RESULT=FAIL
                RESULT_MSG="${COLOR_FAILURE}FAIL (CRASH!!! THIS IS A BUG!)${COLOR_NORMAL}"
            else
                RESULT=FAIL
                RESULT_MSG="${COLOR_FAILURE}FAIL (Failed to exit with error!)${COLOR_NORMAL}"
            fi
        elif [ $TEST_TYPE = errorlog ]
        then
            if [ $RETVAL != 0 ]
            then
                RESULT=FAIL
                RESULT_MSG="${COLOR_FAILURE}FAIL (NON-ZERO EXIT CODE!)${COLOR_NORMAL}"
            elif fgrep 'error: ' "$OUTFILE" > /dev/null
            then
                RESULT=Pass
                RESULT_MSG="${COLOR_SUCCESS}Pass${COLOR_NORMAL}"
            else
                RESULT=FAIL
                RESULT_MSG="${COLOR_FAILURE}FAIL (No errors printed)${COLOR_NORMAL}"
            fi
        else # TEST_TYPE is not errorlog or errorexit
            # We need to be careful when matching test outcomes. Because of convergence
            # passes, a test may output FAIL before it outputs Pass; in this case the
            # latter trumps the former. Also be careful when matching an [XS]FAIL; the
            # test should not be allowed to output any other outcome in the same run,
            # except for FAIL.

            # Some states are output by dcs.cf.sub, therefore check for both TEST
            # prefix and dcs.cf.sub prefix.
            ESCAPED_TEST="$(echo "($TEST|dcs.cf.sub)" | sed -e 's/\./\\./g')"
            if egrep "R: .*$ESCAPED_TEST [XS]FAIL" $OUTFILE > /dev/null && ! egrep "R: .*$ESCAPED_TEST Wait/" $OUTFILE > /dev/null
            then
                # Check for passed outcome, which should not happen.
                if egrep "R: .*$ESCAPED_TEST " $OUTFILE | egrep "R: .*$ESCAPED_TEST Pass" > /dev/null
                then
                    RESULT=FAIL
                    RESULT_MSG="${COLOR_FAILURE}FAIL (The test Passed, but failure was expected)${COLOR_NORMAL}"
                elif egrep "R: .*$ESCAPED_TEST " $OUTFILE | egrep -v "R: .*$ESCAPED_TEST [XS]?FAIL" > /dev/null
                then
                    # Other test case outcomes than fail. Should not happen.
                    RESULT=FAIL
                    RESULT_MSG="${COLOR_FAILURE}FAIL (Failure was expected, but the test had an unexpected test outcome, check test output, Pass/FAIL need to match exactly)${COLOR_NORMAL}"
                else
                    TICKET="$(egrep "R: .*$ESCAPED_TEST [XS]FAIL" $OUTFILE | sed -e "s,.*[XS]FAIL/\(.*\),\1,")"
                    RESULT="$(egrep "R: .*$ESCAPED_TEST [XS]FAIL" $OUTFILE | sed -e "s,.*\([XS]FAIL\).*,\1,")"
                    if [ "$RESULT" = "XFAIL" ]
                    then
                        RESULT_MSG="${COLOR_WARNING}FAIL (Suppressed, $TICKET)${COLOR_NORMAL}"
                    else
                        RESULT_MSG="${COLOR_WARNING}Soft fail ($TICKET)${COLOR_NORMAL}"
                    fi
                fi
            elif [ $RETVAL -ne 0 ]
            then
                RESULT=FAIL
            elif egrep "R: .*$ESCAPED_TEST FAIL/no_ticket_number" $OUTFILE > /dev/null
            then
                RESULT=FAIL
                RESULT_MSG="${COLOR_FAILURE}FAIL (Tried to suppress failure, but no issue number is provided)${COLOR_NORMAL}"
            elif egrep "R: .*$ESCAPED_TEST Wait/[0-9]+" $OUTFILE > /dev/null
            then
                if [ -z "$NEXT_TIMEOUT_VAR" ]
                then
                    RESULT=FAIL
                    RESULT_MSG="${COLOR_FAILURE}FAIL (Test tried to wait but is not in \"timed\" directory)${COLOR_NORMAL}"
                else
                    WAIT_TIME=$(egrep "R: .*$ESCAPED_TEST Wait/[0-9]+" $OUTFILE | sed -e 's,.*Wait/\([0-9][0-9]*\).*,\1,')
                    eval $NEXT_TIMEOUT_VAR=$(($TEST_END_TIME+$WAIT_TIME))
                    RESULT=Wait
                    RESULT_MSG="Awaiting ($WAIT_TIME seconds)..."
                fi
            elif egrep "R: .*$ESCAPED_TEST Pass" $OUTFILE > /dev/null
            then
                RESULT=Pass
                RESULT_MSG="${COLOR_SUCCESS}Pass${COLOR_NORMAL}"
            elif egrep "R: .*$ESCAPED_TEST Skip/unsupported" $OUTFILE > /dev/null
            then
                RESULT=Skip
                RESULT_MSG="${COLOR_WARNING}Skipped (No platform support)${COLOR_NORMAL}"
            elif egrep "R: .*$ESCAPED_TEST Skip/needs_work" $OUTFILE > /dev/null
            then
                RESULT=Skip
                RESULT_MSG="${COLOR_WARNING}Skipped (Test needs work)${COLOR_NORMAL}"
            elif egrep "R: .*$ESCAPED_TEST FLAKEY" $OUTFILE > /dev/null
            then
                RESULT=FLAKEY
                TICKET="$(egrep "R: .*$ESCAPED_TEST FLAKEY" $OUTFILE | sed -e "s,.*FLAKEY/\(.*\),\1,")"
                RESULT_MSG="${COLOR_WARNING}Flakey fail ($TICKET)${COLOR_NORMAL}"

            else
                RESULT=FAIL
                RESULT_MSG="${COLOR_FAILURE}FAIL${COLOR_NORMAL}"
            fi
        fi

        if [ "x$RESULT_MSG" = "x" ]
        then
            RESULT_MSG=$RESULT
        fi

        if [ "$RESULT" = "FAIL" ] && [ -e .succeeded/"$FLATNAME" ]
        then
            RESULT_MSG="${COLOR_FAILURE}$RESULT_MSG (UNEXPECTED FAILURE)${COLOR_NORMAL}"
        fi
    fi

    if [ "$RESULT" = "XFAIL" -o "$RESULT" = "SFAIL" ]
    then
        echo "    <testcase name=\"$(basename $TEST)\""
        echo "              classname=\"$TEST $RESULT_MSG\""
        echo "              time=\"$(($TEST_END_TIME - $TEST_START_TIME)) seconds\">"
    elif [ "$RESULT" != Wait ]
    then
        echo "    <testcase name=\"$(basename $TEST)\""
        echo "              classname=\"$TEST\""
        echo "              time=\"$(($TEST_END_TIME - $TEST_START_TIME)) seconds\">"
    fi >> "$WORKDIR/$XML"


    # Fill test metadata if any
    if [ ! -z "$TEST_DESCRIPTION" ] || [ ! -z "$TEST_STORY" ] || [ ! -z "$TEST_COVERS" ]
    then
        if [ ! -z "$TEST_DESCRIPTION" ]
        then
            TEST_DESCRIPTION_METATAG="name=\"$TEST_DESCRIPTION\""
        fi
        if [ ! -z "$TEST_STORY" ]
        then
            TEST_STORY_METATAG="story_id=\"$TEST_STORY\""
        fi
        if [ ! -z "$TEST_COVERS" ]
        then
            TEST_COVERS_METATAG="covers=\"$TEST_COVERS\""
        fi
        echo "        <system-out>$TEST_DESCRIPTION_METATAG $TEST_STORY_METATAG $TEST_COVERS_METATAG</system-out>"
    fi >> "$WORKDIR/$XML"


    echo $RESULT $TEST >> "$WORKDIR/$SUMMARY"
    case "$RESULT" in
        Pass)
            PASSED_TESTS=$(($PASSED_TESTS + 1))

            mkdir -p '.succeeded'
            touch .succeeded/"$FLATNAME"
            ;;

        FAIL|XFAIL)
            ( echo "        <failure type=\"$RESULT\""
              echo "                 message=\"$RESULT_MSG $TEST\">"
              cat $OUTFILE | sed -e "s/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/'/\&quot;/g"
              echo "        </failure>"
            ) >> "$WORKDIR/$XML"
            FAILED_TESTS=$(($FAILED_TESTS + 1))
            if [ "$RESULT" = "XFAIL" ]
            then
                SUPPRESSED_FAILURES=$(($SUPPRESSED_FAILURES + 1))
            fi
            ;;

        SFAIL)
            ( echo "        <skipped type=\"$RESULT_MSG\">"
              echo "                 message=\"$RESULT_MSG $TEST\">"
              cat $OUTFILE | sed -e "s/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/'/\&quot;/g"
              echo "        </skipped>"
            ) >> "$WORKDIR/$XML"
            SOFT_FAILURES=$(($SOFT_FAILURES + 1))
            ;;

        Skip)
            ( echo "        <skipped type=\"$RESULT_MSG\">"
              echo "        </skipped>"
            ) >> "$WORKDIR/$XML"
            SKIPPED_TESTS=$(($SKIPPED_TESTS + 1))
            ;;
    esac

    if [ "$RESULT" != Wait ]
    then
        echo "    </testcase>" >> "$WORKDIR/$XML"
    fi

    if [ -e "$WORKDIR" -a "$RESULT" != "FAIL" -a "$RESULT" != "Wait" -a "$NO_CLEAN" = "0" ]
    then
        # Delete everything except the logs from the workdir.
        ls -1 "$WORKDIR" | while read s
        do
            case "$s" in
                "$LOG"|"$SUMMARY"|"$XML")
                ;;
                *)
                    $GAINROOT rm -fr "$WORKDIR/$s"
                    ;;
            esac
        done
        if [ "$GITHUB_ACTIONS" = "true" ] && [ "$RUNNER_OS" = "Windows" ]
        then
            # on Github Actions Windows runners, `rm -rf` moves files into /d/d/ directory.
            # To clean this "Recycle bin", we delete this directory.
            # Otherwise, runner promptly run out of disk space.
            rm -rf /d/d/
        fi
    fi

    if [ -z "$QUIET" ]
    then
        if [ -n "$STAY_IN_WORKDIR" ]
        then
            # See comment about STAY_IN_WORKDIR near start of runtest.
            echo "$TEST $RESULT_MSG"
        else
            echo $RESULT_MSG
        fi
    else
        if [ "$RESULT" = Pass ]
        then
            printf '.'
        elif [ "$RESULT" = Skip ]
        then
            printf '-'
        elif [ "$RESULT" = FAIL ]
        then
            if [ $TEST_TYPE = errorexit ]
            then
                printf '!'
            else
                printf 'x'
            fi
        fi
    fi

    (
        echo
        echo '  ==>' "$RESULT_MSG"
        echo
    ) >> "$WORKDIR/$LOG"
}


NO_LOCK="K"
ORIG_ARGS=
while true
do
    case "$1" in
        -h|--help)
            usage
            exit;;
        -q|--quiet)
            QUIET=1;;
        --gainroot=*)
            GAINROOT=${1#--gainroot=};;
        --valgrind)
            USE_VALGRIND=1;;
        --callgrind)
            USE_VALGRIND=1
            VALGRIND_OPTS="--suppressions=valgrind-suppressions --tool=callgrind";;
        --inotifywatch)
            USE_INOTIFYWATCH=1;;
        --inotifywait)
            USE_INOTIFYWAIT=1;;
        --tests=*)

            # Deselect all test types, in order to have only the ones
            # given as arguments
            COMMON_TESTS=0
            TIMED_TESTS=0
            SLOW_TESTS=0
            ERROREXIT_TESTS=0
            ERRORLOG_TESTS=0
            SERIAL_TESTS=0
            NETWORK_TESTS=0
            LIBXML2_TESTS=0
            LIBCURL_TESTS=0
            UNSAFE_TESTS=0
            STAGING_TESTS=0

            TESTS_SELECTED=${1#--tests=}
            for testtype in `echo $TESTS_SELECTED | tr , ' '`
            do
                case "$testtype" in
                    common)      COMMON_TESTS=1;;
                    timed)       TIMED_TESTS=1;;
                    slow)        SLOW_TESTS=1;;
                    errorexit)   ERROREXIT_TESTS=1;;
                    errorlog)    ERRORLOG_TESTS=1;;
                    serial)      SERIAL_TESTS=1;;
                    network)     NETWORK_TESTS=1;;
                    libxml2)     LIBXML2_TESTS=1;;
                    libcurl)     LIBCURL_TESTS=1;;
                    unsafe)      UNSAFE_TESTS=1;;
                    staging)     STAGING_TESTS=1;;
                esac
            done
            ;;

        --agent=*)
            AGENT=${1#--agent=};;
        --baseclasses=*)
            BASECLASSES=${1#--baseclasses=};;
        --bc=*)
            BASECLASSES=${1#--bc=};;
        --extraclasses=*)
            EXTRACLASSES=${1#--extraclasses=};;
        --ec=*)
            EXTRACLASSES=${1#--ec=};;
        --cfpromises=*)
            CF_PROMISES=${1#--cfpromises=};;
        --cfserverd=*)
            CF_SERVERD=${1#--cfserverd=};;
        --cfexecd=*)
            CF_EXECD=${1#--cfexecd=};;
        --cfkey=*)
            CF_KEY=${1#--cfkey=};;
        --cfkeycrypt=*)
            CF_SECRET=${1#--cfkeycrypt=};;
        --cfnet=*)
            CF_NET=${1#--cfnet=};;
        --cfcheck=*)
            CF_CHECK=${1#--cfcheck=};;
        --cfrunagent=*)
            CF_RUNAGENT=${1#--cfrunagent=};;
        --rpmvercmp=*)
            RPMVERCMP=${1#--rpmvercmp=};;
        --bindir=*)
            BINDIR=${1#--bindir=};;
        --libtool=*)
            LIBTOOL=${1#--libtool=};;
        --include=*)
            INCLUDE_IN_WORKDIR="$INCLUDE_IN_WORKDIR${INCLUDE_IN_WORKDIR:+ }${1#--include=}";;
        -j*|--jobs*)
            MAKEFLAGS="$MAKEFLAGS $1";;
        --printlog)
            PRINTLOG=1;;
        --gdb)
            GDB=1;;
        --no-clean)
            NO_CLEAN=1;;
        --verbose)
            VERBOSE="-v";;
        --debug)
           DEBUG="--debug";;
        --no-no-lock)
           NO_LOCK="";;
        --stay-in-workdir)
            # Internal option. Meant to keep sub invocations from interfering by
            # writing files only into the workdir.
            STAY_IN_WORKDIR=1;;
        --base-workdir=*)
            BASE_WORKDIR="${1#--base-workdir=}";;
        -*)
            echo "Unknown option: $1"
            exit 1;;
        *)
            break;;
    esac
    # Make sure spaces are preserved by escaping them.
    ORIG_ARGS="$ORIG_ARGS $(echo "$1" | sed -e 's/ /\\ /g')"
    shift
done

#
# Close stdin file descriptor to avoid tty_interactive check in cf-agent being success.
# Do this iff GDB AND Valgrind are not in use.
#
if [ "$GDB" != 1 ] && [ "$USE_VALGRIND" != 1 ]
then
    exec </dev/null
fi

if [ "$OSTYPE" = "msys" ] && [ "$TESTALL_DO_NOT_RECURSE" != 1 ] &&
       ! "$WHOAMI" -priv | grep SeTakeOwnershipPrivilege > /dev/null
then
    # On Windows we run the entire test run under GAINROOT, because doing it for
    # each test is horribly slow.
    echo "export GAINROOT=" > runtests.sh
    echo "export TESTALL_DO_NOT_RECURSE=1" >> runtests.sh
    echo "export DEBUG='$DEBUG'" >> runtests.sh
    echo "export VERBOSE='$VERBOSE'" >> runtests.sh
    echo "$0 $ORIG_ARGS $@ &" >> runtests.sh
    # Note quote change. We want to keep below variables unexpanded.
    echo 'trap "kill $!" INT' >> runtests.sh
    echo 'trap "kill $!" TERM' >> runtests.sh
    # Traps do not fire during commands, but *do* fire during wait.
    echo 'wait $!' >> runtests.sh
    $GAINROOT "./runtests.sh"
    exit $?
fi

# Check last -j flag, and check if it is -j1.
for arg in $MAKEFLAGS
do
    case "$arg" in
        -j|--jobs) PARALLEL=1 ;;
        -j*) if [ 1 -eq "${arg#-j}" ]
             then PARALLEL=
             else PARALLEL=1
             fi ;;
        --jobs=*) if [ 1 -eq "${arg#--jobs=}" ]
                  then PARALLEL=
                  else PARALLEL=1
                  fi ;;
    esac
done

# check if rpm is used on this host, sometimes it can be installed but not used say on Ubuntu
if [ -n "$(command -v rpm)" ] && [ "$(rpm -qa | wc -l)" != "0" ]
then
  NEED_RPMVERCMP="yes"
else
  NEED_RPMVERCMP="no"
fi

if [ -n "$AGENT" -o -n "$CF_PROMISES" -o -n "$CF_SERVERD" -o -n "$CF_EXECD" -o -n "$CF_KEY" -o -n "$CF_SECRET" -o -n "$CF_NET" -o -n "$CF_CHECK" -o -n "$CF_RUNAGENT" -o \( "$NEED_RPMVERCMP" = "yes" -a -n "$RPMVERCMP" \) -o -n "$DIFF" ]
then
    if [ -n "$BINDIR" ]
    then
        echo "--bindir is mutually exclusive with specifying individual binaries."
        exit 2
    fi
fi

# We assume we're running this script from one of the following
#   $objdir
#   $objdir/tests/acceptance
#   /var/cfengine/tests/acceptance or from
#   $core_objdir/../{enterprise,masterfiles}/tests/acceptance
find_default_binary()
{
    [ -x "`pwd`/../../../core/$2/$2" ] && eval $1=\""`pwd`/../../../core/$2/$2"\"
    [ -x "`pwd`/../../../core/ext/$2" ] && eval $1=\""`pwd`/../../../core/ext/$2"\"
    [ -x "`pwd`/../../ext/$2" ] && eval $1=\""`pwd`/../../ext/$2"\"
    [ -x "`pwd`/../../bin/$2" ] && eval $1=\""`pwd`/../../bin/$2"\"
    [ -x "`pwd`/../../$2/$2" ] && eval $1=\""`pwd`/../../$2/$2"\"
    [ -x "`pwd`/$2/$2" ] && eval $1=\""`pwd`/$2/$2"\"
    [ -n "$BINDIR" -a -x "$BINDIR/$2" ] && eval $1=\""$BINDIR/$2"\"
    [ $2 = "diff" ] && eval $1=\""`command -v diff`"\"
}
find_default_binary DEFAGENT cf-agent
find_default_binary DEFCF_PROMISES cf-promises
find_default_binary DEFCF_SERVERD cf-serverd
find_default_binary DEFCF_EXECD cf-execd
find_default_binary DEFCF_KEY cf-key
find_default_binary DEFCF_SECRET cf-secret
find_default_binary DEFCF_NET cf-net
find_default_binary DEFCF_CHECK cf-check
find_default_binary DEFCF_RUNAGENT cf-runagent
if [ "$NEED_RPMVERCMP" = "yes" ]
then
  find_default_binary DEFRPMVERCMP rpmvercmp
fi
find_default_binary DIFF diff

[ -x "`pwd`/libtool" ] && DEFLIBTOOL="`pwd`/libtool"
[ -x "`pwd`/../../libtool" ] && DEFLIBTOOL="`pwd`/../../libtool"

AGENT=${AGENT:-${DEFAGENT}}
CF_PROMISES=${CF_PROMISES:-${DEFCF_PROMISES}}
CF_SERVERD=${CF_SERVERD:-${DEFCF_SERVERD}}
CF_EXECD=${CF_EXECD:-${DEFCF_EXECD}}
CF_KEY=${CF_KEY:-${DEFCF_KEY}}
CF_SECRET=${CF_SECRET:-${DEFCF_SECRET}}
CF_NET=${CF_NET:-${DEFCF_NET}}
CF_CHECK=${CF_CHECK:-${DEFCF_CHECK}}
CF_RUNAGENT=${CF_RUNAGENT:-${DEFCF_RUNAGENT}}
RPMVERCMP=${RPMVERCMP:-${DEFRPMVERCMP}}
LIBTOOL=${LIBTOOL:-${DEFLIBTOOL}}

if [ ! -x "$AGENT" -o ! -x "$CF_PROMISES" -o ! -x "$CF_SERVERD" -o ! -x "$CF_EXECD" -o ! -x "$CF_KEY" -o ! -x "$CF_SECRET" -o ! -x "$CF_NET" -o ! -x "$CF_CHECK" -o ! -x "$CF_RUNAGENT" -o \( "$NEED_RPMVERCMP" = "yes" -a ! -x "$RPMVERCMP" \) ]
then
    echo "ERROR: can't find cf-agent or other binary;"     \
         " Are you sure you're running this from OBJDIR"   \
         " or {core,masterfiles,enterprise}_OBJDIR/tests/acceptance ?"
    echo '$AGENT        =' "$AGENT"
    echo '$CF_PROMISES  =' "$CF_PROMISES"
    echo '$CF_SERVERD   =' "$CF_SERVERD"
    echo '$CF_EXECD     =' "$CF_EXECD"
    echo '$CF_KEY       =' "$CF_KEY"
    echo '$CF_SECRET    =' "$CF_SECRET"
    echo '$CF_RUNAGENT  =' "$CF_RUNAGENT"
    echo '$CF_NET       =' "$CF_NET"
    echo '$CF_CHECK     =' "$CF_CHECK"
    echo '$RPMVERCMP    =' "$RPMVERCMP"
    exit 1
fi

if [ "$UNSAFE_TESTS" = "1" ]
then
    if [ "$GAINROOT" = "fakeroot" ]
    then
        echo "Unsafe tests do not play well together with fakeroot. Please use a different"
        echo "--gainroot (like \"sudo\"), or you will get incorrect results."
        exit 1
    fi

    # Make sure test dir is accessible to everyone, because unsafe tests may
    # switch user during the test (users promises do this).
    DIR="$(cd "$(dirname "$0")"; pwd)"
    while [ "$DIR" != "/" -a "$DIR" != "" ]
    do
        $GAINROOT chmod go+rx "$DIR"
        DIR="$(dirname "$DIR")"
    done
fi

if [ $# -gt 0 ]
then
    # We run all specified tests according to the defaults (no unsafe ones).
    for test in "$@"
    do
        if ! expr "$test" : '[/.]' >/dev/null
        then
            test="./$test"
        fi

        if [ -f $test ]
        then
            ALL_TESTS="$ALL_TESTS${ALL_TESTS:+ }$test"
        elif [ -d $test ]
        then
            ALL_TESTS="$ALL_TESTS${ALL_TESTS:+ }$(find "$test" -name workdir -prune -o -name '*.cf' -and -not -name '*.sub.cf' -print | sort)"
        else
            echo "Unable to open test file/directory: $test"
        fi
    done
else
    ALL_TESTS="$ALL_TESTS${ALL_TESTS:+ }$(find . \( -name selftest -o -name workdir \) -prune -o -name '*.cf' -and -not -name '*.sub.cf' -print | sort)"
fi

for addtest in $ALL_TESTS
do
    if echo "$addtest" | fgrep "/timed/" > /dev/null
    then
        if [ "$TIMED_TESTS" = 1 ]
        then
            eval TESTS_TIMED_$TEST_TIMED_INDEX="$addtest"
            eval TESTS_TIMEOUT_$TEST_TIMED_INDEX=0
            eval TESTS_PASSES_$TEST_TIMED_INDEX=0
            TEST_TIMED_INDEX=$(($TEST_TIMED_INDEX+1))
        fi
    else
        eval TESTS_$TEST_INDEX="$addtest"
        TEST_INDEX=$(($TEST_INDEX+1))
    fi
done

TESTS_NORMAL_COUNT=$TEST_INDEX
TESTS_TIMED_COUNT=$TEST_TIMED_INDEX
TESTS_COUNT=$(($TESTS_NORMAL_COUNT + $TESTS_TIMED_COUNT))
TESTS_TIMED_REMAINING=$TEST_TIMED_INDEX

#
# fd 7 is a /dev/null for quiet execution and stdout for default one
#
if [ -z "$QUIET" ]
then
    exec 7>&1
else
    exec 7>/dev/null
fi

print_header() {
    ( echo ======================================================================
      echo Testsuite started at $(date "+%Y-%m-%d %T")
      echo ----------------------------------------------------------------------
      echo Total tests: $TESTS_COUNT
      echo
      for feature in COMMON_TESTS TIMED_TESTS SLOW_TESTS ERROREXIT_TESTS SERIAL_TESTS NETWORK_TESTS LIBXML2_TESTS LIBCURL_TESTS UNSAFE_TESTS STAGING_TESTS
      do
          if eval "[ \${$feature} != 0 ]"
          then
              printf '%20s: %s\n' $feature enabled
          else
              printf '%20s: %s\n' $feature disabled
          fi
      done
      echo
      if [ x$PARALLEL = x1 ]
      then
          echo "Test run is PARALLEL with MAKEFLAGS=$MAKEFLAGS"
      else
          echo "Test run is not parallel ${MAKEFLAGS:+(MAKEFLAGS=$MAKEFLAGS)}"
      fi
      echo
    ) | tee "$LOG" | tee "$SUMMARY" >&7
}

print_footer() {
    # Recalculate results since sub invocations may not have been recorded.
    PASSED_TESTS=`egrep '^Pass ' $SUMMARY | wc -l | tr -d ' '`
    FAILED_TESTS=`egrep '^(FAIL|XFAIL) ' $SUMMARY | wc -l | tr -d ' '`
    SUPPRESSED_FAILURES=`egrep '^XFAIL ' $SUMMARY | wc -l | tr -d ' '`
    SOFT_FAILURES=`egrep '^SFAIL ' $SUMMARY | wc -l | tr -d ' '`
    FLAKEY_FAILURES=`egrep '^FLAKEY ' $SUMMARY | wc -l | tr -d ' '`
    SKIPPED_TESTS=`egrep '^Skip ' $SUMMARY | wc -l | tr -d ' '`

    ( echo
      echo ======================================================================
      echo "Testsuite finished at $(date  "+%F %T") ($(($END_TIME - $START_TIME)) seconds)"
    ) | tee -a "$LOG" | tee -a "$SUMMARY" >&7

    ( echo
      echo   "Passed tests:    $PASSED_TESTS"
      printf "Failed tests:    $FAILED_TESTS"
      if [ $SUPPRESSED_FAILURES -gt 0 ]
      then
          echo " ($SUPPRESSED_FAILURES are known and suppressed)"
      else
          echo
      fi
      echo   "Skipped tests:   $SKIPPED_TESTS"
      echo   "Soft failures:   $SOFT_FAILURES"
      echo   "Flakey failures: $FLAKEY_FAILURES"
      echo   "Total tests:     $TESTS_COUNT"
    ) | tee -a "$LOG" | tee -a "$SUMMARY"

    if [ -n "$PRINTLOG" ]
    then
        cat "$LOG"
    fi
}

finish_xml() {
    mv "$XML" xml.tmp
    (
        cat <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="$(pwd)"
           timestamp="$(date "+%F %T")"
           hostname="localhost"
           tests="$TESTS_COUNT"
           failures="$FAILED_TESTS"
           skipped="$SKIPPED_TESTS"
           time="$(($END_TIME - $START_TIME)) seconds">
EOF
        cat xml.tmp
        cat <<EOF
</testsuite>
EOF
    ) > "$XML"
    rm -f xml.tmp
}

collect_results()
{
    if [ -n "$STAY_IN_WORKDIR" ]
    then
        return 0
    fi

    WORKDIR="$(workdir "$1")"
    for file in "$LOG" "$SUMMARY" "$XML"
    do
        if [ -e "$WORKDIR/$file" ]
        then
            cat "$WORKDIR/$file" >> "$file"
            if [ "$NO_CLEAN" = "0" ]
            then
                # For unknown reason after the second pass, these file are root-owned, so we need GAINROOT here
                $GAINROOT rm -f "$WORKDIR/$file"
            fi
        fi
    done
    rmdir "$WORKDIR" >/dev/null 2>&1 || true
}

check_and_run_timed_tests() {
    TEST_TIMED_INDEX=0
    time=$(unix_seconds)
    # Run timed tests if any deadlines have expired.
    while [ $TEST_TIMED_INDEX -lt $TESTS_TIMED_COUNT ]
    do
        eval test=\$TESTS_TIMED_$TEST_TIMED_INDEX
        eval timeout=\$TESTS_TIMEOUT_$TEST_TIMED_INDEX
        if [ -n "$timeout" ] && [ "$time" -ge "$timeout" ]
        then
            eval TESTS_PASSES_$TEST_TIMED_INDEX="\$((\$TESTS_PASSES_$TEST_TIMED_INDEX+1))"
            eval pass=\$TESTS_PASSES_$TEST_TIMED_INDEX
            runtest "$AGENT" "$test" "$pass" "TESTS_TIMEOUT_$TEST_TIMED_INDEX"
            collect_results "$test"
            eval timeout=\$TESTS_TIMEOUT_$TEST_TIMED_INDEX
            if [ -z "$timeout" ]
            then
                TESTS_TIMED_REMAINING=$(($TESTS_TIMED_REMAINING - 1))
            fi
        fi
        TEST_TIMED_INDEX=$(($TEST_TIMED_INDEX+1))
    done
}

run_all_tests() {
    TEST_INDEX=0
    while [ $TEST_INDEX -lt $TESTS_NORMAL_COUNT -o $TESTS_TIMED_REMAINING -gt 0 ]
    do
        check_and_run_timed_tests

        # Run normal test.
        if [ $TEST_INDEX -lt $TESTS_NORMAL_COUNT ]
        then
            eval test=\$TESTS_$TEST_INDEX
            runtest "$AGENT" "$test"
            collect_results "$test"
            TEST_INDEX=$(($TEST_INDEX+1))
        elif [ $TESTS_TIMED_REMAINING -gt 0 ]
        then
            sleep 1
        fi
    done
}


# TODO this function is currently creating a Makefile with all possible
#      tests, either they will run or be skipped. The proper way is to
#      do all the checks that now are in runtest(), and add the test
#      here only if it will run. This should be done higher up, where
#      ALL_TESTS is populated.
#
#      When this is done, there won't be a need to pass --tests=... down
#      the makefile subexecutions.

run_all_tests_using_make() {
    MAKEFILE=Makefile.testall

    TEST_BLOCKS=1
    LAST_WAS_SERIAL=0

    for curr_test in $ALL_TESTS
    do
        # Keep serial tests after preceding tests and before following tests.
        case "$curr_test" in
            */serial_*|*_serial_*|*_serial.*|*/serial/*|*/unsafe/*)
                if [ $LAST_WAS_SERIAL = 0 ]
                then
                    TEST_BLOCKS=$(($TEST_BLOCKS + 1))
                    LAST_WAS_SERIAL=1
                fi
                ;;
            *)
                if [ $LAST_WAS_SERIAL = 1 ]
                then
                    TEST_BLOCKS=$(($TEST_BLOCKS + 1))
                    LAST_WAS_SERIAL=0
                fi
                ;;
        esac
        case "$curr_test" in
            */serial_*|*_serial_*|*_serial.*|*/serial/*|*/unsafe/*|*/timed/*|*/slow/*)
                eval MAKE_RECIPE_LIST$TEST_BLOCKS='$MAKE_RECIPE_LIST'$TEST_BLOCKS'${MAKE_RECIPE_LIST'$TEST_BLOCKS':+ }$curr_test'
                ;;
            *)
                # Separate make target list (notice the "_rule"), because we do not want the target to be a
                # file name, since make will skip it if it exists (which of course it does).
                eval MAKE_TARGET_LIST$TEST_BLOCKS='$MAKE_TARGET_LIST'$TEST_BLOCKS'${MAKE_TARGET_LIST'$TEST_BLOCKS':+ }${curr_test}_rule'
                ;;
        esac
    done

    # Redirect to makefile all at once.
    (
        printf "tests:\n\n"
        printf "parallel_block0:\n\n"
        printf "serial_block0:\n\n"
        printf "tests: serial_block0 parallel_block0\n\n"

        i=1
        while [ $i -le $TEST_BLOCKS ]
        do
            if eval test -n '"$MAKE_RECIPE_LIST'$i'"'
            then
                eval printf '"serial_block$i:\n\t@-$0 --stay-in-workdir -j1 ${TESTS_SELECTED:+--tests=$TESTS_SELECTED} $MAKE_RECIPE_LIST'$i'\n\n"'
            fi
            printf "serial_block$i: serial_block$(($i-1)) parallel_block$(($i-1))\n\n"
            eval printf '"parallel_block$i: $MAKE_TARGET_LIST'$i'\n\n"'
            if eval test -n '"$MAKE_TARGET_LIST'$i'"'
            then
                eval printf '"$MAKE_TARGET_LIST'$i': serial_block$(($i-1)) parallel_block$(($i-1))\n\n"'
            fi
            printf "tests: serial_block$i parallel_block$i\n\n"

            i=$(($i + 1))
        done

        printf "%%_rule: %%\n\t@-$0 --stay-in-workdir -j1 ${TESTS_SELECTED:+--tests=$TESTS_SELECTED} \$<\n\n"
    ) > $MAKEFILE

    ${MAKE:-make} -k -f $MAKEFILE

    for curr_test in $ALL_TESTS
    do
        collect_results "$curr_test"
    done
}



#
# ========== RUN THE TESTS ==========
#

# isEnabled? USE_INOTIFYWATCH
if [ -n "$USE_INOTIFYWATCH" ]
then
    if which inotifywatch 2>&1 > /dev/null
    then
        mkdir -p "$BASE_WORKDIR" 2> /dev/null
        inotifywatch ${INOTIFYWATCH_OPTS} $BASE_WORKDIR < /dev/null 2>&1 > ${INOTIFYWATCH_LOG} &
        INOTIFYWATCH_PID=$!
    else
        echo "info: inotifywatch not detected."
        exit 0
    fi
fi

# isEnabled? USE_INOTIFYWAIT
if [ -n "$USE_INOTIFYWAIT" ]
then
    if which inotifywait 2>&1 > /dev/null
    then
        mkdir -p "$BASE_WORKDIR" 2> /dev/null
        inotifywait ${INOTIFYWAIT_OPTS} $BASE_WORKDIR < /dev/null 2>&1 > ${INOTIFYWAIT_LOG} &
        INOTIFYWAIT_PID=$!
    else
        echo "info: inotifywait not detected."
        exit 0
    fi
fi

trap_handler() {
    [ -n "$INOTIFYWATCH_PID" ] && kill -9 ${INOTIFYWATCH_PID} 2>&1 > /dev/null
    [ -n "$INOTIFYWAIT_PID"  ] && kill -9 ${INOTIFYWAIT_PID}  2>&1 > /dev/null
    exit 0
}
if [ -n "$USE_INOTIFYWATCH" -o -n "$USE_INOTIFYWAIT"  ]
then
    trap trap_handler EXIT INT TERM
fi

START_TIME=$(unix_seconds)
if [ -z "$STAY_IN_WORKDIR" ]
then
    # This is top level invocation.
    print_header
    rm -f "$XML"
fi
if [ -z "$PARALLEL" -o "$TESTS_COUNT" -eq 1 ]
then
    run_all_tests
else
    run_all_tests_using_make
fi
END_TIME=$(unix_seconds)
if [ -z "$STAY_IN_WORKDIR" ]
then
    # This is top level invocation.
    print_footer
    finish_xml
fi

if [ $(($PASSED_TESTS+$FAILED_TESTS+$SOFT_FAILURES+$SKIPPED_TESTS+$FLAKEY_FAILURES)) -ne $TESTS_COUNT ]
then
    echo "WARNING: Number of test results does not match number of tests!"
    echo "Did something go wrong during execution?"
    exit 2
fi
if [ "$FAILED_TESTS" -gt "$SUPPRESSED_FAILURES" ]
then
    exit 1
elif [ "$FLAKEY_FAILURES" -gt "0" ] && [ -n "$FLAKEY_IS_FAIL" ]
then
    exit 4
else
    exit 0
fi