Test tools, verification and some bug fixes

pyperf/pyperf_run

@@ -25,6 +25,10 @@ PYPERF_VERSION="1.11.0"
 copy_dirs=""
 benchmarks="all"
 test_name="pyperf"
+rtc=0
+
+TOOLS_BIN="$HOME/test_tools"
+export TOOLS_BIN
 
 base_dir=$(dirname $(realpath $0))
 #
@@ -73,6 +77,30 @@ verify_benchmark_names()
 	fi
 }
 
+setup_pyperformance()
+{
+	major=$(echo $PYPERF_VERSION | cut -d. -f1)
+	minor=$(echo $PYPERF_VERSION | cut -d. -f2)
+
+	$python_exec -m pyperformance venv create
+	venv_path=$($python_exec -m pyperformance venv show | sed -e "s/Virtual environment path: //g" -e "s/ (already created)//" )
+
+	if [[ -z "$venv_path" ]]; then
+		exit_out "Error: Could not determine pyperformance venv path, exiting" 1
+	fi
+	if [[ ! -x "$venv_path/bin/python3" ]]; then
+		exit_out "Error: Could not find interpreter at $venv_path/bin/python3, exiting" 1
+	fi
+
+	# Revert to setuptools v81.0.0 due to `pkg_resources` removal at v82.0.0
+	# https://setuptools.pypa.io/en/stable/history.html#deprecations-and-removals
+	# Skipping using pkg_tool to ensure that all relevant venv PATHs are honored
+	if [[ "$major" -le 1 ]] && [[ "$minor" -le 11 ]]; then
+		echo "Detected <=1.11.0, applying setuptools fix"
+		"$venv_path/bin/python3" -m pip install --upgrade setuptools==81.0.0 # Execute venv pip instead
+	fi
+}
+
 install_tools()
 {
 	show_usage=0
@@ -104,8 +132,8 @@ install_tools()
 	# Check to see if the test tools directory exists.  If it does, we do not need to
 	# clone the repo.
 	#
-	if [ ! -d "test_tools" ]; then
-		git clone $tools_git test_tools
+	if [ ! -d "$TOOLS_BIN" ]; then
+		git clone $tools_git "$TOOLS_BIN"
 		if [ $? -ne 0 ]; then
 			exit_out "pulling git $tools_git failed." 1
 		fi
@@ -243,24 +271,9 @@ else
 fi
 
 # Gather hardware information
-${curdir}/test_tools/gather_data ${curdir}
-
-
-if [ ! -f "/tmp/pyperf.out" ]; then
-        command="${0} $@"
-        echo $command
-        $command 2>&1 | tee /tmp/pyperf.out
-	rtc=$?
-	if [ -f /tmp/pyperf.out ]; then
-		echo =================================
-		echo Output from the test.
-		echo =================================
-        	cat /tmp/pyperf.out
-        	rm /tmp/pyperf.out
-	fi
-        exit $rtc
-fi
+$TOOLS_BIN/gather_data ${curdir}
 
+exec &> >(tee /tmp/pyperf.out)
 
 if [ -d "workloads" ]; then
 	#
@@ -277,7 +290,7 @@ if [ -d "workloads" ]; then
 	done
 fi
 
-source test_tools/general_setup "$@"
+source $TOOLS_BIN/general_setup "$@"
 
 ARGUMENT_LIST=(
 	"pyperf_version"
@@ -389,6 +402,9 @@ if [[ "$benchmarks" != "all" ]]; then
 	verify_benchmark_names
 	pyperf_flags="-b $benchmarks"
 fi
+
+setup_pyperformance
+
 start_time=$(retrieve_time_stamp)
 $python_exec -m pyperformance run --output  ${pyresults}.json $pyperf_flags
 if [ $? -ne 0 ]; then
@@ -406,6 +422,10 @@ fi
 
 generate_csv_file ${pyresults} ${start_time} ${end_time}
 
+$TOOLS_BIN/csv_to_json $to_json_flags --csv_file ${pyresults}.csv --output_file pyperf.json
+$TOOLS_BIN/verify_results $to_verify_flags --file pyperf.json --schema_file $to_script_dir/../pyperf_schema.py --class_name PyperfResults
+rtc=$?
+
 if [[ $to_use_pcp -eq 1 ]]; then
 	stop_pcp_subset
 	stop_pcp
@@ -420,5 +440,5 @@ fi
 #
 # Process the data.
 #
-${curdir}/test_tools/save_results --curdir $curdir --home_root $to_home_root --results /tmp/pyperf.out  --test_name pyperf --tuned_setting=$to_tuned_setting --version NONE --user $to_user --other_files "python_results/*,test_results_report" $copy_dirs
-exit 0
+${TOOLS_BIN}/save_results --curdir $curdir --home_root $to_home_root --results /tmp/pyperf.out  --test_name pyperf --tuned_setting=$to_tuned_setting --version NONE --user $to_user --other_files "python_results/*,test_results_report" $copy_dirs
+exit $rtc

pyperf_schema.py

@@ -0,0 +1,124 @@
+import pydantic
+import datetime
+from enum import Enum
+
+class TestNames(Enum):
+    two_to_three = "2to3"
+    async_generators = "async_generators"
+    async_tree_none = "async_tree_none"
+    async_tree_cpu_io_mixed = "async_tree_cpu_io_mixed"
+    async_tree_cpu_io_mixed_tg = "async_tree_cpu_io_mixed_tg"
+    async_tree_eager = "async_tree_eager"
+    async_tree_eager_cpu_io_mixed = "async_tree_eager_cpu_io_mixed"
+    async_tree_eager_cpu_io_mixed_tg = "async_tree_eager_cpu_io_mixed_tg"
+    async_tree_eager_io = "async_tree_eager_io"
+    async_tree_eager_io_tg = "async_tree_eager_io_tg"
+    async_tree_eager_memoization = "async_tree_eager_memoization"
+    async_tree_eager_memoization_tg = "async_tree_eager_memoization_tg"
+    async_tree_eager_tg = "async_tree_eager_tg"
+    async_tree_io = "async_tree_io"
+    async_tree_io_tg = "async_tree_io_tg"
+    async_tree_memoization = "async_tree_memoization"
+    async_tree_memoization_tg = "async_tree_memoization_tg"
+    async_tree_none_tg = "async_tree_none_tg"
+    asyncio_tcp = "asyncio_tcp"
+    asyncio_tcp_ssl = "asyncio_tcp_ssl"
+    asyncio_websockets = "asyncio_websockets"
+    chameleon = "chameleon"
+    chaos = "chaos"
+    comprehensions = "comprehensions"
+    bench_mp_pool = "bench_mp_pool"
+    bench_thread_pool = "bench_thread_pool"
+    coroutines = "coroutines"
+    coverage = "coverage"
+    crypto_pyaes = "crypto_pyaes"
+    dask = "dask"
+    deepcopy = "deepcopy"
+    deepcopy_reduce = "deepcopy_reduce"
+    deepcopy_memo = "deepcopy_memo"
+    deltablue = "deltablue"
+    django_template = "django_template"
+    docutils = "docutils"
+    dulwich_log = "dulwich_log"
+    fannkuch = "fannkuch"
+    float_test = "float"
+    create_gc_cycles = "create_gc_cycles"
+    gc_traversal = "gc_traversal"
+    generators = "generators"
+    genshi_text = "genshi_text"
+    genshi_xml = "genshi_xml"
+    go = "go"
+    hexiom = "hexiom"
+    html5lib = "html5lib"
+    json_dumps = "json_dumps"
+    json_loads = "json_loads"
+    logging_format = "logging_format"
+    logging_silent = "logging_silent"
+    logging_simple = "logging_simple"
+    mako = "mako"
+    mdp = "mdp"
+    meteor_contest = "meteor_contest"
+    nbody = "nbody"
+    nqueens = "nqueens"
+    pathlib = "pathlib"
+    pickle = "pickle"
+    pickle_dict = "pickle_dict"
+    pickle_list = "pickle_list"
+    pickle_pure_python = "pickle_pure_python"
+    pidigits = "pidigits"
+    pprint_safe_repr = "pprint_safe_repr"
+    pprint_pformat = "pprint_pformat"
+    pyflate = "pyflate"
+    python_startup = "python_startup"
+    python_startup_no_site = "python_startup_no_site"
+    raytrace = "raytrace"
+    regex_compile = "regex_compile"
+    regex_dna = "regex_dna"
+    regex_effbot = "regex_effbot"
+    regex_v8 = "regex_v8"
+    richards = "richards"
+    richards_super = "richards_super"
+    scimark_fft = "scimark_fft"
+    scimark_lu = "scimark_lu"
+    scimark_monte_carlo = "scimark_monte_carlo"
+    scimark_sor = "scimark_sor"
+    scimark_sparse_mat_mult = "scimark_sparse_mat_mult"
+    spectral_norm = "spectral_norm"
+    sqlalchemy_declarative = "sqlalchemy_declarative"
+    sqlalchemy_imperative = "sqlalchemy_imperative"
+    sqlglot_normalize = "sqlglot_normalize"
+    sqlglot_optimize = "sqlglot_optimize"
+    sqlglot_parse = "sqlglot_parse"
+    sqlglot_transpile = "sqlglot_transpile"
+    sqlite_synth = "sqlite_synth"
+    sympy_expand = "sympy_expand"
+    sympy_integrate = "sympy_integrate"
+    sympy_sum = "sympy_sum"
+    sympy_str = "sympy_str"
+    telco = "telco"
+    tomli_loads = "tomli_loads"
+    tornado_http = "tornado_http"
+    typing_runtime_protocols = "typing_runtime_protocols"
+    unpack_sequence = "unpack_sequence"
+    unpickle = "unpickle"
+    unpickle_list = "unpickle_list"
+    unpickle_pure_python = "unpickle_pure_python"
+    xml_etree_parse = "xml_etree_parse"
+    xml_etree_iterparse = "xml_etree_iterparse"
+    xml_etree_generate = "xml_etree_generate"
+    xml_etree_process = "xml_etree_process"
+
+
+class DurationUnits(Enum):
+    second = "sec"
+    millisecond = "ms"
+    microsecond = "us"
+    nanosecond = "ns"
+
+
+class PyperfResults(pydantic.BaseModel):
+    Test: TestNames
+    Avg: float = pydantic.Field(gt=0, allow_inf_nan=False)
+    Unit: DurationUnits
+    Start_Date: datetime.datetime
+    End_Date: datetime.datetime