diff --git a/pyperf/pyperf_run b/pyperf/pyperf_run index 9e3463c..e178597 100755 --- a/pyperf/pyperf_run +++ b/pyperf/pyperf_run @@ -25,6 +25,10 @@ PYPERF_VERSION="1.11.0" copy_dirs="" benchmarks="all" test_name="pyperf" +rtc=0 + +TOOLS_BIN="$HOME/test_tools" +export TOOLS_BIN base_dir=$(dirname $(realpath $0)) # @@ -73,6 +77,30 @@ verify_benchmark_names() fi } +setup_pyperformance() +{ + major=$(echo $PYPERF_VERSION | cut -d. -f1) + minor=$(echo $PYPERF_VERSION | cut -d. -f2) + + $python_exec -m pyperformance venv create + venv_path=$($python_exec -m pyperformance venv show | sed -e "s/Virtual environment path: //g" -e "s/ (already created)//" ) + + if [[ -z "$venv_path" ]]; then + exit_out "Error: Could not determine pyperformance venv path, exiting" 1 + fi + if [[ ! -x "$venv_path/bin/python3" ]]; then + exit_out "Error: Could not find interpreter at $venv_path/bin/python3, exiting" 1 + fi + + # Revert to setuptools v81.0.0 due to `pkg_resources` removal at v82.0.0 + # https://setuptools.pypa.io/en/stable/history.html#deprecations-and-removals + # Skipping using pkg_tool to ensure that all relevant venv PATHs are honored + if [[ "$major" -le 1 ]] && [[ "$minor" -le 11 ]]; then + echo "Detected <=1.11.0, applying setuptools fix" + "$venv_path/bin/python3" -m pip install --upgrade setuptools==81.0.0 # Execute venv pip instead + fi +} + install_tools() { show_usage=0 @@ -104,8 +132,8 @@ install_tools() # Check to see if the test tools directory exists. If it does, we do not need to # clone the repo. # - if [ ! -d "test_tools" ]; then - git clone $tools_git test_tools + if [ ! -d "$TOOLS_BIN" ]; then + git clone $tools_git "$TOOLS_BIN" if [ $? -ne 0 ]; then exit_out "pulling git $tools_git failed." 1 fi @@ -243,24 +271,9 @@ else fi # Gather hardware information -${curdir}/test_tools/gather_data ${curdir} - - -if [ ! -f "/tmp/pyperf.out" ]; then - command="${0} $@" - echo $command - $command 2>&1 | tee /tmp/pyperf.out - rtc=$? - if [ -f /tmp/pyperf.out ]; then - echo ================================= - echo Output from the test. - echo ================================= - cat /tmp/pyperf.out - rm /tmp/pyperf.out - fi - exit $rtc -fi +$TOOLS_BIN/gather_data ${curdir} +exec &> >(tee /tmp/pyperf.out) if [ -d "workloads" ]; then # @@ -277,7 +290,7 @@ if [ -d "workloads" ]; then done fi -source test_tools/general_setup "$@" +source $TOOLS_BIN/general_setup "$@" ARGUMENT_LIST=( "pyperf_version" @@ -389,6 +402,9 @@ if [[ "$benchmarks" != "all" ]]; then verify_benchmark_names pyperf_flags="-b $benchmarks" fi + +setup_pyperformance + start_time=$(retrieve_time_stamp) $python_exec -m pyperformance run --output ${pyresults}.json $pyperf_flags if [ $? -ne 0 ]; then @@ -406,6 +422,10 @@ fi generate_csv_file ${pyresults} ${start_time} ${end_time} +$TOOLS_BIN/csv_to_json $to_json_flags --csv_file ${pyresults}.csv --output_file pyperf.json +$TOOLS_BIN/verify_results $to_verify_flags --file pyperf.json --schema_file $to_script_dir/../pyperf_schema.py --class_name PyperfResults +rtc=$? + if [[ $to_use_pcp -eq 1 ]]; then stop_pcp_subset stop_pcp @@ -420,5 +440,5 @@ fi # # Process the data. # -${curdir}/test_tools/save_results --curdir $curdir --home_root $to_home_root --results /tmp/pyperf.out --test_name pyperf --tuned_setting=$to_tuned_setting --version NONE --user $to_user --other_files "python_results/*,test_results_report" $copy_dirs -exit 0 +${TOOLS_BIN}/save_results --curdir $curdir --home_root $to_home_root --results /tmp/pyperf.out --test_name pyperf --tuned_setting=$to_tuned_setting --version NONE --user $to_user --other_files "python_results/*,test_results_report" $copy_dirs +exit $rtc diff --git a/pyperf_schema.py b/pyperf_schema.py new file mode 100644 index 0000000..438f5a3 --- /dev/null +++ b/pyperf_schema.py @@ -0,0 +1,124 @@ +import pydantic +import datetime +from enum import Enum + +class TestNames(Enum): + two_to_three = "2to3" + async_generators = "async_generators" + async_tree_none = "async_tree_none" + async_tree_cpu_io_mixed = "async_tree_cpu_io_mixed" + async_tree_cpu_io_mixed_tg = "async_tree_cpu_io_mixed_tg" + async_tree_eager = "async_tree_eager" + async_tree_eager_cpu_io_mixed = "async_tree_eager_cpu_io_mixed" + async_tree_eager_cpu_io_mixed_tg = "async_tree_eager_cpu_io_mixed_tg" + async_tree_eager_io = "async_tree_eager_io" + async_tree_eager_io_tg = "async_tree_eager_io_tg" + async_tree_eager_memoization = "async_tree_eager_memoization" + async_tree_eager_memoization_tg = "async_tree_eager_memoization_tg" + async_tree_eager_tg = "async_tree_eager_tg" + async_tree_io = "async_tree_io" + async_tree_io_tg = "async_tree_io_tg" + async_tree_memoization = "async_tree_memoization" + async_tree_memoization_tg = "async_tree_memoization_tg" + async_tree_none_tg = "async_tree_none_tg" + asyncio_tcp = "asyncio_tcp" + asyncio_tcp_ssl = "asyncio_tcp_ssl" + asyncio_websockets = "asyncio_websockets" + chameleon = "chameleon" + chaos = "chaos" + comprehensions = "comprehensions" + bench_mp_pool = "bench_mp_pool" + bench_thread_pool = "bench_thread_pool" + coroutines = "coroutines" + coverage = "coverage" + crypto_pyaes = "crypto_pyaes" + dask = "dask" + deepcopy = "deepcopy" + deepcopy_reduce = "deepcopy_reduce" + deepcopy_memo = "deepcopy_memo" + deltablue = "deltablue" + django_template = "django_template" + docutils = "docutils" + dulwich_log = "dulwich_log" + fannkuch = "fannkuch" + float_test = "float" + create_gc_cycles = "create_gc_cycles" + gc_traversal = "gc_traversal" + generators = "generators" + genshi_text = "genshi_text" + genshi_xml = "genshi_xml" + go = "go" + hexiom = "hexiom" + html5lib = "html5lib" + json_dumps = "json_dumps" + json_loads = "json_loads" + logging_format = "logging_format" + logging_silent = "logging_silent" + logging_simple = "logging_simple" + mako = "mako" + mdp = "mdp" + meteor_contest = "meteor_contest" + nbody = "nbody" + nqueens = "nqueens" + pathlib = "pathlib" + pickle = "pickle" + pickle_dict = "pickle_dict" + pickle_list = "pickle_list" + pickle_pure_python = "pickle_pure_python" + pidigits = "pidigits" + pprint_safe_repr = "pprint_safe_repr" + pprint_pformat = "pprint_pformat" + pyflate = "pyflate" + python_startup = "python_startup" + python_startup_no_site = "python_startup_no_site" + raytrace = "raytrace" + regex_compile = "regex_compile" + regex_dna = "regex_dna" + regex_effbot = "regex_effbot" + regex_v8 = "regex_v8" + richards = "richards" + richards_super = "richards_super" + scimark_fft = "scimark_fft" + scimark_lu = "scimark_lu" + scimark_monte_carlo = "scimark_monte_carlo" + scimark_sor = "scimark_sor" + scimark_sparse_mat_mult = "scimark_sparse_mat_mult" + spectral_norm = "spectral_norm" + sqlalchemy_declarative = "sqlalchemy_declarative" + sqlalchemy_imperative = "sqlalchemy_imperative" + sqlglot_normalize = "sqlglot_normalize" + sqlglot_optimize = "sqlglot_optimize" + sqlglot_parse = "sqlglot_parse" + sqlglot_transpile = "sqlglot_transpile" + sqlite_synth = "sqlite_synth" + sympy_expand = "sympy_expand" + sympy_integrate = "sympy_integrate" + sympy_sum = "sympy_sum" + sympy_str = "sympy_str" + telco = "telco" + tomli_loads = "tomli_loads" + tornado_http = "tornado_http" + typing_runtime_protocols = "typing_runtime_protocols" + unpack_sequence = "unpack_sequence" + unpickle = "unpickle" + unpickle_list = "unpickle_list" + unpickle_pure_python = "unpickle_pure_python" + xml_etree_parse = "xml_etree_parse" + xml_etree_iterparse = "xml_etree_iterparse" + xml_etree_generate = "xml_etree_generate" + xml_etree_process = "xml_etree_process" + + +class DurationUnits(Enum): + second = "sec" + millisecond = "ms" + microsecond = "us" + nanosecond = "ns" + + +class PyperfResults(pydantic.BaseModel): + Test: TestNames + Avg: float = pydantic.Field(gt=0, allow_inf_nan=False) + Unit: DurationUnits + Start_Date: datetime.datetime + End_Date: datetime.datetime