feat(GHA): distribute tests based on runtime

Author: vringar

.github/workflows/run-tests.yaml

@@ -29,19 +29,20 @@ jobs:
       matrix:
         test-groups:
           [
-            "test/test_[a-e]*",
-            "test/test_[f-h]*",
-            "test/test_[i-o,q-r,t-z]*",
-            "test/test_[p]*",
-            "test/test_[s]*",
-            "test/storage/*",
-            "test/extension/*",
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            "other"
           ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
-      - run: ./scripts/ci.sh
+      - run: python ./scripts/ci.py ./scripts/distribution.json
         env:
           DISPLAY: ":99.0"
           TESTS: ${{ matrix.test-groups }}

junit-report.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="8" time="547.997" timestamp="2024-08-06T23:49:05.138967" hostname="SZ1"><testcase classname="test.test_profile" name="test_profile_recovery[on_crash_during_launch-stateful-without_seed_tar]" time="143.851" /><testcase classname="test.test_profile" name="test_profile_recovery[on_crash_during_launch-stateless-with_seed_tar]" time="138.431" /><testcase classname="test.test_profile" name="test_profile_recovery[on_crash_during_launch-stateful-with_seed_tar]" time="123.576" /><testcase classname="test.test_profile" name="test_profile_saved_when_launch_crashes" time="118.191" /><testcase classname="test.test_simple_commands" name="test_save_screenshot_valid[headless]" time="23.129" /><testcase classname="test.test_profile" name="test_load_tar_file" time="0.261" /><testcase classname="test.test_js_instrument_py" name="test_validate_bad__log_settings_missing" time="0.024" /><testcase classname="test.test_js_instrument_py" name="test_validate_good" time="0.202" /></testsuite></testsuites>

scripts/ci.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# Usage: TESTS=0 python scripts/ci.py scripts/distribution.json
+import json
+import os
+import subprocess
+from sys import argv
+
+test_selection = os.getenv("TESTS")
+assert len(argv) == 2
+if test_selection is None:
+    print("Please set TESTS environment variable")
+    exit(1)
+with open(argv[1]) as f:
+    test_distribution = json.load(f)
+if test_selection == "other":
+    res = subprocess.run("pytest --collect-only -q", capture_output=True, shell=True)
+    res.check_returncode()
+    actual_tests = res.stdout.decode("utf-8").splitlines()
+    for index, test in enumerate(actual_tests):
+        if len(test) == 0:  # cut off warnings
+            actual_tests = actual_tests[:index]
+
+    all_known_tests = set(sum(test_distribution, []))
+    actual_tests_set = set(actual_tests)
+    known_but_dont_exist = all_known_tests.difference(actual_tests_set)
+    exist_but_arent_known = actual_tests_set.difference(all_known_tests)
+    if len(known_but_dont_exist) > 0 or len(exist_but_arent_known) > 0:
+        print("known_but_dont_exist:", known_but_dont_exist)
+        print("exist_but_arent_known", exist_but_arent_known)
+        print("Uncovered or outdated tests")
+        exit(2)
+else:
+    index = int(test_selection)
+    tests = " ".join('"' + test + '"' for test in test_distribution[index])
+    subprocess.run(
+        "pytest "
+        "--cov=openwpm --junit-xml=junit-report.xml "
+        f"--cov-report=xml {tests} "
+        "-s -v --durations=10;",
+        shell=True,
+        check=True,
+    )
+    subprocess.run("codecov -f coverage.xml", shell=True, check=True)

scripts/ci.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# Usage: python scripts/distribute_tests.py junit-report.xml 7 scripts/distribution.json
+# Where junit-report.xml is the result of running
+# python -m pytest --cov=openwpm --junit-xml=junit-report.xml --cov-report=xml test/ -s -v --durations=10
+import json
+from sys import argv
+from typing import Any
+from xml.etree import ElementTree as ET
+
+assert len(argv) == 4
+
+num_runner = int(argv[2])
+tree = ET.parse(argv[1])
+root = tree.getroot()
+testcases: list[dict[str, Any]] = []
+for testcase in root.iter("testcase"):
+    # Build correct test name based on naming convention
+    classname = testcase.get("classname")
+    assert isinstance(classname, str)
+    split = classname.rsplit(".", 1)
+    if split[1][0].isupper():  # Test is in a class
+        split[0] = split[0].replace(".", "/")
+        split[0] = split[0] + ".py"
+        path = split[0] + "::" + split[1]
+    else:
+        path = classname.replace(".", "/") + ".py"
+    time = testcase.get("time")
+    assert isinstance(time, str)
+    testcases.append({"path": f'{path}::{testcase.get("name")}', "time": float(time)})
+
+sorted_testcases = sorted(testcases, key=lambda x: x["time"], reverse=True)
+total_time = sum(k["time"] + 0.5 for k in sorted_testcases)
+time_per_runner = total_time / num_runner
+print(f"Total time: {total_time} Total time per runner: {total_time / num_runner}")
+distributed_testcases: list[list[str]] = [[] for _ in range(num_runner)]
+estimated_time = []
+for subsection in distributed_testcases:
+    time_spent = 0
+    tmp = []
+    for testcase in sorted_testcases:
+        if time_spent + testcase["time"] < time_per_runner:
+            tmp.append(testcase)
+            time_spent += testcase["time"] + 0.5  # account for overhead per testcase
+    for testcase in tmp:
+        sorted_testcases.remove(testcase)
+    subsection[:] = [testcase["path"] for testcase in tmp]
+    estimated_time.append(time_spent)
+
+assert len(sorted_testcases) == 0, print(len(sorted_testcases))
+print(estimated_time)
+with open(argv[3], "w") as f:
+    json.dump(distributed_testcases, f, indent=0)

scripts/distribute_tests.py

@@ -0,0 +1,142 @@
+[
+[
+"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateful-without_seed_tar]",
+"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateless-with_seed_tar]",
+"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateful-with_seed_tar]",
+"test/test_profile.py::test_profile_saved_when_launch_crashes",
+"test/test_simple_commands.py::test_save_screenshot_valid[headless]",
+"test/test_profile.py::test_load_tar_file",
+"test/test_js_instrument_py.py::test_validate_bad__log_settings_missing",
+"test/test_js_instrument_py.py::test_validate_good"
+],
+[
+"test/test_profile.py::test_profile_recovery[on_crash-stateless-with_seed_tar]",
+"test/test_profile.py::test_profile_recovery[on_crash-stateful-with_seed_tar]",
+"test/test_profile.py::test_profile_recovery[on_timeout-stateful-with_seed_tar]",
+"test/test_profile.py::test_profile_recovery[on_normal_operation-stateless-with_seed_tar]",
+"test/test_profile.py::test_profile_recovery[on_crash-stateful-without_seed_tar]",
+"test/test_js_instrument_py.py::test_validate_bad__log_settings_invalid",
+"test/test_js_instrument_py.py::test_api_collection_fingerprinting",
+"test/test_js_instrument_py.py::test_validate_bad__not_a_list",
+"test/test_js_instrument_py.py::test_validate_bad__missing_object",
+"test/test_js_instrument_py.py::test_validated_bad__missing_instrumentedName",
+"test/test_js_instrument_py.py::test_merge_and_validate_multiple_overlap_properties_to_instrument_properties_to_exclude",
+"test/storage/test_storage_providers.py::test_basic_access[memory_structured]",
+"test/storage/test_storage_providers.py::test_basic_access[memory_arrow]",
+"test/storage/test_storage_providers.py::test_basic_access[sqlite]",
+"test/test_js_instrument_py.py::test_complete_pass",
+"test/storage/test_storage_providers.py::test_basic_unstructured_storing[memory_unstructured]",
+"test/test_task_manager.py::test_failure_limit_value"
+],
+[
+"test/test_profile.py::test_profile_recovery[on_timeout-stateful-without_seed_tar]",
+"test/test_profile.py::test_profile_recovery[on_timeout-stateless-with_seed_tar]",
+"test/test_profile.py::test_profile_recovery[on_normal_operation-stateful-with_seed_tar]",
+"test/test_profile.py::test_dump_profile_command",
+"test/test_profile.py::test_profile_recovery[on_normal_operation-stateful-without_seed_tar]",
+"test/test_extension.py::TestExtension::test_extension_gets_correct_visit_id",
+"test/storage/test_storage_providers.py::test_basic_unstructured_storing[leveldb]",
+"test/storage/test_storage_providers.py::test_basic_unstructured_storing[local_gzip]",
+"test/test_callstack_instrument.py::test_http_stacktrace",
+"test/test_dataclass_validations.py::test_display_mode",
+"test/test_dataclass_validations.py::test_browser_type",
+"test/test_dataclass_validations.py::test_tp_cookies_opt",
+"test/test_dataclass_validations.py::test_save_content_type",
+"test/test_dataclass_validations.py::test_log_file_extension",
+"test/test_dataclass_validations.py::test_failure_limit",
+"test/test_dataclass_validations.py::test_num_browser_crawl_config"
+],
+[
+"test/test_task_manager.py::test_assertion_error_propagation[False-expectation0]",
+"test/test_xvfb_browser.py::test_display_shutdown",
+"test/extension/test_startup_timeout.py::test_extension_startup_timeout",
+"test/test_simple_commands.py::test_browse_wrapper_http_table_valid[headless]",
+"test/test_task_manager.py::test_assertion_error_propagation[True-expectation1]",
+"test/test_simple_commands.py::test_browse_http_table_valid[headless]",
+"test/test_simple_commands.py::test_browse_http_table_valid[xvfb]",
+"test/test_simple_commands.py::test_browse_wrapper_http_table_valid[xvfb]",
+"test/test_extension.py::test_audio_fingerprinting",
+"test/test_profile.py::test_profile_error",
+"test/test_http_instrumentation.py::TestHTTPInstrument::test_service_worker_requests",
+"test/test_js_instrument_py.py::test_merge_and_validate_multiple_overlap_properties",
+"test/test_js_instrument_py.py::test_merge_when_log_settings_is_null",
+"test/test_webdriver_utils.py::test_parse_neterror"
+],
+[
+"test/test_http_instrumentation.py::test_cache_hits_recorded",
+"test/test_task_manager.py::test_failure_limit_reset",
+"test/test_profile.py::test_crash_during_init",
+"test/test_simple_commands.py::test_dump_page_source_valid[headless]",
+"test/test_profile.py::test_saving",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_x_www_form_urlencoded",
+"test/test_js_instrument.py::TestJSInstrumentRecursiveProperties::test_instrument_object",
+"test/test_simple_commands.py::test_save_screenshot_valid[xvfb]",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_text_plain",
+"test/test_http_instrumentation.py::test_javascript_saving",
+"test/test_http_instrumentation.py::test_content_saving",
+"test/test_http_instrumentation.py::TestHTTPInstrument::test_worker_script_requests",
+"test/test_extension.py::TestExtension::test_js_call_stack",
+"test/test_profile.py::test_crash_profile",
+"test/test_extension.py::TestExtension::test_canvas_fingerprinting",
+"test/test_crawl.py::test_browser_profile_coverage",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_file_upload",
+"test/test_js_instrument_py.py::test_merge_diff_instrumented_names",
+"test/test_js_instrument_py.py::test_merge_multiple_duped_properties",
+"test/test_js_instrument_py.py::test_merge_multiple_duped_properties_different_log_settings",
+"test/test_js_instrument_py.py::test_api_whole_module",
+"test/test_js_instrument_py.py::test_api_two_keys_in_shortcut",
+"test/test_js_instrument_py.py::test_api_instances_on_window",
+"test/test_js_instrument_py.py::test_api_instances_on_window_with_properties",
+"test/test_js_instrument_py.py::test_api_module_specific_properties",
+"test/test_js_instrument_py.py::test_api_passing_partial_log_settings"
+],
+[
+"test/test_simple_commands.py::test_recursive_dump_page_source_valid[xvfb]",
+"test/extension/test_logging.py::test_extension_logging",
+"test/test_timer.py::test_command_duration",
+"test/test_http_instrumentation.py::test_page_visit[True]",
+"test/storage/test_storage_controller.py::test_arrow_provider",
+"test/storage/test_storage_controller.py::test_startup_and_shutdown",
+"test/test_mp_logger.py::test_multiple_instances",
+"test/test_simple_commands.py::test_recursive_dump_page_source_valid[headless]",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax_no_key_value",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_formdata",
+"test/test_js_instrument.py::TestJSInstrumentExistingWindowProperty::test_instrument_object",
+"test/test_simple_commands.py::test_get_http_tables_valid[xvfb]",
+"test/test_simple_commands.py::test_dump_page_source_valid[xvfb]",
+"test/test_js_instrument.py::TestJSInstrument::test_instrument_object",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax",
+"test/test_dns_instrument.py::test_name_resolution",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_binary_post_data",
+"test/test_http_instrumentation.py::test_document_saving",
+"test/test_storage_vectors.py::test_js_profile_cookies",
+"test/test_extension.py::TestExtension::test_js_time_stamp",
+"test/storage/test_storage_providers.py::test_local_arrow_storage_provider"
+],
+[
+"test/test_js_instrument.py::TestJSInstrumentMockWindowProperty::test_instrument_object",
+"test/test_extension.py::TestExtension::test_document_cookie_instrumentation",
+"test/test_custom_function_command.py::test_custom_function",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax_no_key_value_base64_encoded",
+"test/test_extension.py::TestExtension::test_property_enumeration",
+"test/test_simple_commands.py::test_get_site_visits_table_valid[xvfb]",
+"test/test_js_instrument.py::TestJSInstrumentNonExistingWindowProperty::test_instrument_object",
+"test/test_simple_commands.py::test_get_http_tables_valid[headless]",
+"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_multipart_formdata",
+"test/test_http_instrumentation.py::test_page_visit[False]",
+"test/test_js_instrument.py::TestJSInstrumentByPython::test_instrument_object",
+"test/test_profile.py::test_seed_persistence",
+"test/test_callback.py::test_local_callbacks",
+"test/test_simple_commands.py::test_get_site_visits_table_valid[headless]",
+"test/test_simple_commands.py::test_browse_site_visits_table_valid[headless]",
+"test/test_task_manager.py::test_failure_limit_exceeded",
+"test/test_extension.py::TestExtension::test_webrtc_localip",
+"test/test_simple_commands.py::test_browse_site_visits_table_valid[xvfb]",
+"test/storage/test_arrow_cache.py::test_arrow_cache",
+"test/test_profile.py::test_save_incomplete_profile_error",
+"test/test_webdriver_utils.py::test_parse_neterror_integration",
+"test/test_mp_logger.py::test_multiprocess",
+"test/test_mp_logger.py::test_child_process_with_exception",
+"test/test_mp_logger.py::test_child_process_logging"
+]
+]