diff --git a/.gitignore b/.gitignore index 1518ddedd94..d1538f7040b 100644 --- a/.gitignore +++ b/.gitignore @@ -203,6 +203,7 @@ cloned_venvs/ .circleci/config.gen.yml # GitLab CI generated config .gitlab/**/*-gen.yml +.gitlab/ci-allocation-plan.json .gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml .gitlab-ci-local/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58278530e30..40f613681a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,11 @@ workflow: DD_GIT_REPOSITORY_URL: "https://github.com/DataDog/dd-trace-py-release.git" auto_cancel: on_new_commit: none + # Temporary PR validation: run the balanced allocator beside legacy on the + # exact branch under review. Remove after the paired evidence is captured. + - if: '$CI_COMMIT_REF_NAME == "dd/ci-runtime-aware-riot-sharding-20260813-1915"' + variables: + CI_ALLOCATION_SHADOW: "true" - when: always include: @@ -110,6 +115,7 @@ tests-gen: artifacts: paths: - .gitlab/tests-gen.yml + - .gitlab/ci-allocation-plan.json - .gitlab/benchmarks/microbenchmarks-gen.yml - .gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml diff --git a/.gitlab/scripts/get-riot-hashes.sh b/.gitlab/scripts/get-riot-hashes.sh index 801409743ec..519551c3b07 100755 --- a/.gitlab/scripts/get-riot-hashes.sh +++ b/.gitlab/scripts/get-riot-hashes.sh @@ -2,4 +2,55 @@ set -e -u -o pipefail SUITE_NAME="${1:-}" -riot list --hash-only "${SUITE_NAME}" | sort | ./.gitlab/ci-split-input.sh +CI_ALLOCATION_SUITE="${2:-${CI_ALLOCATION_SUITE:-${SUITE_NAME}}}" +strategy_args=() +if [[ -n "${CI_ALLOCATION_STRATEGY:-}" ]]; then + strategy_args=(--strategy "${CI_ALLOCATION_STRATEGY}") +fi + +mapfile -t available_hashes < <(riot list --hash-only "${SUITE_NAME}" | sort) +if [[ -n "${CI_ALLOCATION_ASSIGNMENTS:-}" ]]; then + IFS=';' read -r -a planned_assignments <<< "${CI_ALLOCATION_ASSIGNMENTS}" + node_index="${CI_NODE_INDEX:-1}" + node_total="${CI_NODE_TOTAL:-1}" + if [[ ! "${node_index}" =~ ^[1-9][0-9]*$ || ! "${node_total}" =~ ^[1-9][0-9]*$ ]]; then + echo "CI node index and total must be positive integers" >&2 + exit 1 + fi + if [[ "${#planned_assignments[@]}" -ne "${node_total}" ]]; then + echo "Generated allocation count differs from CI_NODE_TOTAL" >&2 + exit 1 + fi + if [[ "${node_index}" -gt "${#planned_assignments[@]}" ]]; then + echo "CI_NODE_INDEX is outside the generated allocation" >&2 + exit 1 + fi + CI_ALLOCATION_UNITS="${planned_assignments[$((node_index - 1))]}" +fi +if [[ -n "${CI_ALLOCATION_UNITS:-}" ]]; then + declare -A available=() + for riot_hash in "${available_hashes[@]}"; do + available["${riot_hash}"]=1 + done + IFS=',' read -r -a execution_units <<< "${CI_ALLOCATION_UNITS}" + for unit in "${execution_units[@]}"; do + if [[ ! "${unit}" =~ ^([0-9a-f]+)(@([1-9][0-9]*)/([1-9][0-9]*))?$ ]]; then + echo "Invalid Riot execution unit: ${unit}" >&2 + exit 1 + fi + riot_hash="${BASH_REMATCH[1]}" + if [[ -z "${available[${riot_hash}]:-}" ]]; then + echo "Generated Riot execution unit is not in ${SUITE_NAME}: ${unit}" >&2 + exit 1 + fi + printf '%s\n' "${unit}" + done + exit 0 +fi + +printf '%s\n' "${available_hashes[@]}" | \ + ./scripts/ci_allocation_cli.py select \ + --suite "${CI_ALLOCATION_SUITE}" \ + --node-index "${CI_NODE_INDEX:-1}" \ + --node-total "${CI_NODE_TOTAL:-1}" \ + "${strategy_args[@]}" diff --git a/.gitlab/scripts/get-riot-pip-cache-key.sh b/.gitlab/scripts/get-riot-pip-cache-key.sh index bedfcab80f3..6254b7a6625 100755 --- a/.gitlab/scripts/get-riot-pip-cache-key.sh +++ b/.gitlab/scripts/get-riot-pip-cache-key.sh @@ -2,7 +2,9 @@ set -e -u -o pipefail SUITE_NAME="${1:-}" -hashes=( $(./.gitlab/scripts/get-riot-hashes.sh "${SUITE_NAME}") ) +# Cache identity covers the full semantic suite and must not depend on one +# physical allocation job's node index or runtime test slice. +hashes=( $(riot list --hash-only "${SUITE_NAME}" | sort -u) ) # Get the sha256sum of all the requirements files combined for hash in "${hashes[@]}"; do req_file="./.riot/requirements/${hash}.txt" diff --git a/.gitlab/testrunner.yml b/.gitlab/testrunner.yml index ae2734a0e3d..0e2929606ec 100644 --- a/.gitlab/testrunner.yml +++ b/.gitlab/testrunner.yml @@ -25,6 +25,7 @@ variables: when: always paths: - core.* + - test-results/ci-test-shard-inventory.*.json reports: junit: test-results/junit*.xml expire_in: 1 week diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index c2974d8a98e..f3274e43a34 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -36,11 +36,24 @@ include: echo "No riot hashes found for ${SUITE_NAME}" exit 1 fi - for hash in "${hashes[@]}" + for unit in "${hashes[@]}" do + hash="${unit%%@*}" + if [[ "${unit}" =~ ^[0-9a-f]+@([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then + test_shard_index="${BASH_REMATCH[1]}" + test_shard_total="${BASH_REMATCH[2]}" + else + test_shard_index=1 + test_shard_total=1 + fi echo "Running riot hash: ${hash}" + echo "Runtime test shard: ${test_shard_index}/${test_shard_total}" riot list "${hash}" - export _CI_DD_TAGS="test.configuration.riot_hash:${hash}" + export RIOT_HASH="${hash}" + export RIOT_TEST_SHARD_INDEX="${test_shard_index}" + export RIOT_TEST_SHARD_TOTAL="${test_shard_total}" + export RIOT_CI_ALLOCATION_STRATEGY="${CI_ALLOCATION_STRATEGY:-legacy}" + export _CI_DD_TAGS="test.configuration.riot_hash:${RIOT_HASH},test.configuration.ci_allocation_strategy:${RIOT_CI_ALLOCATION_STRATEGY},test.configuration.runtime_shard_index:${RIOT_TEST_SHARD_INDEX},test.configuration.runtime_shard_total:${RIOT_TEST_SHARD_TOTAL}" ${RIOT_RUN_CMD} "${hash}" -- --ddtrace done ./scripts/check-diff ".riot/requirements/" \ diff --git a/ci/ci-allocation-policy.json b/ci/ci-allocation-policy.json new file mode 100644 index 00000000000..46bfc9ce1c0 --- /dev/null +++ b/ci/ci-allocation-policy.json @@ -0,0 +1,37 @@ +{ + "allocation": { + "active_strategy": "legacy", + "maximum_parallelism_per_suite": 50, + "maximum_runtime_model_bytes": 100000, + "maximum_slices_per_hash": 5, + "target_jobs": 200, + "target_shard_seconds": 300 + }, + "model": { + "estimate_quantile": 0.9, + "half_life_days": 30, + "history_window_days": 90, + "holdout_days": 14, + "minimum_samples": 5, + "sparse_safety_factor": 1.25 + }, + "ratchets": { + "historical_replay": { + "maximum_runner_seconds_increase_ratio": 0.05, + "minimum_median_improvement_ratio": 0.05, + "minimum_runs": 30 + }, + "live_shadow": { + "maximum_queue_p90_increase_ratio": 0.05, + "maximum_runner_seconds_increase_ratio": 0.05, + "minimum_median_improvement_ratio": 0.5, + "minimum_runs": 15 + }, + "pr_shape_replay": { + "maximum_runner_seconds_increase_ratio": 0.05, + "minimum_median_improvement_ratio": 0.05, + "minimum_shapes": 30 + } + }, + "schema_version": 1 +} diff --git a/ci/ci-allocation-runtime-model.json b/ci/ci-allocation-runtime-model.json new file mode 100644 index 00000000000..54a8fe84ad9 --- /dev/null +++ b/ci/ci-allocation-runtime-model.json @@ -0,0 +1,2136 @@ +{ + "schema_version": 1, + "planner_version": "weighted-lpt-v1", + "generated_at": "2026-08-14T18:52:55Z", + "dataset": { + "source": "same-head-ci-job-calibrated-shadow-seed", + "source_commit": "3bd95510c20255b89fa41e1ad9822d617c1ffe3d", + "source_pipeline_id": "131076783", + "source_pipeline_url": "https://gitlab.ddbuild.io/DataDog/apm-reliability/dd-trace-py/-/pipelines/131076783", + "fingerprint_sha256": "16629a7640690afe67febf613c1acf2f2eda864b6a1be6f17349f4ef1a3dd0ef", + "job_fingerprint_sha256": "e6e81a46d054ed914d6702e3db67f6823bc79d82bd59897416102b52ae04cf7b", + "history_window_days": 0, + "holdout_days": 0, + "holdout_observations": 0, + "job_observations": 867, + "censored_observations": 0, + "identity_missing_observations": 15, + "planned_hashes": 1827, + "modeled_hashes": 1802, + "unmodeled_plan_hashes": [ + "10f2939", + "132f162", + "1443b2d", + "17b66d6", + "1807b73", + "1a6865c", + "1aa7f8c", + "1b6a350", + "1bc5921", + "1d04c8d", + "1d41360", + "1f75b21", + "3957288", + "4487fa7", + "4a90061", + "59e7d85", + "6382845", + "69e1cb1", + "6e664eb", + "87b8661", + "88841c7", + "a4aa6ca", + "c0d357f", + "c5214fe", + "f8f807c" + ], + "training_end": "2026-08-13T20:21:30.647Z", + "training_observations": 1864, + "status_counts": { + "pass": 1864 + }, + "window_end": "2026-08-14T18:52:55Z", + "session_seed": { + "source": "single-production-run-shadow-seed", + "source_commit": "464429cfc4ca4f29e09ce11527907b5eb61d7fa5", + "source_pipeline_id": "130835568", + "fingerprint_sha256": "477bbf3e263396594e459451632d3a47ebab7019018d7739919c9f8cab4a871d" + }, + "calibration": { + "timing_source": "ci-jobs-and-test-sessions", + "legacy_riot_jobs": 867, + "legacy_job_seconds": 218256, + "legacy_sessions": 1864, + "legacy_session_seconds": 107089.795235349, + "balanced_riot_jobs": 578, + "balanced_job_seconds": 195007, + "balanced_sessions": 1865, + "legacy_max_job_seconds": 1057, + "balanced_max_job_seconds": 977, + "unit_overhead_seconds": { + "global": 59.638522, + "aiguard": 67.654978, + "appsec": 94.484851, + "ci_visibility": 68.357517, + "contrib": 50.501257, + "core": 52.474301, + "debugging": 43.574604, + "errortracking": 22.825299, + "llmobs": 76.693147, + "profiling": 50.360256 + } + } + }, + "estimates": { + "1030725": 84.833032, + "1038948": 87.119828, + "1048705": 129.377237, + "1059060": 113.009076, + "1059304": 158.529677, + "1064582": 145.830761, + "1072660": 227.44394, + "1101787": 60.137103, + "1193154": 227.44394, + "1204574": 75.442197, + "1260019": 51.675738, + "1307807": 104.825819, + "1349413": 129.377237, + "1360370": 138.251907, + "1370206": 412.624161, + "1381214": 74.427743, + "1384411": 65.654884, + "1407476": 51.681649, + "1418434": 293.539111, + "1431337": 60.137103, + "1435097": 71.249249, + "1436100": 136.483316, + "1437520": 61.425958, + "1463930": 53.775468, + "1475020": 63.244627, + "1477633": 833.648852, + "1528286": 227.44394, + "1531241": 127.800685, + "1544047": 98.684245, + "1544815": 73.925764, + "1558546": 350.030002, + "1567689": 136.483316, + "1577306": 60.137103, + "1588200": 103.735603, + "1592050": 123.445341, + "1602479": 83.081041, + "1659232": 236.897201, + "1661512": 223.471128, + "1721018": 51.649728, + "1753169": 83.102429, + "1763009": 133.687311, + "1782179": 93.672488, + "1833817": 93.93915, + "1842297": 75.894682, + "1842452": 93.672488, + "1916976": 110.166789, + "1927469": 244.226557, + "1949111": 96.984723, + "1949639": 136.483316, + "2215008": 51.681649, + "2246229": 130.131006, + "2720069": 287.865513, + "3185459": 60.137103, + "4532043": 163.387529, + "5420667": 525.374747, + "5455699": 211.706577, + "6289286": 129.574923, + "6738855": 227.44394, + "6875074": 176.231201, + "7365790": 63.244627, + "7473443": 119.024153, + "7628925": 227.44394, + "7670259": 248.954796, + "7691722": 287.865513, + "8227490": 519.037136, + "8239194": 57.686769, + "8400353": 120.09674, + "8704384": 236.897201, + "8830759": 136.483316, + "9204343": 74.057771, + "9232661": 118.663717, + "9710280": 563.241394, + "100eda9": 209.601193, + "1010ab9": 143.009776, + "101b183": 62.703563, + "101f000": 80.137579, + "102b951": 69.022253, + "102dfdd": 85.835803, + "1036e18": 139.232272, + "103ef63": 202.068005, + "104f450": 81.906502, + "1050efa": 88.059968, + "1053a29": 248.954796, + "1053dc0": 287.865513, + "1053dce": 73.925764, + "105c431": 88.052798, + "106bf5d": 96.984723, + "106f2d7": 158.293834, + "107d2ec": 651.383996, + "107e1ae": 87.534033, + "10852ec": 115.713997, + "10853b3": 61.425958, + "108afed": 651.383996, + "109a45b": 223.230556, + "109d638": 201.795038, + "10a0ca1": 85.835803, + "10b3343": 81.906502, + "10b643c": 61.425958, + "10b7fd9": 90.022807, + "10ba06a": 97.603852, + "10ba4fc": 227.44394, + "10bb064": 88.174033, + "10bb96a": 66.136913, + "10bdae9": 136.483316, + "10c216c": 51.796858, + "10c6be8": 99.609222, + "10c6e12": 93.93915, + "10d1da4": 88.059968, + "10d379e": 162.697123, + "10d8f51": 176.231201, + "10d90c0": 227.44394, + "10da678": 89.898332, + "10ddcfd": 77.28048, + "10e0262": 88.176835, + "10e2453": 136.483316, + "10e57ab": 96.984723, + "10ebbfc": 130.131006, + "10ed886": 227.44394, + "10f023c": 69.657014, + "10f2f3e": 143.009776, + "10f41c3": 59.807643, + "10f75ab": 223.471128, + "10fe0d5": 197.451151, + "1102e86": 171.871578, + "1107e3b": 98.150341, + "1110d0c": 279.951665, + "1111da1": 563.241394, + "11193ae": 119.024153, + "111ed90": 88.174033, + "1125dea": 129.377237, + "1127dcb": 169.608714, + "112b805": 111.058122, + "112cf54": 395.754538, + "11335dd": 101.546551, + "113966a": 120.389726, + "113ca1f": 88.176835, + "114620d": 94.278808, + "1148df5": 96.984723, + "114bad8": 136.483316, + "114bf76": 59.807643, + "11518a9": 138.251907, + "1151ca8": 412.624161, + "115595c": 95.229582, + "1156e38": 227.44394, + "115915a": 139.232273, + "11594bd": 77.28048, + "1159a5a": 101.073657, + "115d290": 78.972724, + "115e19f": 60.137103, + "116340d": 72.523807, + "116989a": 133.03799, + "116a58c": 78.972724, + "116b0a1": 136.483316, + "116b0b8": 158.529677, + "116f7b1": 81.904954, + "118065f": 143.009776, + "1182d01": 72.566518, + "1185b58": 291.350119, + "11868bf": 74.057771, + "118c78b": 129.377237, + "118ddd5": 227.44394, + "118f9a8": 89.49123, + "118fec7": 525.374747, + "1193aba": 563.241394, + "119431a": 291.350118, + "1196ac3": 98.684245, + "11a0d76": 108.861018, + "11ab0ab": 158.293834, + "11b0623": 185.725547, + "11b45d7": 130.131006, + "11b6e91": 129.574923, + "11b941f": 60.137103, + "11bb2fd": 108.861018, + "11bd6c7": 89.49123, + "11c2588": 74.057771, + "11c313d": 120.09674, + "11c3907": 60.137103, + "11c7793": 143.529587, + "11c8584": 236.897201, + "11c88b9": 88.176835, + "11cd1a5": 51.681649, + "11d1399": 130.131006, + "11d4944": 93.93915, + "11d5c8b": 103.735603, + "11e37fa": 143.009776, + "11e4e8b": 77.101773, + "11e6ad6": 51.681649, + "11e7bf8": 202.068005, + "11eae4e": 53.738899, + "11f7715": 101.1125, + "11f9495": 60.137103, + "11fd02a": 73.925764, + "1209b80": 478.506884, + "120e7d0": 120.09674, + "120e7ea": 94.054305, + "12113b3": 55.393299, + "1212ab8": 79.015914, + "121518f": 120.09674, + "121a519": 79.015914, + "121fc8d": 71.249249, + "1224d93": 76.294731, + "1224f7d": 112.961919, + "12263ee": 125.771444, + "1229e9a": 223.230556, + "122cffd": 125.771444, + "122d3c5": 73.925764, + "1230ef1": 128.747771, + "1235f1e": 223.471128, + "1246b86": 478.506884, + "1246e96": 455.882938, + "124b91e": 167.943677, + "12594bd": 87.685476, + "12616cb": 112.271377, + "1272ddf": 61.986589, + "127eabf": 350.030002, + "127edb7": 62.703563, + "1285aa4": 57.686768, + "128b106": 651.383996, + "128dc9b": 69.022253, + "1290c29": 95.229582, + "129868b": 129.377237, + "12a25de": 94.376801, + "12a51fd": 78.972724, + "12aafe0": 84.833032, + "12afae4": 267.332157, + "12b3167": 98.684245, + "12b9587": 53.318226, + "12b9e07": 96.984723, + "12bb48f": 81.906502, + "12bdba7": 53.738899, + "12bf701": 88.174033, + "12c10e8": 88.059968, + "12c5734": 133.03799, + "12c877d": 74.057771, + "12cb0e7": 88.059968, + "12ce109": 57.686768, + "12ce83b": 350.030002, + "12d0bda": 99.609222, + "12d24d7": 66.136913, + "12d6455": 143.529587, + "12d6a82": 69.022253, + "12dca17": 93.39095, + "12f0825": 236.897201, + "12f38be": 99.284857, + "12f6833": 93.93915, + "1303be6": 651.383996, + "130a7d6": 83.081041, + "1315bb9": 60.137103, + "13180f0": 51.681649, + "131a701": 69.657014, + "132e4bd": 212.189922, + "132eb35": 96.984723, + "1330cf0": 96.984723, + "1332b9d": 176.231201, + "1334ad5": 227.44394, + "1336cbd": 96.454878, + "13404e3": 227.44394, + "134082f": 201.795038, + "13460b6": 57.686768, + "1346e9d": 158.529677, + "134bcdd": 87.119828, + "134deb1": 65.654884, + "1351aca": 158.529677, + "1356db9": 227.44394, + "1361e46": 162.697123, + "136293c": 223.471128, + "136327d": 77.28048, + "136b4b4": 129.377237, + "136fddd": 60.137103, + "1373a22": 93.93915, + "13873ec": 63.244627, + "138c1ad": 60.137103, + "1390f56": 202.681466, + "1391c58": 833.648852, + "13991ca": 73.925764, + "139b6b2": 158.529677, + "13a0575": 209.601193, + "13a379a": 105.408318, + "13ae267": 87.119828, + "13b56c2": 74.621977, + "13b8341": 60.860192, + "13c0fff": 122.052288, + "13c4b39": 66.297626, + "13cd3bb": 223.471128, + "13cf9b7": 478.506884, + "13de08c": 570.474571, + "13e0d21": 163.387529, + "13ee970": 69.246212, + "13f9d79": 80.390407, + "13fe884": 112.271377, + "140ce37": 478.506884, + "14116fa": 74.427743, + "142ded7": 99.533365, + "142fb86": 87.685476, + "14305cf": 81.318681, + "143e2ab": 74.621977, + "1441a01": 176.814418, + "144e8b5": 130.131006, + "1458a81": 122.655776, + "1458d7e": 201.795038, + "145ed9e": 202.681466, + "145f918": 123.445341, + "1468e09": 807.789116, + "1469bae": 519.037136, + "146f136": 807.789116, + "1475c1a": 120.389726, + "147a89a": 96.509883, + "147bedb": 88.059968, + "14859e9": 88.176835, + "148bd89": 73.925764, + "148c37a": 101.546551, + "148cc44": 143.009776, + "149304f": 133.03799, + "149bd30": 106.656738, + "14a5b18": 105.408319, + "14aa6df": 201.795038, + "14b461b": 106.656738, + "14b54db": 202.681466, + "14b9202": 60.137103, + "14bb28e": 143.009776, + "14be2f6": 81.318681, + "14c34e9": 83.027177, + "14c793e": 87.119828, + "14c9053": 85.835803, + "14cbe98": 83.295266, + "14cfe2e": 98.684245, + "14d6531": 81.904954, + "14d8da4": 130.131006, + "14e26cb": 219.291397, + "14e3100": 591.890823, + "14e5fc5": 76.792243, + "14e98b5": 129.377237, + "14ebf3b": 83.102429, + "14f0a7d": 807.789116, + "14f1594": 112.271377, + "14fc413": 66.297626, + "14fceda": 350.030002, + "1504e4c": 71.249249, + "1509aa1": 408.933627, + "150beac": 120.389726, + "1512a1b": 57.686768, + "151f23f": 833.648852, + "1522dd0": 120.389726, + "152e97f": 60.137103, + "15322d3": 209.601193, + "1532cbc": 227.44394, + "153586b": 143.009776, + "153608c": 138.251907, + "1538bcb": 167.943677, + "153b471": 79.480275, + "153fe56": 80.471866, + "1540c33": 66.297626, + "1547cc9": 201.795038, + "15558e4": 80.913027, + "1560cbf": 87.119828, + "1560cda": 80.913027, + "1564dd5": 60.137103, + "15770fa": 73.113077, + "157ef2b": 238.123504, + "1581ea5": 83.081041, + "1586b69": 73.925764, + "158b41a": 171.871578, + "1591bf5": 88.174033, + "1591c59": 95.229582, + "159a2a4": 93.93915, + "15a365d": 77.280481, + "15a503b": 208.089715, + "15a8df6": 90.022808, + "15adb8d": 227.44394, + "15afa58": 108.861018, + "15b093e": 350.030002, + "15b58f8": 286.983341, + "15b8c41": 66.136913, + "15b9e28": 119.024153, + "15c9f1f": 167.943677, + "15cab00": 89.842511, + "15cc9b9": 79.015914, + "15cd0eb": 167.943677, + "15d0624": 129.377237, + "15db176": 143.009776, + "15dee3b": 83.228517, + "15e6955": 176.231201, + "15e7251": 236.897201, + "15e76f9": 79.015914, + "15eae35": 139.232272, + "15eaf5b": 72.523807, + "15fbf61": 113.009076, + "15fd7ec": 120.389726, + "15fec28": 66.136913, + "16054bb": 93.672488, + "1609bd2": 88.052798, + "160bd16": 350.030002, + "160ce6c": 74.621977, + "160ea38": 71.249249, + "1611a53": 61.425958, + "1612a26": 833.648852, + "16138c7": 66.297626, + "16181c1": 119.024153, + "161aef0": 93.672488, + "161b2ce": 81.906502, + "161f1c8": 99.364088, + "1622fff": 94.376801, + "16250bb": 51.675738, + "1626f45": 98.150341, + "162b59e": 202.068005, + "162cf2e": 201.795038, + "162f3ce": 125.771444, + "16313f3": 61.425958, + "1631cdb": 241.698955, + "1632a0e": 62.784432, + "163bc46": 88.176835, + "163c8d2": 80.137579, + "1652e36": 87.534033, + "165add9": 415.120891, + "165d803": 82.235192, + "165faec": 106.656738, + "166880c": 98.150341, + "166aa1b": 241.698955, + "166d447": 162.899337, + "1674af7": 60.137103, + "16781e7": 143.009776, + "167b853": 74.057771, + "167c1e6": 77.28048, + "168abb5": 563.241394, + "168cc07": 81.904954, + "168ee03": 66.297626, + "169477d": 129.574923, + "1694e39": 130.131006, + "16969ec": 83.081041, + "1696b86": 72.844938, + "169ae94": 119.011538, + "169ce58": 96.454878, + "16a63d7": 201.795038, + "16a6e70": 78.972725, + "16ac1f1": 61.425958, + "16af3aa": 71.249249, + "16b0319": 115.713997, + "16b04fe": 227.44394, + "16b152c": 87.534034, + "16b741f": 66.297626, + "16bd71d": 88.174033, + "16c1c69": 176.814418, + "16ca618": 98.150341, + "16cc81d": 66.136913, + "16d286c": 99.284857, + "16d3c69": 87.729325, + "16d58df": 197.451151, + "16d8026": 238.123503, + "16dd69c": 140.884021, + "16e6824": 563.241394, + "16e767e": 158.529677, + "16ec0c2": 60.137103, + "16eec26": 167.943677, + "16f089d": 651.383996, + "16f2923": 106.656738, + "16f33ce": 74.427743, + "16f8e4b": 106.656738, + "16f97b5": 60.137103, + "1703ea4": 93.39095, + "170c530": 128.747771, + "170e1e9": 248.954796, + "17148ee": 176.814418, + "171c54c": 176.671424, + "171d43c": 99.284857, + "171d4b1": 455.882938, + "171e4a4": 53.318226, + "172a329": 59.807643, + "172d362": 227.44394, + "173260e": 88.174033, + "173555b": 158.529677, + "17391df": 74.621977, + "173ba30": 125.771444, + "1746c1c": 61.425958, + "1747b09": 77.27177, + "174d88f": 88.052798, + "17513e7": 96.509884, + "175a6ba": 169.608714, + "175d0d6": 83.102429, + "175eeba": 127.255901, + "175f930": 478.506884, + "176838a": 241.698955, + "176aab2": 101.546551, + "1778c11": 93.93915, + "177912e": 110.166789, + "177b157": 158.529677, + "177daf3": 563.241394, + "17806ff": 107.318674, + "1785cfd": 308.037929, + "17879d0": 87.978308, + "1787fb7": 236.897201, + "178dbc8": 65.654884, + "178f7d5": 81.906502, + "179c655": 158.529677, + "179d78b": 105.408318, + "179eaa2": 197.451151, + "17a0ecd": 209.601193, + "17a194c": 79.06218, + "17a234c": 143.529587, + "17a8226": 158.529677, + "17a868e": 53.738899, + "17ab061": 53.738899, + "17b723d": 139.232272, + "17b7249": 78.972724, + "17ba38a": 209.601193, + "17c1db9": 111.058122, + "17cb22b": 85.835803, + "17cd03c": 75.894682, + "17d40ef": 87.119828, + "17d4731": 103.735603, + "17d96ef": 93.39095, + "17d9faf": 103.735603, + "17dae6a": 120.09674, + "17df13b": 53.318226, + "17ec3e0": 130.131006, + "17edf5a": 128.747771, + "17efeae": 63.244627, + "17f2a52": 60.137103, + "17f7f1d": 218.915469, + "17fe359": 350.030002, + "18036be": 62.703563, + "180731e": 133.03799, + "181128c": 110.166789, + "181184d": 69.022253, + "1812e30": 106.640778, + "1814da7": 287.865513, + "181895c": 60.483851, + "1819a02": 88.176835, + "1819cb6": 136.483316, + "181e2d5": 158.529677, + "18269eb": 101.546551, + "18278c9": 80.191541, + "1829a8a": 227.44394, + "182bf3a": 78.972724, + "182dc13": 52.153442, + "1831d67": 72.523807, + "183e307": 120.389726, + "18421e5": 66.297626, + "1844abd": 519.037136, + "18515c6": 107.318675, + "18538d1": 366.596566, + "185a095": 80.355682, + "187aa61": 107.550708, + "187d6f8": 93.39095, + "187df5b": 81.906502, + "188244e": 53.738899, + "1882fe7": 201.795038, + "188a403": 87.978308, + "18913cd": 98.693213, + "18941c9": 130.131006, + "189633d": 133.687311, + "18a4a8d": 68.163559, + "18a6687": 87.119828, + "18ab9e9": 143.529587, + "18b32f4": 110.166789, + "18b7202": 248.954796, + "18b8b8f": 529.13895, + "18bf990": 62.703563, + "18c82f1": 248.954796, + "18ca8de": 83.081041, + "18cfbb0": 77.101773, + "18da66a": 169.608714, + "18dd95d": 130.131006, + "18e95df": 89.842511, + "18f859e": 88.174033, + "18f877f": 104.825819, + "18f95e2": 51.675738, + "18f9ba2": 248.954796, + "18fa2e7": 238.157061, + "18fce4a": 79.015914, + "19022d0": 57.686768, + "19099fb": 60.137103, + "190c811": 98.684245, + "190cc1a": 89.842511, + "190d82d": 651.383996, + "190e5df": 101.546551, + "190ee75": 52.562285, + "190fcc7": 651.383996, + "191027d": 80.913027, + "19109da": 167.943677, + "19138f9": 563.241394, + "19153ba": 89.49123, + "191885c": 65.441154, + "191bdb7": 101.1125, + "191bffe": 136.483316, + "191cea2": 87.490967, + "192c7c0": 60.137103, + "192e690": 60.137103, + "193762c": 88.176835, + "193fd52": 99.284857, + "1948b78": 91.770991, + "194c56a": 107.318674, + "194d749": 113.009076, + "194e789": 74.621977, + "19507e4": 408.933627, + "19508cd": 74.427743, + "1959ed5": 84.833032, + "195aef2": 72.523807, + "195ecad": 110.166789, + "196a8f0": 223.471128, + "196e8cf": 74.621977, + "19753a5": 127.255901, + "1979ceb": 120.09674, + "197c6fd": 93.672488, + "1987c1c": 53.738899, + "1989fbc": 158.529677, + "1994bde": 63.244627, + "199bb00": 366.596566, + "19a891c": 84.165004, + "19a8ed0": 75.894682, + "19a9a80": 62.866362, + "19aa242": 162.697123, + "19aa387": 98.684245, + "19bbf6d": 112.271377, + "19be394": 201.795038, + "19c6982": 133.03799, + "19c85cf": 83.102429, + "19c8864": 88.176835, + "19c9071": 651.383996, + "19ca09f": 158.529677, + "19d1a31": 143.009776, + "19d94ed": 59.807643, + "19db357": 65.654884, + "19db522": 209.601193, + "19dee8b": 80.137579, + "19e0c13": 807.789116, + "19e4934": 158.293834, + "19ed1c1": 60.137103, + "19f1d9b": 96.984723, + "19f3b8d": 136.483316, + "19f423c": 366.596566, + "19f5ff8": 73.113077, + "19f8b6e": 83.081041, + "19f9f09": 244.226557, + "19fc0b5": 519.037136, + "1a06176": 279.951665, + "1a21d86": 60.137103, + "1a22dee": 79.015914, + "1a2ae3e": 84.975181, + "1a2e084": 105.408318, + "1a38af9": 93.93915, + "1a42ba9": 95.229582, + "1a485c9": 94.867839, + "1a4c54d": 158.529677, + "1a4ea78": 107.318674, + "1a59a5f": 131.725201, + "1a67f8a": 93.39095, + "1a683e5": 110.166789, + "1a68ae7": 75.442197, + "1a69754": 529.13895, + "1a6ce84": 120.09674, + "1a6cf31": 87.490967, + "1a736ea": 94.376801, + "1a78e3a": 412.624161, + "1a7b44e": 209.601193, + "1a7c7c3": 127.255901, + "1a7f51a": 97.603852, + "1a862f5": 96.509883, + "1a8b5b1": 85.411258, + "1a8c53c": 291.350118, + "1a8f71a": 120.09674, + "1a92267": 80.137579, + "1a9e432": 158.529677, + "1aa359d": 123.445341, + "1aa7e48": 96.509883, + "1ab2cd6": 79.015914, + "1ab3dac": 563.241394, + "1ab75a5": 87.534033, + "1ac29e1": 51.681649, + "1ac5fb6": 97.071105, + "1acabe0": 73.925764, + "1ad28e8": 119.024153, + "1ad3ffa": 158.293834, + "1ada48c": 350.030002, + "1ae24f1": 103.735603, + "1ae2854": 120.09674, + "1aed5dc": 136.483316, + "1aef832": 136.483316, + "1af9cfa": 87.978308, + "1afeb67": 83.102429, + "1b13f04": 98.684245, + "1b18942": 73.925764, + "1b1913f": 74.057771, + "1b1c34d": 136.483316, + "1b1dcf6": 167.943677, + "1b1eee5": 167.943677, + "1b1f73d": 202.068005, + "1b2137c": 61.425958, + "1b254f8": 158.529677, + "1b28f6b": 60.137103, + "1b2b6cf": 218.915469, + "1b3d47d": 291.350118, + "1b445ce": 82.235192, + "1b4f797": 57.686768, + "1b5081e": 651.383996, + "1b526a2": 202.068005, + "1b544ab": 201.795038, + "1b5c1a9": 103.735603, + "1b62531": 287.865513, + "1b6ed54": 61.425958, + "1b85263": 108.861018, + "1b8b4e7": 98.150341, + "1b95281": 96.509883, + "1b9dceb": 74.621977, + "1b9f856": 79.004585, + "1ba07f5": 73.113077, + "1ba390a": 93.93915, + "1bbd711": 133.687311, + "1bc194f": 138.251907, + "1bc28ae": 129.377237, + "1bc8d55": 209.601193, + "1bc972c": 53.984241, + "1bcb455": 167.943677, + "1bcb6c6": 60.137103, + "1bccebd": 136.483316, + "1bd5d5f": 99.533365, + "1bdb819": 53.535246, + "1be8615": 227.44394, + "1be8f07": 525.374747, + "1bf4d76": 93.93915, + "1bf9721": 51.649728, + "1bfaa0f": 227.44394, + "1bfb854": 98.150341, + "1c0c034": 237.590325, + "1c0f0d6": 563.241394, + "1c11c55": 219.291397, + "1c13579": 72.523807, + "1c1bb1f": 120.389726, + "1c1c656": 209.601193, + "1c21210": 93.672488, + "1c22cf9": 76.792243, + "1c25eb1": 61.425958, + "1c299c5": 248.954796, + "1c2ac7a": 128.747771, + "1c2c464": 106.640778, + "1c31e90": 73.925764, + "1c39e96": 98.684245, + "1c3ef81": 82.235192, + "1c40ae6": 66.136913, + "1c414f2": 55.393299, + "1c48d4b": 60.137103, + "1c4a762": 120.389726, + "1c53a7f": 145.937382, + "1c55d86": 77.28048, + "1c60274": 79.015914, + "1c64cfc": 76.826697, + "1c65635": 106.656738, + "1c67f9c": 98.684245, + "1c68cd4": 395.754538, + "1c6984e": 108.861018, + "1c6c710": 136.483316, + "1c72bfb": 163.387529, + "1c76020": 139.232272, + "1c77c0e": 99.609222, + "1c7e197": 125.771444, + "1c8641e": 133.03799, + "1c86789": 95.229582, + "1c89113": 75.894682, + "1c97cf2": 651.383996, + "1ca3564": 120.09674, + "1cb27f2": 106.656738, + "1cb554e": 83.081041, + "1cb6659": 651.383996, + "1cc0636": 89.49123, + "1cc0b24": 53.738899, + "1cc2b88": 62.703563, + "1cc47fc": 807.789116, + "1cc4dd1": 227.44394, + "1cc84f1": 96.984723, + "1ccf91d": 81.318681, + "1cd0e13": 93.39095, + "1cd2a90": 60.137103, + "1cd7351": 79.015914, + "1cd7717": 223.471128, + "1cd7c2e": 87.534033, + "1cd7daa": 563.241394, + "1cdebe0": 133.03799, + "1ce083a": 395.754538, + "1ce3960": 125.771444, + "1ce4995": 267.332157, + "1ce53fb": 291.350118, + "1ce7bd9": 80.913027, + "1ceb856": 78.972724, + "1ceebcd": 111.058122, + "1cefe54": 83.027177, + "1cf7f11": 62.703563, + "1cfa59c": 88.059968, + "1cfc8b7": 53.738899, + "1d07c9f": 103.735603, + "1d07e1a": 89.49123, + "1d0ce87": 74.621977, + "1d0d96c": 93.93915, + "1d10c25": 136.483316, + "1d14180": 60.137103, + "1d14cdc": 201.795038, + "1d15df5": 88.656425, + "1d20b78": 241.698955, + "1d27b17": 77.101773, + "1d282b1": 61.425958, + "1d2d50f": 98.150341, + "1d2df56": 209.601193, + "1d2ff18": 51.675738, + "1d32f58": 110.166789, + "1d36b1d": 218.915469, + "1d36df8": 79.015914, + "1d3e0cc": 107.318674, + "1d3e756": 104.825819, + "1d41aca": 106.640778, + "1d45d3e": 81.906502, + "1d46e6d": 55.393299, + "1d50090": 73.925764, + "1d5012c": 73.925764, + "1d52546": 96.806015, + "1d536c3": 60.137103, + "1d55347": 158.529677, + "1d6049b": 87.685476, + "1d6137c": 158.529677, + "1d61bb7": 119.024153, + "1d65880": 241.698955, + "1d6a897": 72.523807, + "1d71e80": 79.015914, + "1d760c6": 62.703563, + "1d77f1d": 125.771444, + "1d79243": 366.596566, + "1d7b20f": 73.113077, + "1d7cb11": 93.39095, + "1d86a10": 61.425958, + "1d8d3c6": 202.068005, + "1d915ff": 61.425958, + "1d92ad2": 106.656738, + "1d945e9": 563.241394, + "1d9a544": 87.978308, + "1da0270": 80.137579, + "1da9e5b": 81.904954, + "1da9fd6": 106.640778, + "1dacc91": 62.703563, + "1dadf2b": 83.081041, + "1daf82a": 51.675738, + "1db0994": 103.735603, + "1db410d": 219.291397, + "1db8fe7": 73.872866, + "1dbb110": 112.271377, + "1dbdbea": 104.446993, + "1dbeaa3": 77.28048, + "1dc3684": 125.771444, + "1dc5517": 113.009076, + "1dc5917": 79.015914, + "1dc9122": 115.713997, + "1dcce79": 94.376801, + "1dcf144": 98.150341, + "1dcf293": 108.861018, + "1dcfbb2": 75.894682, + "1ddcf3c": 94.935822, + "1ddd671": 209.601193, + "1deb5fd": 478.506884, + "1ded764": 163.387529, + "1df4aa0": 408.933627, + "1df6dfb": 197.451151, + "1df916a": 120.09674, + "1e0312b": 97.960995, + "1e050b8": 223.471128, + "1e05e0c": 72.566518, + "1e07125": 97.603852, + "1e09557": 99.284857, + "1e0ec0b": 88.059968, + "1e1166f": 833.648852, + "1e11733": 129.377237, + "1e126f8": 162.697123, + "1e15309": 115.713997, + "1e2c1f1": 51.675738, + "1e2d4d2": 74.621977, + "1e2e9de": 120.09674, + "1e311f5": 73.925764, + "1e35304": 79.015914, + "1e38375": 75.442197, + "1e3d6f0": 111.058122, + "1e3f661": 80.471866, + "1e457f1": 88.176835, + "1e47112": 167.943677, + "1e4bf1b": 218.915469, + "1e4eb10": 395.754538, + "1e52980": 78.972724, + "1e537de": 105.408318, + "1e53fef": 83.102429, + "1e54104": 98.150341, + "1e5870e": 97.603852, + "1e5b079": 162.697123, + "1e5b975": 95.229582, + "1e5b9c4": 81.906502, + "1e5c11f": 238.123503, + "1e5cdec": 529.13895, + "1e60db0": 112.271377, + "1e60e3c": 202.681466, + "1e62aea": 202.681466, + "1e659c4": 95.502219, + "1e675c0": 236.897201, + "1e73157": 563.241394, + "1e77d23": 51.681649, + "1e7fb87": 197.451151, + "1e82f55": 84.833032, + "1e8336a": 366.596566, + "1e8652f": 60.137103, + "1e87e36": 74.057771, + "1e893b9": 223.230556, + "1e9125b": 223.471128, + "1e98e9b": 97.849232, + "1e9ae39": 158.529677, + "1ea0ab7": 62.703563, + "1ea5080": 279.951665, + "1ea7124": 158.529677, + "1eaf3b8": 66.136913, + "1eb1254": 109.341399, + "1eb408a": 279.951665, + "1eb9abd": 112.271377, + "1ebb239": 80.137579, + "1ec79db": 91.770991, + "1ecc45c": 119.011538, + "1ecd9c2": 79.004585, + "1ecf535": 209.601193, + "1edaced": 408.933627, + "1edb5f0": 169.608714, + "1eefa95": 143.009776, + "1ef26c5": 91.770991, + "1ef3b53": 129.377237, + "1ef5a52": 670.985186, + "1ef773e": 87.978308, + "1ef9287": 82.235192, + "1efcde5": 180.030022, + "1f08b51": 79.015914, + "1f0959b": 57.686768, + "1f09c40": 241.698955, + "1f0ede7": 60.137103, + "1f18768": 65.441154, + "1f18ea8": 279.951665, + "1f1c431": 248.954796, + "1f1e236": 93.672488, + "1f218f2": 78.972724, + "1f23a69": 136.483316, + "1f24364": 167.943677, + "1f24375": 105.408318, + "1f27343": 75.894682, + "1f280ce": 60.137103, + "1f2ce86": 143.009776, + "1f30a84": 135.107822, + "1f3b209": 88.059968, + "1f3e043": 202.681466, + "1f41eb9": 563.241394, + "1f467b3": 91.770991, + "1f491b6": 115.713997, + "1f4e01a": 162.697123, + "1f4f93f": 60.137103, + "1f512b5": 60.137103, + "1f5205e": 136.483316, + "1f6865a": 60.137103, + "1f6cc38": 127.255901, + "1f73abc": 143.529587, + "1f823cc": 83.027177, + "1f861b6": 291.350118, + "1f8c44d": 66.858304, + "1f907a4": 227.44394, + "1f937c5": 218.915469, + "1f9398b": 132.001239, + "1f94b6b": 287.865513, + "1f99050": 87.534033, + "1f9dd35": 93.672488, + "1fa3005": 74.057771, + "1fa38a1": 651.383996, + "1fa51f6": 89.49123, + "1fab05e": 89.842511, + "1faca2f": 99.609222, + "1fb0d21": 89.49123, + "1fc39d7": 287.865513, + "1fc50b1": 73.925764, + "1fc9ecc": 88.059968, + "1fce108": 77.28048, + "1fcefbc": 60.137103, + "1fd0884": 112.271377, + "1fe0eaa": 110.861483, + "1fe6270": 176.231201, + "1fe7613": 106.656738, + "1fe881e": 75.31631, + "1febba9": 96.984723, + "1fed53f": 99.609222, + "1ff2f1b": 79.015914, + "1fff452": 60.137103, + "20e4398": 133.687311, + "20fd4c0": 833.648852, + "21226ae": 158.529677, + "213dcfe": 167.943677, + "2164da7": 136.483316, + "21a9dd6": 88.174033, + "222495c": 529.13895, + "223123f": 57.686768, + "22b6635": 104.825819, + "23e7ade": 87.119828, + "2400f2e": 202.068005, + "248da41": 158.293834, + "249a2b8": 99.284857, + "2502b82": 93.672488, + "2538ed0": 60.137103, + "257c9c5": 85.835803, + "25a0b59": 130.131006, + "26054ba": 291.350118, + "26aada0": 51.649728, + "26b7f73": 96.454878, + "276b2c8": 51.649728, + "27a0418": 93.93915, + "27afe82": 223.230556, + "27d0ff8": 83.102429, + "27d8bd1": 79.06218, + "27e3d7b": 87.490967, + "282a7b4": 133.687311, + "2877cc1": 51.68165, + "28f1677": 60.137103, + "2953aa1": 88.782418, + "2975d9e": 53.318226, + "29f95c4": 529.13895, + "2ab4a50": 120.389726, + "2b426ba": 143.003018, + "2b4e2d5": 74.057771, + "2b7ab63": 120.389726, + "2b9c78d": 84.165005, + "2be0986": 113.871008, + "2cfada2": 53.318226, + "2d6c3d0": 291.350118, + "2da4f4c": 87.119828, + "2dd0811": 87.490967, + "2dde9bb": 158.529677, + "2e4f80d": 219.291397, + "2e9f3b5": 146.421524, + "2ec9e52": 69.022253, + "2f01c64": 143.529587, + "2f0fd21": 76.792243, + "2f6439d": 76.792243, + "2f72b04": 807.789116, + "2fc0d7a": 94.278808, + "30228fe": 93.93915, + "30b65e2": 136.483316, + "30d14f7": 66.136913, + "30ef239": 135.107822, + "31125c5": 412.624161, + "31152cb": 202.068005, + "31333df": 223.471128, + "31b4d3f": 287.865513, + "3209b92": 98.684245, + "32280c2": 94.278808, + "325f927": 69.022253, + "329b0ed": 165.353692, + "33b0144": 120.09674, + "34517c6": 158.529677, + "34f3f75": 76.792243, + "3569cf8": 125.771444, + "359778b": 209.601193, + "35bdce1": 119.011538, + "35c454e": 93.39095, + "35e5cdb": 73.925764, + "35f0cba": 201.795038, + "36759c0": 107.318674, + "3684eab": 287.865513, + "36a011d": 60.137103, + "36bfea6": 129.377237, + "372b57b": 279.951665, + "37646c9": 103.735603, + "38771a9": 139.232272, + "38f510f": 93.672488, + "3934da1": 88.338767, + "398cb7c": 227.44394, + "39b1dc8": 167.943677, + "39c94a2": 241.698955, + "3a31be0": 85.835803, + "3a3f49e": 98.325438, + "3a9fb88": 77.101773, + "3ab1d30": 143.009776, + "3adcfe7": 218.915469, + "3b1a760": 106.656738, + "3b28562": 51.649728, + "3b723d4": 202.068005, + "3b7c935": 74.621977, + "3bf076f": 93.672488, + "3c3f295": 60.137103, + "3cb274b": 93.93915, + "3cbb6c7": 72.566518, + "3cbe634": 136.483316, + "3d84480": 88.174033, + "3d924d3": 651.383996, + "3dfb58a": 83.689818, + "3e6dcb6": 119.011538, + "3ec038b": 415.120891, + "3ed7683": 171.871578, + "3f1be84": 60.561899, + "3f472ba": 65.654884, + "3faec3d": 66.136913, + "3fe78f9": 129.266778, + "3feb72d": 62.703563, + "401d7e2": 94.376801, + "402deda": 99.284857, + "404933a": 81.318681, + "4061c90": 223.230556, + "407c34d": 115.713997, + "4087ac1": 106.656738, + "409087d": 325.033996, + "40aa3b2": 59.807643, + "414b02d": 103.735603, + "4197bde": 119.011538, + "41b0f95": 201.795038, + "423d409": 129.377237, + "42964a4": 750.574713, + "42a952a": 807.789116, + "42da45b": 115.713997, + "432f978": 87.490967, + "4354fc5": 133.687311, + "437caff": 60.137103, + "440e361": 79.004585, + "44abb4f": 202.681466, + "44e9793": 167.943677, + "450acd3": 91.770991, + "452c0ec": 81.904954, + "458c79d": 478.506884, + "45c1c7f": 98.684245, + "45f9c27": 74.057771, + "460bcb3": 97.849233, + "460df49": 66.297626, + "461797f": 85.835803, + "4688b07": 130.131006, + "468d0c4": 103.735603, + "46e7fca": 75.894682, + "47aa8cc": 287.865513, + "481655f": 176.231201, + "4864b91": 162.697123, + "4881c46": 243.962185, + "489ffd5": 60.302922, + "4920d3f": 79.015914, + "492b83f": 227.44394, + "49f68b3": 94.278808, + "4a31628": 519.037136, + "4a422e1": 77.368463, + "4a59bb7": 563.241394, + "4a79851": 162.697123, + "4aa2a2a": 97.849232, + "4b23d25": 98.552618, + "4b40218": 120.09674, + "4b9ed85": 83.102429, + "4be94bf": 61.213363, + "4c41c56": 87.490967, + "4c6b7c3": 73.925764, + "4c87f15": 87.685476, + "4cdef4b": 248.954796, + "4ce4ec1": 61.425958, + "4d95852": 143.009776, + "4dc83b1": 238.123503, + "4e26a6c": 106.640778, + "4e9a8ca": 62.703563, + "4ed631d": 92.304046, + "4edb741": 99.609222, + "4edb820": 87.534033, + "4efad1c": 89.49123, + "4f70b3c": 133.03799, + "4fb06db": 83.081041, + "4fcf978": 651.383996, + "4fd1520": 93.93915, + "4fe37f9": 51.649728, + "507a7eb": 78.972724, + "517236e": 219.291397, + "51ae308": 201.795038, + "51b9c26": 93.93915, + "51c004c": 88.174033, + "51c8a5c": 99.284857, + "51d9412": 88.174033, + "51de86b": 130.131006, + "51e2096": 79.06218, + "522a546": 93.672488, + "52cc04c": 120.09674, + "52d1484": 60.137103, + "52e614f": 65.654884, + "5301b11": 201.795038, + "538bd65": 101.546551, + "538f024": 163.387529, + "53b1ba3": 88.174033, + "546aa25": 176.231201, + "5484ca0": 77.28048, + "559bbf2": 91.54995, + "55abc5e": 172.451304, + "5646fdd": 73.925764, + "569b521": 107.318674, + "573fdbf": 83.228517, + "57d003f": 162.697123, + "57d2961": 120.09674, + "57de376": 96.984723, + "57e9dce": 57.686768, + "580224f": 89.49123, + "584adc8": 137.291332, + "588e8fa": 111.058122, + "58c4ca5": 93.93915, + "58c9c5d": 103.735603, + "58d7730": 60.137103, + "590286a": 219.291397, + "5a48bdf": 112.271377, + "5a4a2ee": 412.624161, + "5a978d2": 123.445341, + "5b09682": 75.894682, + "5b1ab5f": 202.068005, + "5b41073": 69.657014, + "5b43a4a": 69.022253, + "5b4a20e": 529.13895, + "5b628de": 162.697123, + "5b6d5bd": 98.684245, + "5b8161f": 59.807643, + "5ccc957": 81.904954, + "5cea1c3": 807.789116, + "5cfa9d1": 209.601193, + "5d2e301": 95.229582, + "5db6f26": 97.960995, + "5ddbef6": 94.376801, + "5ea1f55": 91.770991, + "5eb6b4f": 71.24925, + "5ec239b": 63.244627, + "5f63374": 97.071105, + "5fd3204": 74.621977, + "5ff3018": 360.551623, + "6028c6e": 89.842511, + "605a6de": 87.490967, + "606dcae": 98.684245, + "60ad98e": 279.951665, + "60b507f": 350.030002, + "60dc244": 69.657014, + "60de1df": 72.523807, + "610527e": 93.39095, + "6161dc8": 108.861018, + "61ad049": 227.44394, + "61ae1ec": 93.93915, + "622ac0c": 99.284857, + "622c7eb": 112.271377, + "62c4442": 60.137103, + "62dfd2d": 95.229582, + "638973a": 74.427743, + "640d59b": 80.137579, + "6444f67": 158.529677, + "645a194": 223.471128, + "64e19b6": 130.131006, + "6518ecc": 202.681466, + "654f8c0": 408.933627, + "65aafe7": 60.137103, + "65ac2ea": 209.601193, + "65c09d3": 163.387529, + "662817c": 126.318192, + "663ca38": 201.795038, + "6682e06": 87.490967, + "672002e": 93.39095, + "672a50f": 59.807643, + "675e082": 143.009776, + "67c0ba5": 86.576631, + "6850ed5": 85.835803, + "6851a3c": 273.887382, + "689a3fb": 60.137103, + "68dc670": 73.925764, + "6939c9a": 123.445341, + "694a5dc": 79.06218, + "6980d7a": 120.09674, + "699f6fe": 128.747771, + "69b607b": 241.698955, + "69f8b8e": 57.686768, + "6a14d43": 162.697123, + "6c3e5ec": 227.583834, + "6c76bd7": 227.44394, + "6c995e2": 99.284857, + "6cb445e": 112.271377, + "6ceadae": 80.471866, + "6cf373b": 113.009076, + "6d1e866": 201.795038, + "6d77667": 115.713997, + "6d820e6": 61.425958, + "6da10ca": 69.657014, + "6dbf615": 136.483316, + "6dcdfb3": 169.608714, + "6e0f20e": 562.473414, + "6e616b1": 87.490967, + "6e78b72": 94.376801, + "6e85bcc": 60.137103, + "6ebd15f": 87.685476, + "6f12901": 61.425958, + "6f431c9": 52.562284, + "6f4af29": 115.713997, + "6f9ac87": 107.318674, + "6f9b709": 94.278808, + "6fb117c": 106.640778, + "6fb24b4": 158.529677, + "701cd18": 120.09674, + "705b210": 61.561825, + "70966a9": 127.255901, + "70b60a0": 238.123503, + "7219cf4": 97.849232, + "722cafc": 176.231201, + "724adbd": 136.483316, + "7263bf5": 83.582187, + "728c914": 143.009776, + "72aa2be": 60.137103, + "72c03ec": 93.672488, + "7359c8e": 133.687311, + "74acf7c": 366.596566, + "74b58c1": 108.861018, + "74e07bf": 110.166789, + "750c562": 287.865513, + "7521ca4": 96.984723, + "759749c": 563.241394, + "75d9e47": 88.052798, + "765862d": 66.297626, + "7667b27": 366.596566, + "769aa27": 88.059968, + "76c89e7": 175.575738, + "770db03": 153.258158, + "77994b3": 201.795038, + "77b1594": 88.176835, + "785dd21": 107.318674, + "785f3f9": 75.442197, + "786bd16": 227.44394, + "7878a79": 158.529677, + "78ed7c7": 223.471128, + "792479a": 120.09674, + "792f843": 51.681649, + "793e383": 80.137579, + "79de8bb": 120.09674, + "79ef099": 135.107822, + "79f2ab7": 53.738899, + "7a22fd3": 94.278808, + "7a40e08": 60.137103, + "7a6a528": 60.137103, + "7b6bce5": 112.316932, + "7c2d6af": 101.546551, + "7c88ce5": 84.833032, + "7c90047": 415.120891, + "7d76ff9": 62.703563, + "7d96f3b": 55.393299, + "7da78f0": 563.241394, + "7dec5d4": 127.255901, + "7e1a2a6": 104.825819, + "7e2d120": 162.697123, + "7e32ec0": 115.713997, + "7e7fe30": 130.131006, + "7e85837": 287.865513, + "7ed64b0": 113.009076, + "7eec131": 162.697124, + "7f3af66": 96.984723, + "7f56123": 73.925764, + "7f62003": 80.137579, + "7f84968": 83.027177, + "7fa00cf": 73.925764, + "7fa153d": 74.427743, + "8049cd3": 74.057771, + "808a746": 143.009776, + "816352e": 72.566518, + "81720f2": 72.523807, + "81e3c73": 60.137103, + "8218ff8": 244.305977, + "82b119b": 81.904954, + "836a469": 227.44394, + "83b1b06": 209.601193, + "846e6df": 106.640778, + "848bcfc": 101.546551, + "8510e2e": 94.189593, + "853b5f0": 65.654884, + "8567c69": 287.865513, + "85acf6e": 88.059968, + "85deb9a": 248.954796, + "862273e": 75.442197, + "8638dc9": 93.672488, + "878e6c6": 98.684245, + "881e49e": 81.904954, + "89f632a": 120.09674, + "8a19cba": 209.601193, + "8a57317": 478.506884, + "8b1a0d1": 96.509883, + "8b7e1b6": 267.332157, + "8c0d574": 106.640778, + "8c5e899": 65.441154, + "8cd7168": 120.09674, + "8d10412": 223.230556, + "8d15996": 77.101773, + "8d92aac": 395.754538, + "8e33c6d": 129.574923, + "8e47e0a": 81.904954, + "8e6df85": 120.09674, + "8ef4a62": 79.015914, + "8f2dccf": 89.49123, + "8f46789": 112.271377, + "8f50d1d": 375.443055, + "8f61b5d": 412.624161, + "8f9b04b": 60.137103, + "902be05": 69.022253, + "908f9c9": 60.137103, + "91629cd": 136.483316, + "91d42a8": 60.137103, + "91fe586": 89.842511, + "92132f5": 66.136913, + "923ba84": 227.44394, + "92fcc12": 88.052798, + "933558b": 96.509883, + "93524be": 83.081041, + "936e77e": 96.454878, + "93b1e3b": 60.137103, + "93d5541": 227.44394, + "94509b6": 85.835803, + "94be3f5": 133.03799, + "94de9f8": 87.490967, + "9539a94": 563.241394, + "9581f36": 455.882938, + "95aa957": 96.454878, + "95d28c3": 201.795038, + "95ed7fd": 130.655183, + "95f8b96": 563.241394, + "965b029": 90.022807, + "97d2271": 96.454878, + "9818a7b": 104.825819, + "98b02a4": 81.906502, + "98b12b1": 267.332157, + "98ec6ba": 69.657014, + "994f426": 60.137103, + "9a07d4a": 84.833032, + "9a13b9a": 88.174033, + "9a2fcc3": 98.684245, + "9a5c0d9": 94.376801, + "9a81f68": 87.978308, + "9a8d5f9": 158.293834, + "9adbf36": 120.389726, + "9ae58d0": 90.022807, + "9aea1c4": 83.027177, + "9b17c9b": 74.621977, + "9b48cd8": 96.509883, + "9cea290": 53.318226, + "9da4f77": 136.483316, + "9e9a4a0": 125.771444, + "9f052d0": 80.137579, + "9f4d6f1": 87.534033, + "9f95734": 88.176835, + "a012a26": 75.442197, + "a031170": 76.022789, + "a039894": 525.374747, + "a0454b7": 112.271377, + "a06729a": 519.037136, + "a0b5e82": 76.792243, + "a0cc2a4": 83.081041, + "a20816c": 197.451151, + "a2a2e2e": 133.03799, + "a2b9112": 201.795038, + "a2c65bc": 162.697123, + "a30c2f1": 128.747771, + "a34686d": 111.179476, + "a36a30e": 89.842511, + "a38c704": 133.03799, + "a3c3dfa": 136.483316, + "a3e327c": 120.155909, + "a40995b": 478.506884, + "a41adfe": 79.015914, + "a421c15": 572.433225, + "a42e1fb": 88.174033, + "a4331a5": 103.735603, + "a4d4867": 143.009776, + "a4fc6be": 88.176835, + "a503806": 107.318674, + "a52ca01": 143.009776, + "a53d339": 65.441154, + "a54b2db": 111.444881, + "a55b017": 93.93915, + "a5abd83": 115.713997, + "a5c98ed": 101.546551, + "a5eb94b": 83.027177, + "a61304c": 103.735603, + "a7998f4": 98.684245, + "a7f9374": 120.09674, + "a827c2f": 201.795038, + "a85d3e6": 291.350118, + "a944bde": 53.318226, + "a971ee3": 223.230556, + "a972630": 83.102429, + "a98b986": 112.271377, + "a9c7746": 158.529677, + "a9f0bf3": 201.795038, + "aa305b8": 197.451151, + "aa4ae37": 93.93915, + "aa8261a": 111.058122, + "aaf6987": 87.685476, + "ab32063": 366.596566, + "ab5767e": 143.009776, + "ab96a60": 115.713997, + "aba00fe": 108.861018, + "abc8aee": 115.713997, + "ac53b06": 244.226557, + "ad22fca": 87.534033, + "ad3a56c": 90.022807, + "ad7633a": 519.037136, + "adec509": 79.06218, + "ae7e800": 169.608714, + "af72903": 52.562284, + "afc1791": 176.814418, + "afdf8ce": 158.529677, + "b06371b": 519.037136, + "b084483": 65.441154, + "b0b51fa": 106.656738, + "b0d5dee": 103.735603, + "b1072c1": 119.024153, + "b12a18a": 79.004585, + "b13655a": 415.120891, + "b1eb794": 94.376801, + "b1f6b59": 112.271377, + "b26db48": 60.137103, + "b29075f": 136.483316, + "b2cb8af": 78.972724, + "b34ec02": 98.150341, + "b3b9ce6": 236.897201, + "b3bdd52": 66.297626, + "b42974d": 227.44394, + "b44f8fd": 88.176835, + "b48f657": 98.684245, + "b4e5c07": 88.176835, + "b5233ea": 106.656738, + "b56d9af": 96.454878, + "b5739b8": 197.451151, + "b5e9131": 53.318226, + "b5fb73e": 79.015914, + "b66280d": 82.235192, + "b6d51fd": 162.697123, + "b767984": 72.566519, + "b77de6a": 158.529677, + "b783dae": 833.648852, + "b90472a": 83.081041, + "b910bfb": 80.137579, + "b92b3b0": 73.925764, + "b947449": 77.101773, + "b959120": 236.897201, + "b96b665": 97.849232, + "b970d9a": 87.685476, + "b9fa4af": 84.833032, + "ba6302f": 99.609222, + "bade9f1": 162.697123, + "bb514db": 120.389726, + "bb7aaff": 98.150341, + "bb7d091": 103.735603, + "bbcdb10": 201.795038, + "bc5cfa5": 87.685476, + "bc64f49": 80.137579, + "bc7a1f4": 60.137103, + "bc8e8c6": 130.131006, + "bc9aff8": 65.654884, + "bd14427": 79.06218, + "bd87c18": 125.771444, + "bd89eb3": 201.795038, + "be25791": 129.574923, + "be3147f": 73.925764, + "bebdd41": 88.052798, + "becad20": 79.015914, + "bef9b3d": 53.318226, + "bf481d9": 241.698955, + "bf99122": 60.137103, + "bfaf096": 73.113077, + "c05715c": 101.546551, + "c0bc2fa": 60.137103, + "c0e2ef5": 167.943677, + "c123ddc": 80.913027, + "c1351c9": 110.166789, + "c18a3b5": 79.015914, + "c2420c2": 202.681466, + "c285110": 84.833032, + "c32fba4": 103.735603, + "c36f019": 833.648852, + "c3912b5": 136.483316, + "c48b0f7": 651.383996, + "c54cef3": 103.735603, + "c5a1aac": 59.807643, + "c5c7253": 129.377237, + "c5dcf84": 563.241394, + "c6176a9": 115.713997, + "c61da82": 408.933627, + "c6373ab": 95.229582, + "c69f571": 65.441154, + "c6df201": 106.656738, + "c6f2827": 163.387529, + "c6fa72d": 350.030002, + "c724a8e": 244.226557, + "c7c679a": 60.137103, + "c815af0": 123.445341, + "c894f67": 95.229582, + "c8b476b": 94.376801, + "c8ba76f": 88.174033, + "c8ff47b": 83.102429, + "c952599": 74.057771, + "c961281": 110.166789, + "c9aa18f": 202.822022, + "c9de0b6": 120.09674, + "cac5000": 88.176835, + "cb2ca5e": 120.09674, + "cb657ca": 143.009776, + "cc2f3f8": 81.318681, + "ccd445e": 97.603853, + "cd184c1": 97.603852, + "cd83bf1": 98.684245, + "cdab08a": 66.297626, + "cdfce2e": 93.672488, + "ce2bb40": 171.871578, + "ce98c3e": 119.024153, + "cf1ae9f": 88.176835, + "cf86081": 651.383996, + "d0116c6": 81.906502, + "d056ddd": 129.574923, + "d0b2693": 130.131006, + "d0fc014": 112.271377, + "d184b05": 106.640778, + "d2b8f24": 350.030002, + "d300b85": 199.706539, + "d308d1f": 83.081041, + "d33fd55": 53.318226, + "d3c9ec8": 83.027177, + "d44f455": 73.925764, + "d4a2967": 129.377237, + "d4a8f85": 120.09674, + "d5098dd": 60.137103, + "d5214d5": 125.771444, + "d598449": 158.529677, + "d59b088": 60.137103, + "d625448": 95.229582, + "d638313": 87.534033, + "d68083c": 76.792243, + "d6bb8aa": 72.523807, + "d6d5131": 130.131006, + "d712663": 93.39095, + "d728b27": 233.253562, + "d75deb2": 74.621977, + "d78868d": 51.681649, + "d7dfbc2": 112.271377, + "d7e97af": 138.251907, + "d7f052d": 96.454878, + "d811511": 167.943677, + "d819d11": 120.09674, + "d85a7c2": 98.684245, + "d8af6dc": 143.529587, + "d8bb960": 223.230556, + "d95803b": 80.137579, + "d982137": 73.925764, + "d9d72d2": 238.123503, + "da15808": 60.137103, + "da475fd": 94.376801, + "da79693": 87.119828, + "daa4242": 120.389726, + "daada28": 85.835803, + "dab7cce": 83.081041, + "daba82d": 563.241394, + "db0f71f": 120.389726, + "db343a1": 73.925764, + "db4c577": 99.284857, + "db78045": 60.137103, + "dbc6a48": 408.933627, + "dbcf3c6": 65.654884, + "dbdd97d": 52.562284, + "dbeb1d7": 52.562284, + "dbf2d5f": 750.574714, + "dc250d4": 158.529677, + "dcd1818": 202.068005, + "dcfed5e": 76.792243, + "dd174aa": 99.609222, + "dd2bb3b": 60.137103, + "dd346e6": 83.081041, + "dd68acc": 97.849232, + "ddd8721": 291.350118, + "de38314": 79.015914, + "de7d3ce": 116.426281, + "dea8aa5": 74.621977, + "dedea98": 85.835803, + "deec456": 94.289124, + "df250b9": 143.529587, + "df5c335": 60.137103, + "df7a937": 279.951665, + "e02e81a": 72.566518, + "e06abee": 136.483316, + "e090db4": 72.523807, + "e09a90b": 98.150341, + "e1220d6": 81.318681, + "e126ba4": 83.027177, + "e13bf52": 415.120891, + "e1faa28": 75.894682, + "e20152c": 279.951665, + "e26d820": 96.509883, + "e2ae847": 73.925764, + "e2bf559": 60.137103, + "e2d2cc8": 112.271377, + "e321c89": 158.529677, + "e385023": 223.471128, + "e4226c8": 120.09674, + "e45d6bf": 53.738899, + "e4781b7": 525.374747, + "e580d94": 163.38753, + "e5a3994": 108.861019, + "e660d69": 98.684245, + "e6872f6": 136.483316, + "e6e4cca": 51.681649, + "e712306": 227.44394, + "e7249f1": 267.332157, + "e73e989": 525.374747, + "e75aea6": 88.059968, + "e776347": 227.44394, + "e778ce8": 120.09674, + "e82070b": 80.137579, + "e86781c": 223.471128, + "e895ba1": 93.93915, + "e8d8aa5": 83.102429, + "e8dec3f": 60.137103, + "e8fbd30": 61.425958, + "e98519b": 53.738899, + "e98b1fe": 366.596566, + "e9955b3": 138.251907, + "e9aeb44": 98.684245, + "e9c65d0": 66.136913, + "e9e35ef": 136.483316, + "eaeea2d": 98.952294, + "eb2f2a5": 120.09674, + "eb6e579": 95.229582, + "eb94001": 279.951665, + "ebd4d1f": 72.523807, + "ebf73f9": 123.445341, + "ec11642": 150.489752, + "ec404a0": 207.871436, + "ec6fa8e": 60.137103, + "ec8c580": 223.471128, + "ed437ab": 651.383996, + "ed78a8f": 87.978308, + "ee0b75a": 176.814418, + "ee0c760": 223.471128, + "ee48b16": 60.137103, + "ee80c7e": 136.483316, + "eeaed0d": 87.978308, + "eef30c1": 135.107822, + "ef257ac": 136.483316, + "efbceb1": 77.28048, + "efbfad6": 519.037136, + "efc40e8": 158.529677, + "f027911": 81.906502, + "f05659a": 79.004585, + "f0a9034": 60.137103, + "f151048": 87.119828, + "f15bee1": 65.654884, + "f169434": 108.861018, + "f179eea": 61.000592, + "f1a0a59": 98.684245, + "f1c0963": 143.529587, + "f20c964": 136.483316, + "f269fac": 83.081041, + "f2d92e1": 83.081041, + "f30334b": 88.176835, + "f32655d": 83.027177, + "f33b994": 72.566518, + "f343cca": 55.393299, + "f3bb079": 87.490967, + "f3bee4b": 145.917987, + "f3fb520": 72.566518, + "f3fdfae": 60.137103, + "f424ead": 395.754538, + "f46a802": 119.011538, + "f4e4b12": 98.150341, + "f4ec092": 89.842511, + "f5256ad": 69.022253, + "f5ecf02": 89.842511, + "f63a4f0": 80.913027, + "f65661f": 176.814418, + "f66dc0b": 136.483316, + "f69af7e": 120.09674, + "f6b5a5d": 60.137103, + "f6bd23d": 98.150341, + "f6ccb86": 143.009776, + "f81bf39": 84.833032, + "f849aca": 59.807643, + "f850b22": 142.728429, + "f87779b": 176.231201, + "f8ee464": 135.107822, + "f912787": 563.241394, + "f95117e": 89.49123, + "f953f1c": 651.383996, + "f957816": 120.09674, + "f969c41": 167.943677, + "f98475b": 291.350118, + "f9c2ba1": 99.284857, + "f9d7735": 83.027177, + "fa62093": 62.703563, + "fa9267f": 87.685476, + "fa9fe1c": 60.137103, + "faa42e9": 120.09674, + "fae216f": 455.882938, + "fae918b": 93.93915, + "faf1e22": 119.024153, + "fb50881": 88.059968, + "fb8986c": 77.101773, + "fbcf227": 87.534033, + "fbd9c5b": 163.387529, + "fbe6b3d": 163.387529, + "fc173b8": 52.562284, + "fc54849": 223.230556, + "fc7a41b": 162.697123, + "fcfaa6e": 136.483316, + "fd57e36": 415.120891, + "fdaebf2": 79.004585, + "fe1d595": 163.387529, + "fe43c7c": 80.137579, + "fe50ba7": 123.445341, + "ff873f4": 123.445341, + "ffa69c7": 77.28048, + "ffc7e44": 279.951665, + "ffee599": 241.698955 + }, + "fallbacks": { + "global_seconds": 186.251665, + "suite_seconds": { + "aiguard::ai_guard_anthropic": 71.550041, + "aiguard::ai_guard_api": 78.261417, + "aiguard::ai_guard_langchain": 75.35538, + "aiguard::ai_guard_litellm_guardrail": 68.748798, + "aiguard::ai_guard_openai": 73.228577, + "aiguard::ai_guard_strands": 72.021457, + "appsec::appsec": 145.532037, + "appsec::appsec_iast_default": 335.500601, + "appsec::appsec_iast_memcheck": 103.223625, + "appsec::appsec_iast_packages": 383.603321, + "appsec::appsec_integrations_django": 200.852122, + "appsec::appsec_integrations_fastapi": 197.673873, + "appsec::appsec_integrations_flask": 100.134211, + "appsec::appsec_integrations_flask_testagent": 619.356741, + "appsec::appsec_integrations_langchain": 97.417243, + "appsec::appsec_integrations_stripe": 97.844366, + "appsec::appsec_threats_django_iast": 434.126679, + "appsec::appsec_threats_django_no_iast": 401.702477, + "appsec::appsec_threats_django_rc": 97.265766, + "appsec::appsec_threats_fastapi_iast": 685.816052, + "appsec::appsec_threats_fastapi_no_iast": 665.128263, + "appsec::appsec_threats_fastapi_rc": 98.523662, + "appsec::appsec_threats_flask_iast": 439.196768, + "appsec::appsec_threats_flask_no_iast": 442.20813, + "appsec::appsec_threats_flask_rc": 99.78697, + "appsec::appsec_threats_tornado_iast": 350.993683, + "appsec::appsec_threats_tornado_no_iast": 348.996299, + "appsec::appsec_threats_tornado_rc": 96.553854, + "appsec::iast_tdd_propagation": 130.282788, + "appsec::sca": 96.980052, + "ci_visibility::ci_visibility": 121.757761, + "ci_visibility::ci_visibility:snapshot": 85.689749, + "ci_visibility::dd_coverage": 149.358475, + "ci_visibility::pytest": 106.242701, + "ci_visibility::pytest_bdd": 75.352922, + "ci_visibility::pytest_benchmark": 70.670903, + "ci_visibility::pytest_flaky": 69.397115, + "ci_visibility::pytest:snapshot": 91.259282, + "ci_visibility::selenium": 81.003507, + "ci_visibility::testing": 181.352458, + "ci_visibility::unittest": 108.880734, + "conftest": 53.682253, + "contrib::aiobotocore": 106.412032, + "contrib::aiohttp": 89.528137, + "contrib::aiohttp_jinja2": 73.312983, + "contrib::aiokafka": 140.810275, + "contrib::aiomysql": 84.813011, + "contrib::aiopg": 86.283917, + "contrib::algoliasearch": 75.155196, + "contrib::aredis": 99.256156, + "contrib::asgi": 140.25795, + "contrib::asyncpg": 80.248632, + "contrib::asynctest": 51.823005, + "contrib::avro": 51.440842, + "contrib::aws_durable_execution_sdk_python": 89.787629, + "contrib::aws_lambda": 70.815997, + "contrib::azure_cosmos": 97.189066, + "contrib::azure_durable_functions": 95.412874, + "contrib::azure_eventhubs": 199.618012, + "contrib::azure_functions": 172.245424, + "contrib::azure_functions:cosmos": 73.278431, + "contrib::azure_functions:eventhubs": 113.76019, + "contrib::azure_functions:servicebus": 185.232627, + "contrib::azure_servicebus": 209.264088, + "contrib::botocore": 290.124253, + "contrib::bottle": 76.683065, + "contrib::celery": 337.247153, + "contrib::cherrypy": 74.210315, + "contrib::consul": 80.127478, + "contrib::datastreams": 62.453175, + "contrib::ddtrace_api": 70.454009, + "contrib::django": 240.392662, + "contrib::django_hosts": 51.445571, + "contrib::django:djangorestframework": 60.263102, + "contrib::dogpile_cache": 85.251571, + "contrib::dramatiq": 69.642446, + "contrib::elasticsearch": 58.209934, + "contrib::falcon": 85.038242, + "contrib::fastapi": 110.717407, + "contrib::flask": 119.286904, + "contrib::gevent": 57.946366, + "contrib::google_cloud_pubsub": 87.308158, + "contrib::graphql": 80.093025, + "contrib::graphql:graphene": 52.754832, + "contrib::grpc": 80.639478, + "contrib::gunicorn": 85.601692, + "contrib::httplib": 80.54249, + "contrib::httpx": 80.641719, + "contrib::integration_registry": 127.23747, + "contrib::jinja2": 69.346468, + "contrib::kafka": 117.0501, + "contrib::kombu": 77.966677, + "contrib::logbook": 75.625453, + "contrib::logging": 98.946749, + "contrib::loguru": 79.796114, + "contrib::mako": 76.582195, + "contrib::mariadb": 80.548226, + "contrib::mlflow": 205.481497, + "contrib::molten": 63.009782, + "contrib::opensearch": 59.241018, + "contrib::opentelemetry": 106.177643, + "contrib::protobuf": 51.420034, + "contrib::psycopg": 93.088734, + "contrib::pylibmc": 71.534046, + "contrib::pymemcache": 76.565084, + "contrib::pymongo": 99.917353, + "contrib::pymysql": 76.521993, + "contrib::pynamodb": 73.303919, + "contrib::pyodbc": 80.482898, + "contrib::pyramid": 87.264154, + "contrib::pytorch": 136.923993, + "contrib::ray": 200.599054, + "contrib::ray_serve": 113.098468, + "contrib::redis": 98.233683, + "contrib::rediscluster": 73.349995, + "contrib::requests": 81.97426, + "contrib::rq": 68.153466, + "contrib::sanic": 95.955191, + "contrib::snowflake": 151.085212, + "contrib::sourcecode": 52.150079, + "contrib::sqlalchemy": 62.624159, + "contrib::starlette": 95.425642, + "contrib::stdlib": 69.240863, + "contrib::structlog": 78.768894, + "contrib::subprocess": 100.507512, + "contrib::tornado": 75.624215, + "contrib::urllib3": 81.693235, + "contrib::valkey": 88.379637, + "contrib::wsgi": 60.695953, + "contrib::yaaredis": 74.477744, + "crashtracker": 83.911653, + "ddtracerun": 151.946395, + "debugging::debugger": 84.137967, + "errortracking::errortracker": 48.879699, + "integration_agent": 243.574955, + "integration_registry": 129.210514, + "integration_testagent": 234.456192, + "internal": 116.925252, + "lib_injection": 185.927978, + "llmobs::anthropic": 130.162299, + "llmobs::claude_agent_sdk": 119.443434, + "llmobs::crewai": 229.204355, + "llmobs::google_adk": 176.993033, + "llmobs::google_genai": 125.940155, + "llmobs::langchain": 208.697793, + "llmobs::langgraph": 129.74645, + "llmobs::litellm": 193.923074, + "llmobs::llama_index": 173.29955, + "llmobs::llmobs": 308.615882, + "llmobs::mcp": 114.094902, + "llmobs::mistralai": 110.557952, + "llmobs::openai": 176.77466, + "llmobs::openai_agents": 149.693571, + "llmobs::pydantic_ai": 118.840419, + "llmobs::vertexai": 152.835892, + "openfeature": 63.532961, + "profiling::profile": 460.665166, + "profiling::profile-memalloc": 93.932706, + "profiling::profile-uwsgi": 75.860205, + "telemetry": 112.299581, + "tracer": 531.602057, + "tracer-uwsgi": 75.349606, + "vendor": 53.485979, + "wrapping": 56.644275 + } + }, + "parameters": { + "estimate_quantile": 0.9, + "half_life_days": 30, + "history_window_days": 90, + "holdout_days": 14, + "minimum_samples": 5, + "sparse_safety_factor": 1.25 + }, + "suite_estimates": { + "contrib::integration_registry": { + "2e9f3b5": 127.23747 + }, + "integration_registry": { + "2e9f3b5": 129.210514 + }, + "tracer": { + "16f089d": 651.383996, + "190d82d": 651.383996, + "1c97cf2": 651.383996, + "3d924d3": 651.383996, + "f953f1c": 651.383996 + }, + "tracer-uwsgi": { + "16f089d": 75.349606, + "190d82d": 75.349606, + "1c97cf2": 75.349606, + "3d924d3": 75.349606, + "f953f1c": 75.349606 + } + }, + "test_sharding": { + "command_fingerprints": { + "21c32b52591cbd41dd16f7df087cb15963733fe59de2ce9b011bf033a77c76b7": { + "minimum_items": 820, + "observed_hash": "759749c", + "source_commit": "3bd95510c20255b89fa41e1ad9822d617c1ffe3d", + "source_pipeline_id": "131076783" + }, + "3d6032832c5128ba1764834c1340c9e50a459fb876aab90bb20c6c13585cb148": { + "minimum_items": 6922, + "observed_hash": "107d2ec", + "source_commit": "3bd95510c20255b89fa41e1ad9822d617c1ffe3d", + "source_pipeline_id": "131076783" + }, + "6c31901e916dd7d21856d6795591a3f3f0be7bbaf6d5c3d5e1d1d1661c08bef9": { + "minimum_items": 187, + "observed_hash": "dbf2d5f", + "source_commit": "aa623634dbf9e9ff55ba89f822ad7dda228ce087", + "source_pipeline_id": "131104148" + }, + "986b1f7f162587fd9fda02ea4a2430b87751117fe67cfa6d3273aa8b301604c8": { + "minimum_items": 2086, + "observed_hash": "b783dae", + "source_commit": "3bd95510c20255b89fa41e1ad9822d617c1ffe3d", + "source_pipeline_id": "131076783" + } + }, + "source": "datadog-test-visibility-items" + }, + "overheads": { + "global_seconds": 0, + "suite_seconds": {}, + "unit_global_seconds": 59.638522, + "unit_stage_seconds": { + "aiguard": 67.654978, + "appsec": 94.484851, + "ci_visibility": 68.357517, + "contrib": 50.501257, + "core": 52.474301, + "debugging": 43.574604, + "errortracking": 22.825299, + "llmobs": 76.693147, + "profiling": 50.360256 + }, + "unit_suite_seconds": {}, + "queue_p90_seconds": 1.138776, + "sample_count": 867, + "matched_session_count": 1864 + } +} diff --git a/conftest.py b/conftest.py index b1fc2733be1..4a109b703e4 100644 --- a/conftest.py +++ b/conftest.py @@ -6,7 +6,10 @@ Hook reference: https://docs.pytest.org/en/3.10.1/reference.html#hook-reference """ +import hashlib +import json import os +from pathlib import Path import re import sys from time import time @@ -14,6 +17,10 @@ import hypothesis import pytest +from scripts.ci_allocation.planner import AllocationError +from scripts.ci_allocation.runtime import build_runtime_inventory +from scripts.ci_allocation.runtime import write_runtime_inventory + # DEV: Enable "testdir" fixture https://docs.pytest.org/en/stable/reference.html#testdir pytest_plugins = ("pytester",) @@ -51,13 +58,52 @@ def pytest_configure(config): if os.getenv("CI") != "true": return - # Write JUnit xml results to a file that contains this process' PID - # This ensures running pytest multiple times does not overwrite previous results - # e.g. test-results/junit.xml -> test-results/junit.1797.xml + # AIDEV-NOTE: Keep the allocation identity and execution-metadata digest in the + # filename even though they are also testsuite properties; + # record_testsuite_property is unreliable under xdist. + # Write JUnit XML results to a unique file so consecutive Riot environments do not + # overwrite one another. Allocation CI also encodes its strategy and atomic Riot + # hash in the filename because testsuite properties are not reliable under xdist. if config.option.xmlpath: fname, ext = os.path.splitext(config.option.xmlpath) - # DEV: `ext` will contain the `.`, e.g. `.xml` - config.option.xmlpath = "{0}.{1}{2}".format(fname, os.getpid(), ext) + strategy = os.getenv("RIOT_CI_ALLOCATION_STRATEGY") + riot_hash = os.getenv("RIOT_HASH") + test_shard_index = os.getenv("RIOT_TEST_SHARD_INDEX") + test_shard_total = os.getenv("RIOT_TEST_SHARD_TOTAL") + test_shard_identity = None + if test_shard_index and test_shard_total and int(test_shard_total) > 1: + test_shard_identity = f"s{test_shard_index}of{test_shard_total}" + execution_digest = None + if riot_hash: + execution = {} + for env, value in os.environ.items(): + if not env.startswith("RIOT_") or env in { + "RIOT_HASH", + "RIOT_CI_ALLOCATION_STRATEGY", + "RIOT_TEST_SHARD_INDEX", + "RIOT_TEST_SHARD_TOTAL", + }: + continue + name = env[5:] + prefix, _, suffix = name.partition("_") + property_name = f"riot.{prefix.lower()}.{suffix.lower()}" if suffix else f"riot.{prefix.lower()}" + execution[property_name] = value + if "riot.python.version" not in execution: + raise RuntimeError("Riot allocation JUnit metadata requires RIOT_PYTHON_VERSION") + encoded = json.dumps(dict(sorted(execution.items())), sort_keys=True, separators=(",", ":")).encode() + execution_digest = hashlib.sha256(encoded).hexdigest() + identity = filter( + None, + ( + strategy, + riot_hash, + test_shard_identity, + execution_digest, + str(os.getpid()), + ), + ) + # DEV: ext includes the leading period, for example .xml. + config.option.xmlpath = "{}.{}{}".format(fname, ".".join(identity), ext) # Save per-interpreter benchmark results. if config.pluginmanager.hasplugin("benchmark"): @@ -65,6 +111,52 @@ def pytest_configure(config): config.option.benchmark_save = str(time()).replace(".", "_") + gc + "_py%d_%d" % sys.version_info[:2] +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config, items): + """Select one deterministic runtime slice after the Riot command collects tests.""" + shard_index_value = os.getenv("RIOT_TEST_SHARD_INDEX") + shard_total_value = os.getenv("RIOT_TEST_SHARD_TOTAL") + if shard_index_value is None and shard_total_value is None: + return + if shard_index_value is None or shard_total_value is None: + raise pytest.UsageError("Riot runtime test sharding requires both shard index and total") + try: + shard_index = int(shard_index_value) + shard_total = int(shard_total_value) + except ValueError as exc: + raise pytest.UsageError("Riot runtime test shard index and total must be integers") from exc + if shard_total == 1 and shard_index == 1: + return + + suite = os.getenv("CI_ALLOCATION_SUITE", "") + riot_hash = os.getenv("RIOT_HASH", "") + if not suite or not riot_hash: + raise pytest.UsageError("Riot runtime test sharding requires suite and hash identity") + try: + inventory = build_runtime_inventory( + suite=suite, + riot_hash=riot_hash, + shard_index=shard_index, + shard_total=shard_total, + collected_nodeids=[item.nodeid for item in items], + ) + except AllocationError as exc: + raise pytest.UsageError(str(exc)) from exc + + selected = set(inventory["selected_nodeids"]) + deselected = [item for item in items if item.nodeid not in selected] + items[:] = [item for item in items if item.nodeid in selected] + if deselected: + config.hook.pytest_deselected(items=deselected) + + # AIDEV-NOTE: xdist workers must make the same selection, but only one + # process writes the shared inventory artifact. + worker = os.getenv("PYTEST_XDIST_WORKER") + if worker in (None, "gw0"): + path = Path("test-results") / (f"ci-test-shard-inventory.{riot_hash}.{shard_index}-of-{shard_total}.json") + write_runtime_inventory(path, inventory) + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_runtest_makereport(item, call): # Attach the outcome of the test (failed, passed, skipped) to the test node so that fixtures diff --git a/ddtrace/internal/writer/writer.py b/ddtrace/internal/writer/writer.py index fbb5c7e242d..166c1e7d8a9 100644 --- a/ddtrace/internal/writer/writer.py +++ b/ddtrace/internal/writer/writer.py @@ -9,12 +9,14 @@ from typing import Any from typing import Callable from typing import Optional +from typing import Sequence from typing import TextIO from ddtrace.internal.dist_computing.utils import in_ray_job from ddtrace.internal.hostname import get_hostname import ddtrace.internal.native as native from ddtrace.internal.native import AgentResponse +from ddtrace.internal.native._native import SpanData from ddtrace.internal.native_runtime import get_native_runtime from ddtrace.internal.runtime import get_runtime_id from ddtrace.internal.settings import env @@ -60,7 +62,6 @@ if TYPE_CHECKING: # pragma: no cover - from ddtrace._trace.span import Span # noqa:F401 from ddtrace.vendor.dogstatsd import DogStatsd from .utils.http import ConnectionType # noqa:F401 @@ -129,7 +130,7 @@ def stop(self, timeout: Optional[float] = None) -> None: pass @abc.abstractmethod - def write(self, spans: Optional[list["Span"]] = None) -> None: + def write(self, spans: Optional[Sequence[SpanData]] = None) -> None: pass @abc.abstractmethod @@ -161,7 +162,7 @@ def recreate( def stop(self, timeout: Optional[float] = None) -> None: return - def write(self, spans: Optional[list["Span"]] = None) -> None: + def write(self, spans: Optional[Sequence[SpanData]] = None) -> None: if not spans: return encoded = self.encoder.encode_traces([spans]) @@ -407,7 +408,7 @@ def write(self, spans=None): if self._sync_mode: self.flush_queue() - def _write_with_client(self, client: WriterClientBase, spans: Optional[list["Span"]] = None) -> None: + def _write_with_client(self, client: WriterClientBase, spans: Optional[Sequence[SpanData]] = None) -> None: if spans is None: return @@ -1087,13 +1088,13 @@ def _send_payload(self, payload: bytes, count: int, client: WriterClientBase): ) ) - def write(self, spans: Optional[list["Span"]] = None) -> None: + def write(self, spans: Optional[Sequence[SpanData]] = None) -> None: for client in self._clients: self._write_with_client(client, spans=spans) if self._sync_mode: self.flush_queue() - def _write_with_client(self, client: WriterClientBase, spans: Optional[list["Span"]] = None) -> None: + def _write_with_client(self, client: WriterClientBase, spans: Optional[Sequence[SpanData]] = None) -> None: if spans is None: return diff --git a/ddtrace/llmobs/_integrations/agent_manifest.py b/ddtrace/llmobs/_integrations/agent_manifest.py new file mode 100644 index 00000000000..b546f22d6fa --- /dev/null +++ b/ddtrace/llmobs/_integrations/agent_manifest.py @@ -0,0 +1,155 @@ +"""Value coercion shared by integrations that build an agent manifest.""" + +import math +import types +from typing import Any +from typing import Optional +from typing import TypeVar +from typing import Union +from typing import cast +from typing import get_args +from typing import get_origin + + +# Bounds this function's own recursion, not the payload. metadata is a caller dict, and nesting it +# past the interpreter's limit raises RecursionError here. The span sanitizer truncates deep values +# too, but it runs after this and so cannot prevent that. +MAX_WIRE_DEPTH = 20 + +# Depth alone does not bound the work. A dict whose children are shared expands into a tree, so 20 +# levels of sharing is 2**20 emitted nodes built from 20 dicts in memory. Cycle detection cannot +# catch that, because a shared child is a legitimate second visit rather than an ancestor. +MAX_WIRE_NODES = 10_000 + +# AIDEV-NOTE: allowlist, not denylist. model_settings is the one field whose key set the caller +# controls, and the dangerous keys are provider-specific by name (extra_headers, openai_user, +# xai_user), so only a closed list of generic inference parameters is safe. Widening it is a +# security decision. +ALLOWED_MODEL_SETTINGS_KEYS = frozenset( + { + "frequency_penalty", + "logit_bias", + "logprobs", + "max_tokens", + "parallel_tool_calls", + "presence_penalty", + "seed", + "stop_sequences", + "temperature", + "timeout", + "tool_choice", + "top_k", + "top_logprobs", + "top_p", + } +) + + +def callable_name(fn: Any) -> str: + """Best recoverable name for a callable. Two lambdas both report , as Python does.""" + return getattr(fn, "__name__", None) or getattr(getattr(fn, "func", None), "__name__", None) or type(fn).__name__ + + +def type_name(candidate: Any) -> str: + """Readable name for a declared type, such as list[Fruit]. + + Assembled from the type's parts because str() qualifies each argument with its defining module. + """ + if candidate is type(None): + return "None" + origin, args = get_origin(candidate), get_args(candidate) + if origin is None or not args: + return getattr(candidate, "__name__", None) or str(candidate) + names = [type_name(arg) for arg in args] + if origin is Union or origin is getattr(types, "UnionType", None): + return " | ".join(names) + return "{}[{}]".format(getattr(origin, "__name__", None) or str(origin), ", ".join(names)) + + +def is_flat_scalar_value(value: Any) -> bool: + """True for a JSON scalar, a flat list of scalars, or a flat mapping of scalars. + + No allowlisted setting nests in its declared type, so this bounds a shape that should not + arrive: model_settings is a TypedDict, nothing validates it at run time, and wire_value would + coerce a nested value and pass it through rather than drop it. + """ + if value is None or isinstance(value, (str, int, float, bool)): + return True + if isinstance(value, (list, tuple)): + return all(item is None or isinstance(item, (str, int, float, bool)) for item in value) + if isinstance(value, dict): + # Numeric only: logit_bias is token id to bias, so a string there is already invalid. + return all( + isinstance(key, (str, int)) and isinstance(item, (int, float)) and not isinstance(item, bool) + for key, item in value.items() + ) + return False + + +T = TypeVar("T") + + +def prune_empty(node: T) -> T: + """Drop every value that means "not configured", depth-first. 0, 0.0 and False are kept. + + Runs once over a finished manifest so a section can assign a field without guarding it, and so a + container emptied by its own children drops too. The cast is internal: the walk rebuilds plain + containers, and the caller's type is preserved by construction. + """ + if isinstance(node, dict): + kept: dict[Any, Any] = {} + for key, value in node.items(): + pruned = prune_empty(value) + if pruned is None: + continue + if isinstance(pruned, (str, bytes, list, tuple, dict, set, frozenset)) and len(pruned) == 0: + continue + kept[key] = pruned + return cast(T, kept) + if isinstance(node, list): + return cast(T, [prune_empty(item) for item in node]) + return node + + +def is_number(value: Any) -> bool: + """A finite JSON number. bool is an int subclass, so True would otherwise ship as true here.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + # isfinite on floats only: an int is never non-finite, and converting a huge one would raise. + return math.isfinite(value) if isinstance(value, float) else True + + +def wire_value(value: Any, depth: int = 0, ancestors: tuple[int, ...] = (), budget: Optional[list[int]] = None) -> Any: + """Coerce a config value to a JSON-native one, or None when it cannot ship. + + A dropped entry costs its whole list but only its own key in a mapping. Compacting a list would + shift the surviving indices, and an ordered field such as memory_policies then describes a + pipeline the agent does not run, so omitting the field beats misreporting it. + + budget is internal: a single-element list counting nodes still allowed across the whole walk. + """ + if budget is None: + budget = [MAX_WIRE_NODES] + if budget[0] <= 0: + return None + budget[0] -= 1 + if value is None or isinstance(value, (bool, int, str)): + return value + if isinstance(value, float): + # NaN and Infinity encode as bare tokens that are not valid JSON. + return value if math.isfinite(value) else None + if isinstance(value, (list, tuple, dict)): + if depth > MAX_WIRE_DEPTH or id(value) in ancestors: + return None + ancestors = ancestors + (id(value),) + if isinstance(value, (list, tuple)): + items = [wire_value(item, depth + 1, ancestors, budget) for item in value] + return None if any(item is None for item in items) else items + if isinstance(value, dict): + coerced: dict[str, Any] = {} + for key, item in value.items(): + wired = wire_value(item, depth + 1, ancestors, budget) + if wired is not None: + coerced[str(key)] = wired + return coerced or None + return None diff --git a/ddtrace/llmobs/_integrations/pydantic_ai.py b/ddtrace/llmobs/_integrations/pydantic_ai.py index a12ef626c06..d7adf5aeb36 100644 --- a/ddtrace/llmobs/_integrations/pydantic_ai.py +++ b/ddtrace/llmobs/_integrations/pydantic_ai.py @@ -1,25 +1,107 @@ +import functools from typing import Any from typing import Optional from typing import Sequence +from typing import get_origin from ddtrace.internal import core +from ddtrace.internal.logger import get_logger from ddtrace.internal.utils import get_argument_value from ddtrace.llmobs._constants import DISPATCH_ON_TOOL_CALL +from ddtrace.llmobs._integrations.agent_manifest import ALLOWED_MODEL_SETTINGS_KEYS +from ddtrace.llmobs._integrations.agent_manifest import callable_name +from ddtrace.llmobs._integrations.agent_manifest import is_flat_scalar_value +from ddtrace.llmobs._integrations.agent_manifest import is_number +from ddtrace.llmobs._integrations.agent_manifest import prune_empty +from ddtrace.llmobs._integrations.agent_manifest import type_name +from ddtrace.llmobs._integrations.agent_manifest import wire_value from ddtrace.llmobs._integrations.base import BaseLLMIntegration from ddtrace.llmobs._utils import _annotate_llmobs_span_data from ddtrace.llmobs._utils import _get_attr from ddtrace.llmobs._utils import get_llmobs_span_kind -from ddtrace.llmobs._utils import load_data_value from ddtrace.llmobs._utils import safe_json +from ddtrace.llmobs.types import AgentCapability +from ddtrace.llmobs.types import AgentInstructionResolver +from ddtrace.llmobs.types import AgentManifest from ddtrace.trace import Span +log = get_logger(__name__) + + # in some cases, PydanticAI uses a different provider name than what we expect PYDANTIC_AI_SYSTEM_TO_PROVIDER = { "google-gla": "google", "google-vertex": "google", } +FRAMEWORK_NAME = "PydanticAI" +_OUTPUT_MARKERS = frozenset({"ToolOutput", "NativeOutput", "PromptedOutput", "TextOutput"}) + + +def _iter_agent_tools(agent: Any): + """Yield (name, tool) for the agent's function tools, de-duped first-wins, across versions.""" + seen: set[str] = set() + tool_dicts: list[dict[str, Any]] = [] + function_tools = getattr(agent, "_function_tools", None) + if function_tools: + tool_dicts.append(function_tools) + else: + function_toolset = getattr(agent, "_function_toolset", None) + user_toolsets: Sequence[Any] = getattr(agent, "_user_toolsets", None) or [] + # Only a FunctionToolset exposes a {name: tool} dict; others are captured as custom. + fn_cls = PydanticAIIntegration._function_toolset_cls() + toolsets = [t for t in user_toolsets if fn_cls is None or isinstance(t, fn_cls)] + if function_toolset is not None: + toolsets.append(function_toolset) + for toolset in toolsets: + tools = getattr(toolset, "tools", None) + if isinstance(tools, dict): + tool_dicts.append(tools) + for tools in tool_dicts: + for name, tool in tools.items(): + if name in seen: + continue + seen.add(name) + yield name, tool + + +def _collect_instructions(agent: Any) -> tuple[list[str], list[Any]]: + """Gather (static_texts, dynamic_resolvers) from an agent's instructions. + + _instructions is a str below 1.63.0 and a mixed list after, so a resolver sits in + _instructions_functions below and inline in _instructions after. The two are populated + exclusively, never both, so reading each in turn cannot double-count. + """ + static_texts: list[str] = [] + dynamic: list[Any] = [] + instructions = getattr(agent, "_instructions", None) + if isinstance(instructions, (list, tuple)): + for entry in instructions: + if isinstance(entry, str): + static_texts.append(entry) + elif callable(entry): + dynamic.append(entry) + elif isinstance(instructions, str): + static_texts.append(instructions) + elif callable(instructions): + dynamic.append(instructions) + for runner in getattr(agent, "_instructions_functions", None) or []: + fn = getattr(runner, "function", runner) + if callable(fn): + dynamic.append(fn) + return static_texts, dynamic + + +def _collect_dynamic_system_prompts(agent: Any) -> list[Any]: + """Dynamic system-prompt resolvers. Static prompts are read straight off agent._system_prompts.""" + dynamic: list[Any] = [] + for runner in getattr(agent, "_system_prompt_functions", None) or []: + fn = getattr(runner, "function", runner) + if callable(fn): + dynamic.append(fn) + return dynamic + class PydanticAIIntegration(BaseLLMIntegration): _integration_name = "pydantic_ai" @@ -78,7 +160,6 @@ def _llmobs_set_tags_agent( agent_instance = kwargs.get("instance", None) agent_name = getattr(agent_instance, "name", None) - self._tag_agent_manifest(span, kwargs, agent_instance) user_prompt = get_argument_value(args, kwargs, 0, "user_prompt", optional=True) # AIDEV-NOTE: When callers like VercelAIAdapter pass all messages via message_history # without setting user_prompt, we fall back to extracting the last user message from @@ -102,6 +183,9 @@ def _llmobs_set_tags_agent( input_value=user_prompt, output_value=result, ) + # Manifest last: the annotate above is not failure-isolated, so a manifest problem must + # not cost the span's name, input and output. + self._tag_agent_manifest(span, kwargs, agent_instance) @staticmethod def _extract_user_prompt_from_message_history(kwargs: dict[str, Any]) -> Optional[str]: @@ -144,6 +228,9 @@ def _llmobs_set_tags_tool( tool_description = ( _get_attr(tool_def, "description", "") if tool_def else _get_attr(tool_instance, "description", "") ) + # str-only: the encoder reprs what it cannot encode, which can disclose the object. + if not isinstance(tool_description, str): + tool_description = "" output_val = None if not span.error: @@ -172,71 +259,310 @@ def _llmobs_set_tags_tool( def _tag_agent_manifest(self, span: Span, kwargs: dict[str, Any], agent: Any) -> None: if not agent: return + # dict() rather than a cast: the consumer takes a plain mapping, and a TypedDict is not + # assignable to dict[str, Any] because a dict value is invariant. + _annotate_llmobs_span_data(span, agent_manifest=dict(self._build_agent_manifest(agent))) + + def _build_agent_manifest(self, agent: Any) -> AgentManifest: + """Build the shared agent manifest from a pydantic-ai Agent. - manifest: dict[str, Any] = {} - manifest["framework"] = "PydanticAI" - manifest["name"] = agent.name if hasattr(agent, "name") and agent.name else "PydanticAI Agent" + Sections are built independently so a framework change inside one cannot blank the rest. Only + declared configuration is read, so the manifest is identical run to run, and a field + pydantic-ai does not expose is omitted rather than invented. + """ + manifest: AgentManifest = {} + for name, section in ( + ("labels", self._manifest_labels), + ("instructions", self._manifest_instructions), + ("model", self._manifest_model), + ("capabilities", self._manifest_capabilities), + ("data_contracts", self._manifest_data_contracts), + ("memory_policies", self._manifest_memory_policies), + ("guardrails", self._manifest_guardrails), + ("agent_settings", self._manifest_agent_settings), + ): + try: + manifest.update(section(agent)) + except Exception: + log.debug("failed to build pydantic_ai agent manifest section %s", name, exc_info=True) + # Sections assign unconditionally so mypy can check every key name against the type; one + # prune here is what drops the fields that mean "not configured". + return prune_empty(manifest) + + def _manifest_labels(self, agent: Any) -> AgentManifest: + """Labels that name the agent. Grouped for failure isolation only; the manifest is flat.""" + fields: AgentManifest = {"framework": FRAMEWORK_NAME} + # AIDEV-NOTE: placeholder per review, matching the span name fallback. Two unnamed agents + # therefore share it, so name is not an identity. + agent_name = getattr(agent, "name", None) + fields["name"] = agent_name if isinstance(agent_name, str) and agent_name else "PydanticAI Agent" + metadata = getattr(agent, "_metadata", None) + # metadata may be a callable from 1.39.0 on. Only a static dict is captured; the resolver + # is never invoked. + wired_metadata = wire_value(metadata) if isinstance(metadata, dict) else None + if wired_metadata is not None: + fields["metadata"] = wired_metadata + return fields + + def _manifest_instructions(self, agent: Any) -> AgentManifest: + """What the agent is told. A resolver's text is only known at run time, so it ships by name.""" + fields: AgentManifest = {} + static_texts, dynamic_instructions = _collect_instructions(agent) + fields["instructions"] = "\n".join(text for text in static_texts if text) + # Not validated upstream; a non-string would ship as a repr. + prompts = [p for p in (getattr(agent, "_system_prompts", None) or ()) if isinstance(p, str)] + fields["system_prompts"] = prompts + extra: list[AgentInstructionResolver] = [] + for kind, resolvers in ( + ("dynamic_instructions", dynamic_instructions), + ("dynamic_system_prompt", _collect_dynamic_system_prompts(agent)), + ): + extra.extend({"type": kind, "name": callable_name(fn)} for fn in resolvers) + fields["extra_instructions"] = extra + return fields + + def _manifest_model(self, agent: Any) -> AgentManifest: + """The model and the inference params the user set, filtered by ALLOWED_MODEL_SETTINGS_KEYS.""" + fields: AgentManifest = {} model = getattr(agent, "model", None) - if model: + if isinstance(model, str): + # First colon, not last: rpartition reads "bedrock:anthropic.claude-v1:0" as "0". + _, _, declared_name = model.partition(":") + fields["model"] = declared_name or model + elif model: model_name, _ = self._get_model_and_provider(model) - if model_name: - manifest["model"] = model_name - if hasattr(agent, "model_settings"): - manifest["model_settings"] = load_data_value(agent.model_settings) - if hasattr(agent, "_instructions"): - instructions = agent._instructions - if isinstance(instructions, list): - instructions = ( - " ".join(instructions) if instructions and all(isinstance(i, str) for i in instructions) else None - ) - manifest["instructions"] = instructions - if hasattr(agent, "_system_prompts"): - manifest["system_prompts"] = agent._system_prompts - manifest["tools"] = self._get_agent_tools(agent) + # AIDEV-NOTE: str-only, for the same reason as the tool description read. model_name is + # annotated str, but a custom Model subclass returns whatever it likes and the encoder + # reprs what it cannot encode, which can carry a connection string. + if isinstance(model_name, str): + fields["model"] = model_name + settings = getattr(agent, "model_settings", None) + if isinstance(settings, dict): + allowed: dict[str, Any] = {} + for key, value in settings.items(): + if key not in ALLOWED_MODEL_SETTINGS_KEYS or not is_flat_scalar_value(value): + continue + # prune_empty drops what wire_value could not encode, so assign it either way. + allowed[key] = wire_value(value) + fields["model_settings"] = allowed + return fields + + def _manifest_capabilities(self, agent: Any) -> AgentManifest: + """Function tools, plus the powers that are not plain functions, by name.""" + fields: AgentManifest = {} + fields["tools"] = self._get_agent_tools(agent) + prepared = [ + fn + for fn in (getattr(agent, "_prepare_tools", None), getattr(agent, "_prepare_output_tools", None)) + if callable(fn) + ] + capabilities: list[AgentCapability] = [] + for kind, names in ( + ("mcp", self._mcp_server_names(agent)), + ("builtin", self._builtin_tool_names(agent)), + ("custom", self._toolset_names(agent)), + ("tool_preparation", [callable_name(fn) for fn in prepared]), + ): + capabilities.extend({"name": name, "type": kind} for name in names if name) + fields["capabilities"] = capabilities + return fields + + def _manifest_data_contracts(self, agent: Any) -> AgentManifest: + """The declared output type by name. pydantic-ai declares no input schema.""" + name = self._output_type_name(agent) + return {"data_contracts": {"output": {"name": name}}} if name else {} + + def _manifest_memory_policies(self, agent: Any) -> AgentManifest: + """The message-history pipeline, order preserved: [trim, summarize] is not [summarize, trim]. + + A repeat is kept for the same reason order is: [trim, trim] runs trim twice. + """ + fields: AgentManifest = {} + processors = [fn for fn in getattr(agent, "history_processors", None) or [] if callable(fn)] + fields["memory_policies"] = [callable_name(fn) for fn in processors] + return fields + + def _manifest_guardrails(self, agent: Any) -> AgentManifest: + """Output validators by name, matching the shape the other integrations already emit.""" + fields: AgentManifest = {} + validators = getattr(agent, "_output_validators", None) or [] + fns = [getattr(v, "function", v) for v in validators] + fields["guardrails"] = [callable_name(fn) for fn in fns if callable(fn)] + return fields - _annotate_llmobs_span_data(span, agent_manifest=manifest) + def _manifest_agent_settings(self, agent: Any) -> AgentManifest: + """Loop-level knobs, as opposed to model params. + + retries is the output-validation budget and tool_retries the per-tool one, so + Agent(retries=3, output_retries=2) reports retries 2 with tool_retries 3. + """ + settings: dict[str, Any] = {} + # 1.107.1 renamed _max_result_retries to _max_output_retries, so fall back to the successor. + retries = getattr(agent, "_max_result_retries", None) + if not is_number(retries): + retries = getattr(agent, "_max_output_retries", None) + for name, value in ( + ("retries", retries), + ("tool_retries", getattr(agent, "_max_tool_retries", None)), + ("tool_timeout", getattr(agent, "_tool_timeout", None)), + # The parameter is not retained; it is normalized into a limiter at construction, and that + # limiter is None when unset, which keeps "unset" distinct from a real value. + ("max_concurrency", getattr(getattr(agent, "_concurrency_limiter", None), "max_running", None)), + ): + if is_number(value): + settings[name] = value + end_strategy = getattr(agent, "end_strategy", None) + if isinstance(end_strategy, str): + settings["end_strategy"] = end_strategy + deps_type = getattr(agent, "_deps_type", None) + # Omit the "no deps" default, NoneType below 2.x and object from 2.x on, so it is not noise. + if isinstance(deps_type, type) and deps_type not in (type(None), object): + settings["deps_type"] = deps_type.__name__ + return {"agent_settings": settings} if settings else {} def _get_agent_tools(self, agent: Any) -> list[dict[str, Any]]: + """Function tools as {name, description?, parameters?}, each exactly once. + + For pydantic-ai below 0.4.4 tools live on the agent's _function_tools. From 0.4.4 on they live + on _function_toolset and on any user-supplied FunctionToolset in _user_toolsets. + """ + tools: list[dict[str, Any]] = [] + for tool_name, tool_instance in _iter_agent_tools(agent): + entry: dict[str, Any] = {"name": tool_name if isinstance(tool_name, str) else str(tool_name)} + # AIDEV-NOTE: str-only. pydantic-ai accepts a non-str description and the encoder reprs + # what it cannot encode, which can carry credentials. + description = getattr(tool_instance, "description", None) + entry["description"] = description if isinstance(description, str) else None + entry["parameters"] = self._tool_parameters(tool_instance) + tools.append(entry) + return tools + + @staticmethod + def _tool_parameters(tool_instance: Any) -> dict[str, dict[str, Any]]: + """Extract {param: {type?, required?}} from a tool's function_schema.json_schema.""" + function_schema = getattr(tool_instance, "function_schema", {}) + json_schema = getattr(function_schema, "json_schema", {}) + if not isinstance(json_schema, dict): + return {} + required = json_schema.get("required") + required_params = {str(param) for param in required} if isinstance(required, (list, tuple, set)) else set() + properties = json_schema.get("properties") + if not isinstance(properties, dict): + return {} + parameters: dict[str, dict[str, Any]] = {} + for param, schema in properties.items(): + # Keys coerced: Tool.from_schema takes a caller json_schema, so a non-str key reaches + # here. The span sanitizer stringifies keys too, so this is belt and braces. + param_dict: dict[str, Any] = {} + if isinstance(schema, dict): + param_dict["type"] = wire_value(schema.get("type")) + if str(param) in required_params: + param_dict["required"] = True + parameters[str(param)] = param_dict + return parameters + + def _builtin_tool_names(self, agent: Any) -> list[str]: + """Provider-side builtin tools by name. _builtin_tools is gone from 2.x, so the key drops.""" + tools = getattr(agent, "_builtin_tools", None) or [] + return [getattr(tool, "kind", None) or type(tool).__name__ for tool in tools] + + def _toolset_names(self, agent: Any) -> list[str]: + """Toolsets that are neither function tools nor MCP servers, so none is silently dropped.""" + mcp_classes = self._mcp_server_classes() + fn_cls = self._function_toolset_cls() + names: list[str] = [] + for toolset in getattr(agent, "_user_toolsets", None) or []: + if (mcp_classes and isinstance(toolset, mcp_classes)) or (fn_cls and isinstance(toolset, fn_cls)): + continue + names.append(self._toolset_name(toolset)) + for toolset in getattr(agent, "_dynamic_toolsets", None) or []: + fn = getattr(toolset, "toolset_func", None) + names.append(callable_name(fn) if callable(fn) else self._toolset_name(toolset)) + return names + + def _output_type_name(self, agent: Any) -> str: + """The declared output type by name. An output function is not a declared type.""" + if not hasattr(agent, "output_type"): + return "" + candidates = [c for c in self._unwrap_output_markers(agent.output_type) if not self._is_output_function(c)] + return " | ".join(type_name(c) for c in candidates) + + @staticmethod + @functools.lru_cache(maxsize=1) + def _mcp_server_classes() -> tuple[type, ...]: + """Every MCP class this pydantic-ai defines, for isinstance filtering. + + All present names, not the first found: at 1.107.x MCPServer and MCPToolset are unrelated + subclasses and matching one files the other as a plain toolset. """ - Extract tools from the agent and format them to be used in the agent manifest. + try: + import pydantic_ai.mcp as mcp_module + except Exception: # noqa: BLE001 - the optional mcp extra may not be installed + return () + classes: list[type] = [] + for name in ("MCPServer", "MCPToolset"): + candidate = getattr(mcp_module, name, None) + if isinstance(candidate, type): + classes.append(candidate) + return tuple(classes) - For pydantic-ai < 0.4.4, tools are stored in the agent's _function_tools attribute. - For pydantic-ai >= 0.4.4, tools are stored in the agent's _function_toolset (tools) and - _user_toolsets (user-defined toolsets) attributes. + @staticmethod + @functools.lru_cache(maxsize=1) + def _function_toolset_cls() -> Optional[type]: + """FunctionToolset for isinstance filtering, or None, in which case nothing is filtered out.""" + try: + from pydantic_ai.toolsets import FunctionToolset + except Exception: # noqa: BLE001 - the toolset module layout varies by version + return None + fn_cls: type = FunctionToolset + return fn_cls + + @staticmethod + def _toolset_name(toolset: Any) -> str: + """Toolset or MCP server name: the id the user set, else the class name. + + AIDEV-NOTE: never read label. Without an id it falls back to repr(self), which carries the + connection config, so only an explicit str id or the class name ships. No URI is emitted at + all, which is what keeps a credential in a server's userinfo, path or query off the wire. """ - tools: dict[str, Any] = {} - if hasattr(agent, "_function_tools"): - tools = getattr(agent, "_function_tools", {}) or {} - elif hasattr(agent, "_user_toolsets") or hasattr(agent, "_function_toolset"): - user_toolsets: Sequence[Any] = getattr(agent, "_user_toolsets", []) or [] - function_toolset = getattr(agent, "_function_toolset", None) - combined_toolsets = list(user_toolsets) + [function_toolset] if function_toolset else user_toolsets - for toolset in combined_toolsets: - tools.update(getattr(toolset, "tools", {}) or {}) - - if not tools: + try: + toolset_id = getattr(toolset, "id", None) + except Exception: # noqa: BLE001 - id is a property on some toolsets and may raise + toolset_id = None + # Require a real string: an id can be any object on a custom toolset. + return toolset_id if isinstance(toolset_id, str) and toolset_id else type(toolset).__name__ + + def _mcp_server_names(self, agent: Any) -> list[str]: + """MCP servers by name. No URI: a server address can carry credentials in any component.""" + mcp_classes = self._mcp_server_classes() + if not mcp_classes: return [] + toolsets = getattr(agent, "_user_toolsets", None) or [] + return [self._toolset_name(t) for t in toolsets if isinstance(t, mcp_classes)] - formatted_tools = [] - for tool_name, tool_instance in tools.items(): - tool_dict: dict[str, Any] = {} - tool_dict["name"] = tool_name - if hasattr(tool_instance, "description"): - tool_dict["description"] = tool_instance.description - function_schema = getattr(tool_instance, "function_schema", {}) - json_schema = getattr(function_schema, "json_schema", {}) - required_params = {param: True for param in json_schema.get("required", [])} - parameters: dict[str, dict[str, Any]] = {} - for param, schema in json_schema.get("properties", {}).items(): - param_dict: dict[str, Any] = {} - if "type" in schema: - param_dict["type"] = schema["type"] - if param in required_params: - param_dict["required"] = True - parameters[param] = param_dict - tool_dict["parameters"] = parameters - formatted_tools.append(tool_dict) - return formatted_tools + @staticmethod + def _unwrap_output_markers(output_type: Any) -> list[Any]: + """Candidate output types, with any ToolOutput-style wrapper replaced by what it wraps. + + Matched by class name rather than isinstance: the wrapper attrs are not exclusive to markers, + so a dataclass with an "output" field would otherwise be unwrapped into its own member. + """ + candidates: list[Any] = [] + for item in output_type if isinstance(output_type, (list, tuple)) else [output_type]: + inner = item + if type(item).__name__ in _OUTPUT_MARKERS: + inner = next( + (v for a in ("output", "outputs", "output_function") if (v := getattr(item, a, None))), item + ) + candidates.extend(inner if isinstance(inner, (list, tuple)) else [inner]) + return candidates + + @staticmethod + def _is_output_function(candidate: Any) -> bool: + """Callable but not a class. The get_origin check keeps list[Fruit] from reading as one.""" + if get_origin(candidate) is not None: + return False + return callable(candidate) and not isinstance(candidate, type) def _register_span(self, span: Span, kind: Any) -> None: if kind == "agent": diff --git a/ddtrace/llmobs/types.py b/ddtrace/llmobs/types.py index 624c06572ce..8ccf48b1df5 100644 --- a/ddtrace/llmobs/types.py +++ b/ddtrace/llmobs/types.py @@ -50,6 +50,46 @@ class ToolDefinition(TypedDict, total=False): version: str +class AgentCapability(TypedDict, total=False): + """One declared capability: an MCP server, a builtin tool, a toolset, or a preparation hook.""" + + name: str + type: str + + +class AgentInstructionResolver(TypedDict, total=False): + """A callable that decides instruction text at run time, recorded by name and never evaluated.""" + + name: str + type: str + + +class AgentManifest(TypedDict, total=False): + """Declared agent configuration, reported on an agent span under _dd.agent_manifest. + + One flat document. Every key is optional because a field the framework does not expose is + omitted rather than emitted empty, so an absent key means "not configured". Only declared + configuration is read, never what a single run resolved, so the document is stable run to run. + """ + + framework: str + name: str + instructions: str + system_prompts: list[str] + extra_instructions: list[AgentInstructionResolver] + model: str + model_settings: dict[str, Any] + agent_settings: dict[str, Any] + tools: list[dict[str, Any]] + capabilities: list[AgentCapability] + data_contracts: dict[str, Any] + guardrails: list[str] + handoffs: list[Any] + handoff_description: str + memory_policies: list[str] + metadata: dict[str, Any] + + class ChatMessage(TypedDict): """A single message in a chat prompt template.""" diff --git a/docs/contributing-testing.rst b/docs/contributing-testing.rst index 3650cc16ed1..eb6fcc8d5cc 100644 --- a/docs/contributing-testing.rst +++ b/docs/contributing-testing.rst @@ -125,6 +125,139 @@ Anatomy of a Riot Command * ``-vv``: Be loud about which tests are being run * ``-k 'test1 or test2'``: Test selection by `keyword expression `_ +How CI allocates Riot environments +---------------------------------- + +Semantic test ownership remains in ``tests/suitespec.py`` and the distributed +``suitespec.yml`` files. CI resolves every selected suite to its Riot environment +hashes, then assigns those hashes to physical GitLab shards. A Riot environment is +normally the smallest allocation unit, so duration-based balancing never changes its +command, Python version, services, environment, retry policy, or timeout. For a +measured long-running pytest command, CI may refine one hash into runtime execution +units such as ``107d2ec@1/3``. Pytest still collects the suite-authored command, then +a repository plugin deterministically selects one disjoint slice of the collected +node IDs in each physical job. No item-level partition appears in ``suitespec.yml``. + +``scripts/gen_gitlab_config.py`` writes ``.gitlab/ci-allocation-plan.json`` with the +current round-robin plan and a duration-balanced plan. Generation fails unless both +plans contain the exact same Riot hash set, with no duplicates or empty shards, and +the execution metadata is represented by the same digest. Runtime slices must also +contain every index from one through their declared total. The plan is retained as a +CI artifact for review. The generator embeds every balanced assignment into the +suite's GitLab ``parallel`` job, and each physical job selects its assignment by +``CI_NODE_INDEX``. The semantic suite and compact matrix representation therefore +remain stable even when an assignment contains sub-hash execution units. + +The active strategy and promotion thresholds are in +``ci/ci-allocation-policy.json``. ``legacy`` is the rollback-safe default. The +duration estimates in ``ci/ci-allocation-runtime-model.json`` are generated from +Datadog Test Visibility session exports joined by semantic suite and +``test.configuration.riot_hash``. Riot hashes identify environments rather than +commands and can be shared by semantically different suites, so suite-scoped +estimates override global hash estimates for those collisions. The model uses a +time-decayed p90, conservative +fallbacks for sparse hashes, and a recent holdout that is not used for fitting. +CI job events are joined by pipeline and job identity. Riot setup and activation +time outside the Test Visibility session is distributed over the atomic hashes in +that job before fitting, so reducing the shard count cannot make real work disappear +from the model. Model fitting fails if any training job is missing that timing +evidence. Per-hash estimates use a compact numeric representation, and the checked-in +model must remain below the size limit in the allocation policy. Failed and canceled +observations are retained as censored reliability evidence but are not treated as +normal durations. + +Sub-hash expansion is fail-closed. A command is eligible only when its command +fingerprint has real Test Visibility item-count evidence and the fitted runtime +exceeds the target. Fallback estimates never create slices because they may describe +a one-test or non-pytest command. Each slice writes a compact collection inventory. +Across all slice artifacts, verification requires the same collection digest, an +exact disjoint union, and no empty slice. + +The balanced strategy targets five minutes of modeled work per Riot shard. Promotion +requires at least a 50 percent reduction in the median Riot critical path over paired +live shadow runs. This objective covers the generated Riot child pipeline, not the +entire required-check wall clock. Package builds, performance benchmarks, downstream +pipelines, and GitHub System Tests have independent execution graphs and must be +measured and optimized separately for an end-to-end feedback-time target. +Balanced sizing is constrained to the legacy topology's total job count. When the +duration target requests more jobs, the planner removes shards with the smallest +modeled critical-path penalty, allowing capacity to move between semantic suites +without increasing the job budget. +Live critical-path measurement uses the actual interval from the first Riot job start +to the last Riot job completion. It does not add semantic-stage maxima because the +generated ``needs`` DAG lets those stages overlap. Modeled maxima are planning scores, +not measured CI runtimes; promotion uses completed same-head shadow runs. + +Use the allocation helper to normalize an export, build a candidate model, and +replay it against the untouched holdout: + +.. code-block:: bash + + $ scripts/ci_allocation_cli.py ingest-datadog \ + --input test-sessions.json --output observations.jsonl + $ scripts/ci_allocation_cli.py ingest-jobs \ + --input ci-jobs.json --output jobs.jsonl + $ scripts/ci_allocation_cli.py build-model \ + --observations observations.jsonl --jobs jobs.jsonl \ + --output candidate-model.json --report historical-replay.json + $ scripts/ci_allocation_cli.py check-ratchet \ + --report historical-replay.json + +Historical pull-request paths provide a second workload view. They are replayed +through the current suitespec rules, so common PR cohorts cannot hide a regression +in less frequent AppSec, integration, CI, or core workloads: + +.. code-block:: bash + + $ scripts/ci_allocation_cli.py export-pr-history \ + --since "2 years ago" --output pr-shapes.jsonl + $ scripts/ci_allocation_cli.py replay-pr-history \ + --pr-history pr-shapes.jsonl --model candidate-model.json \ + --output pr-replay.json + $ scripts/ci_allocation_cli.py check-ratchet --report pr-replay.json + +After historical validation, set ``CI_ALLOCATION_SHADOW=true`` on an explicitly +requested pipeline to add non-blocking balanced jobs beside the required legacy +jobs. Export both strategies' Test Visibility sessions, then build and check the +same-head report: + +.. code-block:: bash + + $ scripts/ci_allocation_cli.py build-live-report \ + --observations shadow-observations.jsonl --jobs shadow-jobs.jsonl \ + --output live-shadow.json + $ scripts/ci_allocation_cli.py check-ratchet --report live-shadow.json + +Before promotion, download the legacy and balanced XML test reports and prove that +the collected ``(Riot hash, class, test, file)`` identities, including duplicate +counts, and Riot execution metadata are identical. Allocation jobs encode the +strategy, Riot hash, and a digest of the Riot execution metadata in each report +filename as a fallback for parallel runs that omit test suite properties: + +.. code-block:: bash + + $ scripts/ci_allocation_cli.py verify-junit \ + --legacy legacy/test-results/junit*.xml \ + --balanced balanced/test-results/junit*.xml \ + --output junit-parity.json + +For every sub-hash command, also download the runtime inventory artifacts and prove +that all collected pytest items were executed exactly once: + +.. code-block:: bash + + $ scripts/ci_allocation_cli.py verify-runtime-shards \ + --plan ci-allocation-plan.json \ + --manifests balanced/test-results/ci-test-shard-inventory.*.json \ + --output runtime-shard-parity.json + +Promotion requires the checked thresholds for sample count, median improvement, +p75, p90, runner time, clean-success rate, and retry rate. A scheduled retuning task +may propose a new model, but it must not change the active strategy automatically. +Activate ``balanced`` only after the historical and live ratchets pass; reverting +the policy to ``legacy`` restores the previous assignment without changing suite +authoring. + Why are my tests failing with 404 errors? ----------------------------------------- diff --git a/releasenotes/notes/pydantic-ai-agent-manifest-canonical-schema-ddbf5af7f1d63366.yaml b/releasenotes/notes/pydantic-ai-agent-manifest-canonical-schema-ddbf5af7f1d63366.yaml new file mode 100644 index 00000000000..ac496b54bb7 --- /dev/null +++ b/releasenotes/notes/pydantic-ai-agent-manifest-canonical-schema-ddbf5af7f1d63366.yaml @@ -0,0 +1,8 @@ +--- +upgrade: + - | + LLM Observability: The ``pydantic_ai`` integration now reports agent configuration using the shared + agent manifest schema. ``framework``, ``name``, ``model``, ``model_settings``, ``instructions``, + ``system_prompts`` and ``tools`` keep their names and meaning. New keys are ``metadata``, + ``extra_instructions``, ``capabilities``, ``data_contracts``, ``memory_policies``, ``guardrails`` + and ``agent_settings``. diff --git a/riotfile.py b/riotfile.py index 751c29c4a12..59bf98e8912 100644 --- a/riotfile.py +++ b/riotfile.py @@ -376,6 +376,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "pip": "<25", }, env={ + "BROWSER": "true", # Prevent webbrowser tests from launching the host browser. "_DD_IAST_PATCH_MODULES": "benchmarks.,tests.appsec.", "DD_IAST_REQUEST_SAMPLING": "100", "DD_IAST_DEDUPLICATION_ENABLED": "false", @@ -3734,15 +3735,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "protobuf": latest, }, ), - # safe_memcpy fast-copy path (process_vm_readv is the default) - Venv( - env={ - "_DD_PROFILING_STACK_FAST_COPY": "1", - }, - pkgs={ - "protobuf": latest, - }, - ), ], ), # Python 3.10 @@ -3776,15 +3768,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "protobuf": latest, }, ), - # safe_memcpy fast-copy path (process_vm_readv is the default) - Venv( - env={ - "_DD_PROFILING_STACK_FAST_COPY": "1", - }, - pkgs={ - "protobuf": latest, - }, - ), ], ), # Python >= 3.11 (excluding 3.14) @@ -3818,15 +3801,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "protobuf": latest, }, ), - # safe_memcpy fast-copy path (process_vm_readv is the default) - Venv( - env={ - "_DD_PROFILING_STACK_FAST_COPY": "1", - }, - pkgs={ - "protobuf": latest, - }, - ), ], ), # Python 3.14 - protobuf 4.22.0 is not compatible (TypeError: Metaclasses with custom tp_new) @@ -3861,15 +3835,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "protobuf": latest, }, ), - # safe_memcpy fast-copy path (process_vm_readv is the default) - Venv( - env={ - "_DD_PROFILING_STACK_FAST_COPY": "1", - }, - pkgs={ - "protobuf": latest, - }, - ), ], ), Venv( diff --git a/scripts/ci_allocation/__init__.py b/scripts/ci_allocation/__init__.py new file mode 100644 index 00000000000..4f4b4dd4771 --- /dev/null +++ b/scripts/ci_allocation/__init__.py @@ -0,0 +1,26 @@ +"""Deterministic CI workload modeling and shard allocation.""" + +from .planner import AllocationError +from .planner import build_suite_plan +from .planner import legacy_round_robin +from .planner import weighted_lpt +from .suites import SuiteVenvInfo +from .suites import calculate_parallelism_from_venvs +from .suites import collect_all_suite_venv_info +from .suites import compute_parallelism +from .suites import compute_runtime_parallelism +from .suites import scale_suites + + +__all__ = [ + "AllocationError", + "SuiteVenvInfo", + "build_suite_plan", + "calculate_parallelism_from_venvs", + "collect_all_suite_venv_info", + "compute_parallelism", + "compute_runtime_parallelism", + "legacy_round_robin", + "scale_suites", + "weighted_lpt", +] diff --git a/scripts/ci_allocation/history.py b/scripts/ci_allocation/history.py new file mode 100644 index 00000000000..4579b83ace6 --- /dev/null +++ b/scripts/ci_allocation/history.py @@ -0,0 +1,823 @@ +"""Normalize historical CI data and build a time-decayed runtime model.""" + +from __future__ import annotations + +from collections import Counter +from collections import defaultdict +from dataclasses import asdict +from dataclasses import dataclass +from dataclasses import replace +from datetime import datetime +from datetime import timedelta +from datetime import timezone +import hashlib +import json +import math +from pathlib import Path +import re +import typing as t + +from .planner import AllocationError +from .planner import legacy_round_robin +from .planner import weighted_lpt + + +@dataclass(frozen=True) +class Observation: + riot_hash: str + suite: str + duration_seconds: float + timestamp: str + status: str + pipeline_id: str + commit_sha: str + shard_index: int + shard_total: int + job_name: str + strategy: str = "legacy" + + +def _parse_timestamp(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _iso_from_nanoseconds(value: t.Any) -> str: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise AllocationError("Datadog test session start must be a nanosecond timestamp") + return datetime.fromtimestamp(float(value) / 1_000_000_000, tz=timezone.utc).isoformat().replace("+00:00", "Z") + + +def suite_from_job_name(job_name: str, stage_name: str) -> tuple[str, int, int]: + """Recover the suitespec identity and shard count from a generated GitLab job.""" + match = re.search(r"\s+(\d+)/(\d+)$", job_name) + shard_index = int(match.group(1)) if match else 1 + shard_total = int(match.group(2)) if match else 1 + base = job_name[: match.start()] if match else job_name + base = base.removesuffix("-allocation-shadow") + prefix = f"{stage_name}/" + clean_name = base[len(prefix) :] if stage_name and base.startswith(prefix) else base + suite = clean_name if stage_name in {"", "core"} else f"{stage_name}::{clean_name}" + return suite, shard_index, shard_total + + +def suite_stage(suite: str) -> str: + return suite.split("::", 1)[0] if "::" in suite else "core" + + +def observation_from_datadog(event: t.Mapping[str, t.Any]) -> Observation: + """Normalize one Datadog test-session event exported from CI Visibility.""" + outer_attributes = event.get("attributes") + if not isinstance(outer_attributes, dict): + raise AllocationError("Datadog event is missing attributes") + attributes = outer_attributes.get("attributes", outer_attributes) + if not isinstance(attributes, dict): + raise AllocationError("Datadog event attributes are malformed") + + test = attributes.get("test") + ci = attributes.get("ci") + git = attributes.get("git") + if not isinstance(test, dict) or not isinstance(ci, dict) or not isinstance(git, dict): + raise AllocationError("Datadog event is missing test, ci, or git metadata") + configuration = test.get("configuration") + pipeline = ci.get("pipeline") + job = ci.get("job") + stage = ci.get("stage") + commit = git.get("commit") + if ( + not isinstance(configuration, dict) + or not isinstance(pipeline, dict) + or not isinstance(job, dict) + or not isinstance(stage, dict) + or not isinstance(commit, dict) + ): + raise AllocationError("Datadog event is missing configuration or CI identity") + + riot_hash = configuration.get("riot_hash") + job_name = job.get("name") + stage_name = stage.get("name") + pipeline_id = pipeline.get("id") + duration = attributes.get("duration") + if not isinstance(riot_hash, str) or not riot_hash: + raise AllocationError("Datadog event is missing test.configuration.riot_hash") + if not isinstance(job_name, str) or not isinstance(stage_name, str): + raise AllocationError("Datadog event is missing its job or stage name") + if not isinstance(pipeline_id, (str, int)) or not str(pipeline_id): + raise AllocationError("Datadog event is missing its pipeline identity") + if isinstance(duration, bool) or not isinstance(duration, (int, float)) or duration <= 0: + raise AllocationError("Datadog event duration must be positive nanoseconds") + + suite, shard_index, shard_total = suite_from_job_name(job_name, stage_name) + strategy = configuration.get("ci_allocation_strategy", "legacy") + if strategy not in {"legacy", "balanced"}: + raise AllocationError("Datadog event has an invalid CI allocation strategy") + return Observation( + riot_hash=riot_hash, + suite=suite, + duration_seconds=float(duration) / 1_000_000_000, + timestamp=_iso_from_nanoseconds(attributes.get("start")), + status=str(test.get("status", "unknown")), + pipeline_id=str(pipeline_id), + commit_sha=str(commit.get("sha", "")), + shard_index=shard_index, + shard_total=shard_total, + job_name=job_name, + strategy=strategy, + ) + + +def load_json_documents(path: Path) -> list[t.Mapping[str, t.Any]]: + text = path.read_text(encoding="utf-8") + try: + value = json.loads(text) + except json.JSONDecodeError: + value = [json.loads(line) for line in text.splitlines() if line.strip()] + if isinstance(value, dict) and isinstance(value.get("data"), list): + value = value["data"] + if isinstance(value, dict): + value = [value] + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise AllocationError("history input must be a JSON object, array, API data response, or JSONL") + return value + + +def load_observations(path: Path) -> list[Observation]: + observations = [] + for item in load_json_documents(path): + if item.get("schema_version") == 1 and "riot_hash" in item: + normalized = dict(item) + normalized.setdefault("shard_index", 1) + normalized.setdefault("strategy", "legacy") + observations.append(Observation(**{key: normalized[key] for key in Observation.__dataclass_fields__})) + else: + observations.append(observation_from_datadog(item)) + return observations + + +def write_observations(path: Path, observations: t.Iterable[Observation]) -> None: + lines = [json.dumps({"schema_version": 1, **asdict(item)}, sort_keys=True) for item in observations] + path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") + + +def _weighted_quantile(values: list[tuple[float, float]], quantile: float) -> float: + if not values: + raise AllocationError("cannot estimate a duration from no observations") + if not 0 < quantile <= 1: + raise AllocationError("duration quantile must be in (0, 1]") + ordered = sorted(values) + threshold = sum(weight for _, weight in ordered) * quantile + cumulative = 0.0 + for value, weight in ordered: + cumulative += weight + if cumulative >= threshold: + return value + return ordered[-1][0] + + +def _fingerprint(observations: t.Iterable[Observation]) -> str: + normalized = [asdict(item) for item in observations] + normalized.sort(key=lambda item: tuple(str(item[key]) for key in sorted(item))) + encoded = json.dumps(normalized, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _estimate(observations: list[Observation], reference: datetime, half_life_days: float, quantile: float) -> float: + weighted = [] + for item in observations: + age_days = max(0.0, (reference - _parse_timestamp(item.timestamp)).total_seconds() / 86400) + weight = 0.5 ** (age_days / half_life_days) + weighted.append((item.duration_seconds, weight)) + return _weighted_quantile(weighted, quantile) + + +def _estimate_timed_values( + values: list[tuple[float, str]], reference: datetime, half_life_days: float, quantile: float +) -> float: + weighted = [] + for value, timestamp in values: + age_days = max(0.0, (reference - _parse_timestamp(timestamp)).total_seconds() / 86400) + weighted.append((value, 0.5 ** (age_days / half_life_days))) + return _weighted_quantile(weighted, quantile) + + +def _resolve_job_strategies(observations: t.Iterable[Observation], jobs: t.Iterable[t.Any]) -> list[t.Any]: + observed_strategies: dict[tuple[str, str], set[str]] = defaultdict(set) + for item in observations: + observed_strategies[(item.pipeline_id, item.job_name)].add(item.strategy) + + resolved = [] + for job in jobs: + observed = observed_strategies.get((job.pipeline_id, job.job_name), set()) + if len(observed) > 1: + raise AllocationError(f"CI job strategy is ambiguous for {job.pipeline_id} {job.job_name}") + if job.strategy == "unknown": + if observed: + job = replace(job, strategy=next(iter(observed))) + elif observed and job.strategy not in observed: + raise AllocationError(f"CI job strategy conflicts with test sessions for {job.pipeline_id} {job.job_name}") + resolved.append(job) + return resolved + + +def build_runtime_model( + observations: list[Observation], + policy: t.Mapping[str, t.Any], + job_observations: t.Optional[list[t.Any]] = None, +) -> dict[str, t.Any]: + """Build end-to-end Riot estimates while reserving the newest holdout.""" + if not observations: + raise AllocationError("runtime modeling requires observations") + latest = max(_parse_timestamp(item.timestamp) for item in observations) + history_days = int(policy["history_window_days"]) + earliest = latest - timedelta(days=history_days) + in_window_all = [item for item in observations if _parse_timestamp(item.timestamp) >= earliest] + successful = [item for item in in_window_all if item.status == "pass" and item.strategy == "legacy"] + if not successful: + raise AllocationError("runtime modeling requires at least one passing observation") + + holdout_days = int(policy["holdout_days"]) + half_life_days = float(policy["half_life_days"]) + quantile = float(policy["estimate_quantile"]) + minimum_samples = int(policy["minimum_samples"]) + sparse_safety_factor = float(policy["sparse_safety_factor"]) + if min(history_days, half_life_days, minimum_samples, sparse_safety_factor) <= 0 or holdout_days < 0: + raise AllocationError("runtime model policy values must be positive") + + holdout_start = latest - timedelta(days=holdout_days) + training = [item for item in successful if _parse_timestamp(item.timestamp) < holdout_start] + holdout = [item for item in successful if _parse_timestamp(item.timestamp) >= holdout_start] + if not training: + raise AllocationError("runtime model training window is empty") + + resolved_jobs = _resolve_job_strategies(in_window_all, job_observations or []) + window_jobs = [job for job in resolved_jobs if earliest <= _parse_timestamp(job.timestamp) <= latest] + training_jobs: dict[tuple[str, str], t.Any] = {} + for job in window_jobs: + if ( + job.strategy != "legacy" + or job.status not in {"pass", "success"} + or _parse_timestamp(job.timestamp) >= holdout_start + ): + continue + key = (job.pipeline_id, job.job_name) + if key in training_jobs: + raise AllocationError(f"runtime modeling received duplicate CI job timing for {key[0]} {key[1]}") + training_jobs[key] = job + + sessions_by_job: dict[tuple[str, str], list[Observation]] = defaultdict(list) + for item in training: + sessions_by_job[(item.pipeline_id, item.job_name)].append(item) + missing_jobs = sorted(set(sessions_by_job) - set(training_jobs)) + if missing_jobs: + raise AllocationError(f"runtime modeling is missing CI job timing for {len(missing_jobs)} training jobs") + + # AIDEV-NOTE: Riot activation and dependency checks occur outside pytest's + # Test Visibility session. Distribute that measured gap over the executed + # Riot environments so packing cannot make setup work disappear; runtime + # slicing explicitly repeats this cost for each new physical execution. + effective_training: list[Observation] = [] + unit_overhead_values: list[tuple[float, str]] = [] + unit_overhead_by_suite: dict[str, list[tuple[float, str]]] = defaultdict(list) + queue_values: list[tuple[float, str]] = [] + for key, items in sessions_by_job.items(): + job = training_jobs[key] + session_seconds = sum(item.duration_seconds for item in items) + if job.duration_seconds < session_seconds: + raise AllocationError(f"CI job duration is shorter than its test sessions for {key[0]} {key[1]}") + unit_overhead = (job.duration_seconds - session_seconds) / len(items) + value = (unit_overhead, job.timestamp) + queue_values.append((job.queue_seconds, job.timestamp)) + for item in items: + unit_overhead_values.append(value) + effective_training.append(replace(item, duration_seconds=item.duration_seconds + unit_overhead)) + unit_overhead_by_suite[item.suite].append(value) + + by_hash: dict[str, list[Observation]] = defaultdict(list) + by_suite_hash: dict[tuple[str, str], list[Observation]] = defaultdict(list) + by_suite: dict[str, list[Observation]] = defaultdict(list) + for item in effective_training: + by_hash[item.riot_hash].append(item) + by_suite_hash[(item.suite, item.riot_hash)].append(item) + by_suite[item.suite].append(item) + + global_seconds = _estimate(effective_training, holdout_start, half_life_days, quantile) + suite_seconds = { + suite: _estimate(items, holdout_start, half_life_days, quantile) for suite, items in sorted(by_suite.items()) + } + estimates: dict[str, float] = {} + for riot_hash, items in sorted(by_hash.items()): + observed = _estimate(items, holdout_start, half_life_days, quantile) + suite_counts = Counter(item.suite for item in items) + suite = min(suite_counts, key=lambda value: (-suite_counts[value], value)) + if len(items) >= minimum_samples: + estimate_seconds = observed + else: + estimate_seconds = max(observed, suite_seconds.get(suite, global_seconds)) * sparse_safety_factor + estimates[riot_hash] = round(estimate_seconds, 6) + + # Riot's hash identifies an environment, not its command. Preserve compact + # global estimates for unique hashes, but scope collisions to the semantic + # suite so a short command cannot inherit an unrelated suite's runtime. + colliding_hashes = {riot_hash for riot_hash, items in by_hash.items() if len({item.suite for item in items}) > 1} + suite_estimates: dict[str, dict[str, float]] = defaultdict(dict) + for (suite, riot_hash), items in sorted(by_suite_hash.items()): + if riot_hash not in colliding_hashes: + continue + observed = _estimate(items, holdout_start, half_life_days, quantile) + estimate_seconds = ( + observed + if len(items) >= minimum_samples + else max(observed, suite_seconds.get(suite, global_seconds)) * sparse_safety_factor + ) + suite_estimates[suite][riot_hash] = round(estimate_seconds, 6) + + global_unit_overhead = _estimate_timed_values(unit_overhead_values, holdout_start, half_life_days, quantile) + suite_unit_overheads = { + suite: _estimate_timed_values(items, holdout_start, half_life_days, quantile) + for suite, items in sorted(unit_overhead_by_suite.items()) + } + queue_seconds = ( + _estimate_timed_values(queue_values, holdout_start, half_life_days, quantile) if queue_values else 0.0 + ) + normalized_jobs = [asdict(job) for job in window_jobs] + normalized_jobs.sort(key=lambda item: tuple(str(item[key]) for key in sorted(item))) + job_fingerprint = ( + hashlib.sha256(json.dumps(normalized_jobs, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + if normalized_jobs + else None + ) + + return { + "schema_version": 1, + "planner_version": "weighted-lpt-v1", + "generated_at": latest.isoformat().replace("+00:00", "Z"), + "dataset": { + "source": "datadog-test-visibility-export", + "fingerprint_sha256": _fingerprint(in_window_all), + "job_fingerprint_sha256": job_fingerprint, + "history_window_days": history_days, + "holdout_days": holdout_days, + "training_observations": len(training), + "holdout_observations": len(holdout), + "job_observations": len(normalized_jobs), + "censored_observations": sum(item.strategy == "legacy" and item.status != "pass" for item in in_window_all), + "status_counts": dict(sorted(Counter(item.status for item in in_window_all).items())), + "training_end": holdout_start.isoformat().replace("+00:00", "Z"), + "window_end": latest.isoformat().replace("+00:00", "Z"), + }, + "parameters": { + "estimate_quantile": quantile, + "half_life_days": half_life_days, + "history_window_days": history_days, + "holdout_days": holdout_days, + "minimum_samples": minimum_samples, + "sparse_safety_factor": sparse_safety_factor, + }, + "fallbacks": { + "global_seconds": round(global_seconds, 6), + "suite_seconds": {suite: round(value, 6) for suite, value in suite_seconds.items()}, + }, + "overheads": { + "global_seconds": 0.0, + "suite_seconds": {}, + "unit_global_seconds": round(global_unit_overhead, 6), + "unit_suite_seconds": {suite: round(value, 6) for suite, value in suite_unit_overheads.items()}, + "queue_p90_seconds": round(queue_seconds, 6), + "sample_count": len(training_jobs), + "matched_session_count": len(training), + }, + "estimates": estimates, + "suite_estimates": {suite: dict(sorted(values.items())) for suite, values in sorted(suite_estimates.items())}, + } + + +def validate_runtime_model(model: t.Mapping[str, t.Any]) -> None: + if model.get("schema_version") != 1 or model.get("planner_version") != "weighted-lpt-v1": + raise AllocationError("unsupported runtime model schema or planner version") + fallbacks = model.get("fallbacks") + estimates = model.get("estimates") + dataset = model.get("dataset") + parameters = model.get("parameters") + overheads = model.get("overheads") + if ( + not isinstance(fallbacks, dict) + or not isinstance(estimates, dict) + or not isinstance(dataset, dict) + or not isinstance(parameters, dict) + or not isinstance(overheads, dict) + ): + raise AllocationError("runtime model is missing dataset, parameters, fallbacks, or estimates") + fallback = fallbacks.get("global_seconds") + if isinstance(fallback, bool) or not isinstance(fallback, (int, float)) or fallback <= 0: + raise AllocationError("runtime model global fallback must be positive") + overhead = overheads.get("global_seconds") + if isinstance(overhead, bool) or not isinstance(overhead, (int, float)) or overhead < 0: + raise AllocationError("runtime model global overhead cannot be negative") + if estimates: + unit_overhead = overheads.get("unit_global_seconds") + sample_count = overheads.get("sample_count") + matched_session_count = overheads.get("matched_session_count") + if ( + isinstance(unit_overhead, bool) + or not isinstance(unit_overhead, (int, float)) + or unit_overhead < 0 + or isinstance(sample_count, bool) + or not isinstance(sample_count, int) + or sample_count <= 0 + or isinstance(matched_session_count, bool) + or not isinstance(matched_session_count, int) + or matched_session_count <= 0 + ): + raise AllocationError("populated runtime models require matched CI job timing") + for riot_hash, item in estimates.items(): + if not isinstance(riot_hash, str): + raise AllocationError("runtime model estimates are malformed") + estimate = item.get("estimate_seconds") if isinstance(item, dict) else item + if isinstance(estimate, bool) or not isinstance(estimate, (int, float)) or estimate <= 0: + raise AllocationError(f"runtime model estimate is invalid for {riot_hash}") + suite_estimates = model.get("suite_estimates", {}) + if not isinstance(suite_estimates, dict): + raise AllocationError("runtime model suite estimates are malformed") + for suite, values in suite_estimates.items(): + if not isinstance(suite, str) or not isinstance(values, dict): + raise AllocationError("runtime model suite estimates are malformed") + for riot_hash, estimate in values.items(): + if ( + not isinstance(riot_hash, str) + or isinstance(estimate, bool) + or not isinstance(estimate, (int, float)) + or estimate <= 0 + ): + raise AllocationError("runtime model suite estimate is invalid") + test_sharding = model.get("test_sharding") + if test_sharding is not None: + if not isinstance(test_sharding, dict) or not isinstance(test_sharding.get("command_fingerprints"), dict): + raise AllocationError("runtime model test sharding evidence is malformed") + for fingerprint, evidence in test_sharding["command_fingerprints"].items(): + if not isinstance(fingerprint, str) or re.fullmatch(r"[0-9a-f]{64}", fingerprint) is None: + raise AllocationError("runtime model test command fingerprint is malformed") + if not isinstance(evidence, dict): + raise AllocationError("runtime model test command evidence is malformed") + minimum_items = evidence.get("minimum_items") + if isinstance(minimum_items, bool) or not isinstance(minimum_items, int) or minimum_items <= 1: + raise AllocationError("runtime model test command evidence requires multiple observed items") + + +def runtime_estimates(model: t.Mapping[str, t.Any], suite: t.Optional[str] = None) -> tuple[dict[str, float], float]: + validate_runtime_model(model) + estimates = { + riot_hash: float(item["estimate_seconds"] if isinstance(item, dict) else item) + for riot_hash, item in model["estimates"].items() + } + if suite is not None: + estimates.update( + { + riot_hash: float(item["estimate_seconds"] if isinstance(item, dict) else item) + for riot_hash, item in model.get("suite_estimates", {}).get(suite, {}).items() + } + ) + return estimates, float(model["fallbacks"]["global_seconds"]) + + +def percentile(values: list[float], quantile: float) -> float: + if not values: + raise AllocationError("cannot calculate a percentile from no values") + ordered = sorted(values) + index = max(0, math.ceil(len(ordered) * quantile) - 1) + return ordered[index] + + +def replay_observations( + observations: list[Observation], + model: t.Mapping[str, t.Any], + *, + holdout_only: bool = True, + target_shard_seconds: t.Optional[float] = None, + maximum_parallelism_per_suite: int = 25, +) -> dict[str, t.Any]: + """Replay legacy and balanced assignments against observed execution durations.""" + validate_runtime_model(model) + estimates, fallback_seconds = runtime_estimates(model) + suite_overheads = model["overheads"].get("suite_seconds", {}) + global_overhead = float(model["overheads"]["global_seconds"]) + unit_suite_overheads = model["overheads"].get("unit_suite_seconds", {}) + unit_stage_overheads = model["overheads"].get("unit_stage_seconds", {}) + global_unit_overhead = float(model["overheads"].get("unit_global_seconds", 0.0)) + selected = [item for item in observations if item.status == "pass"] + training_end = model["dataset"].get("training_end") + if holdout_only and training_end: + cutoff = _parse_timestamp(str(training_end)) + selected = [item for item in selected if _parse_timestamp(item.timestamp) >= cutoff] + if not selected: + raise AllocationError("replay dataset is empty") + + grouped: dict[tuple[str, str, int], dict[str, float]] = defaultdict(dict) + for item in selected: + key = (item.pipeline_id, item.suite, item.shard_total) + unit_overhead = float( + unit_suite_overheads.get( + item.suite, + unit_stage_overheads.get(suite_stage(item.suite), global_unit_overhead), + ) + ) + effective_duration = item.duration_seconds + unit_overhead + grouped[key][item.riot_hash] = max(grouped[key].get(item.riot_hash, 0.0), effective_duration) + + stage_legacy: dict[tuple[str, str], float] = defaultdict(float) + stage_balanced: dict[tuple[str, str], float] = defaultdict(float) + runner_seconds: dict[str, float] = defaultdict(float) + legacy_runner_seconds: dict[str, float] = defaultdict(float) + balanced_runner_seconds: dict[str, float] = defaultdict(float) + for (pipeline_id, suite, shard_total), durations in grouped.items(): + suite_estimates, _unused_fallback = runtime_estimates(model, suite) + shard_count = min(shard_total, len(durations)) + hashes = sorted(durations) + overhead = float(suite_overheads.get(suite, global_overhead)) + legacy = legacy_round_robin(hashes, shard_count) + balanced_shard_count = shard_count + if target_shard_seconds is not None: + if target_shard_seconds <= 0 or maximum_parallelism_per_suite <= 0: + raise AllocationError("historical replay shard policy must be positive") + modeled_work = sum(float(suite_estimates.get(item, fallback_seconds)) for item in hashes) + available_seconds = max(1.0, target_shard_seconds - overhead) + balanced_shard_count = min( + len(hashes), + maximum_parallelism_per_suite, + max(1, math.ceil(modeled_work / available_seconds)), + ) + balanced = weighted_lpt(hashes, balanced_shard_count, suite_estimates, fallback_seconds) + legacy_makespan = max(overhead + sum(durations[item] for item in shard) for shard in legacy) + balanced_makespan = max(overhead + sum(durations[item] for item in shard) for shard in balanced) + stage_key = (pipeline_id, suite_stage(suite)) + stage_legacy[stage_key] = max(stage_legacy[stage_key], legacy_makespan) + stage_balanced[stage_key] = max(stage_balanced[stage_key], balanced_makespan) + runner_seconds[pipeline_id] += sum(durations.values()) + legacy_runner_seconds[pipeline_id] += sum(durations.values()) + overhead * len(legacy) + balanced_runner_seconds[pipeline_id] += sum(durations.values()) + overhead * len(balanced) + + pipeline_legacy: dict[str, float] = defaultdict(float) + pipeline_balanced: dict[str, float] = defaultdict(float) + for (pipeline_id, _stage), seconds in stage_legacy.items(): + pipeline_legacy[pipeline_id] = max(pipeline_legacy[pipeline_id], seconds) + for (pipeline_id, _stage), seconds in stage_balanced.items(): + pipeline_balanced[pipeline_id] = max(pipeline_balanced[pipeline_id], seconds) + legacy_values = list(pipeline_legacy.values()) + balanced_values = [pipeline_balanced[pipeline_id] for pipeline_id in pipeline_legacy] + + def summary(values: list[float]) -> dict[str, float]: + return { + "median_seconds": round(percentile(values, 0.5), 6), + "p75_seconds": round(percentile(values, 0.75), 6), + "p90_seconds": round(percentile(values, 0.9), 6), + } + + legacy_summary = summary(legacy_values) + balanced_summary = summary(balanced_values) + improvement = 1 - balanced_summary["median_seconds"] / legacy_summary["median_seconds"] + legacy_runner_total = sum(legacy_runner_seconds.values()) + balanced_runner_total = sum(balanced_runner_seconds.values()) + return { + "schema_version": 1, + "kind": "historical-holdout-replay", + "dataset_fingerprint_sha256": _fingerprint(selected), + "pipeline_count": len(legacy_values), + "observation_count": len(selected), + "legacy": legacy_summary, + "balanced": balanced_summary, + "median_improvement_ratio": round(improvement, 6), + "runner_seconds_change_ratio": round(balanced_runner_total / legacy_runner_total - 1, 6), + "clean_success_rate_change": 0.0, + "retry_rate_change": 0.0, + "reliability_evidence": "censored-history-only", + "test_execution_seconds": round(sum(runner_seconds.values()), 6), + "legacy_runner_seconds": round(legacy_runner_total, 6), + "balanced_runner_seconds": round(balanced_runner_total, 6), + } + + +def live_shadow_report( + observations: list[Observation], job_observations: t.Optional[list[t.Any]] = None +) -> dict[str, t.Any]: + """Compare paired legacy and balanced executions from opt-in shadow pipelines.""" + if not observations: + raise AllocationError("live shadow reporting requires observations") + + hash_sets: dict[tuple[str, str, str], set[str]] = defaultdict(set) + expected_shards: dict[tuple[str, str, str], set[int]] = defaultdict(set) + expected_shard_totals: dict[tuple[str, str, str], set[int]] = defaultdict(set) + shard_seconds: dict[tuple[str, str, str, int], float] = defaultdict(float) + pipeline_statuses: dict[tuple[str, str], list[str]] = defaultdict(list) + execution_counts: Counter[tuple[str, str, str, str]] = Counter() + for item in observations: + hash_sets[(item.pipeline_id, item.suite, item.strategy)].add(item.riot_hash) + expected_shards[(item.pipeline_id, item.suite, item.strategy)].add(item.shard_index) + expected_shard_totals[(item.pipeline_id, item.suite, item.strategy)].add(item.shard_total) + shard_seconds[(item.pipeline_id, item.strategy, item.suite, item.shard_index)] += item.duration_seconds + pipeline_statuses[(item.pipeline_id, item.strategy)].append(item.status) + execution_counts[(item.pipeline_id, item.strategy, item.suite, item.riot_hash)] += 1 + + pipeline_ids = sorted({item.pipeline_id for item in observations}) + paired = [] + for pipeline_id in pipeline_ids: + strategies = {strategy for seen_pipeline, strategy in pipeline_statuses if seen_pipeline == pipeline_id} + if strategies != {"legacy", "balanced"}: + continue + suites = {suite for seen_pipeline, suite, _strategy in hash_sets if seen_pipeline == pipeline_id} + for suite in suites: + if hash_sets.get((pipeline_id, suite, "legacy")) != hash_sets.get((pipeline_id, suite, "balanced")): + raise AllocationError(f"live shadow hash parity failed for pipeline {pipeline_id}, suite {suite}") + for strategy in ("legacy", "balanced"): + key = (pipeline_id, suite, strategy) + totals = expected_shard_totals[key] + if len(totals) != 1: + raise AllocationError( + f"live shadow shard totals are inconsistent for {pipeline_id} {suite} {strategy}" + ) + total = next(iter(totals)) + if total <= 0 or expected_shards[key] != set(range(1, total + 1)): + raise AllocationError( + f"live shadow test sessions are missing shards for {pipeline_id} {suite} {strategy}" + ) + paired.append(pipeline_id) + if not paired: + raise AllocationError("live shadow data contains no paired pipelines") + + makespans: dict[str, list[float]] = {"legacy": [], "balanced": []} + runner_seconds: dict[str, float] = defaultdict(float) + clean_success: dict[str, int] = defaultdict(int) + retries: dict[str, int] = defaultdict(int) + queue_seconds: dict[str, list[float]] = defaultdict(list) + if job_observations: + resolved_jobs = _resolve_job_strategies(observations, job_observations) + relevant_suites = set(hash_sets) + jobs = [ + job + for job in resolved_jobs + if job.pipeline_id in paired and (job.pipeline_id, job.suite, job.strategy) in relevant_suites + ] + job_counts: Counter[tuple[str, str, str]] = Counter( + (job.pipeline_id, job.strategy, job.job_name) for job in jobs + ) + for pipeline_id in paired: + for strategy in ("legacy", "balanced"): + selected_jobs = [job for job in jobs if job.pipeline_id == pipeline_id and job.strategy == strategy] + if not selected_jobs: + raise AllocationError(f"live shadow CI job timings are missing for {pipeline_id} {strategy}") + for suite in sorted( + suite + for seen_pipeline, suite, seen_strategy in hash_sets + if seen_pipeline == pipeline_id and seen_strategy == strategy + ): + key = (pipeline_id, suite, strategy) + expected_total = next(iter(expected_shard_totals[key])) + suite_jobs = [job for job in selected_jobs if job.suite == suite] + observed_indices = {job.shard_index for job in suite_jobs} + if observed_indices != expected_shards[key] or any( + job.shard_total != expected_total for job in suite_jobs + ): + raise AllocationError( + f"live shadow CI job timings are missing shards for {pipeline_id} {suite} {strategy}" + ) + for job in selected_jobs: + runner_seconds[strategy] += job.duration_seconds + queue_seconds[strategy].append(job.queue_seconds) + starts = [_parse_timestamp(job.timestamp) for job in selected_jobs] + ends = [start + timedelta(seconds=job.duration_seconds) for start, job in zip(starts, selected_jobs)] + # AIDEV-NOTE: Generated Riot jobs use needs and overlap across + # semantic GitLab stages. The actual fanout interval is the + # critical-path evidence; summing stage maxima double-counts it. + makespans[strategy].append((max(ends) - min(starts)).total_seconds()) + clean_success[strategy] += int(all(job.status in {"pass", "success"} for job in selected_jobs)) + retries[strategy] += sum( + max(0, count - 1) + for (seen_pipeline, seen_strategy, _job), count in job_counts.items() + if seen_pipeline == pipeline_id and seen_strategy == strategy + ) + timing_source = "ci-jobs" + else: + for pipeline_id in paired: + for strategy in ("legacy", "balanced"): + relevant = [ + seconds + for (seen_pipeline, seen_strategy, _suite, _shard), seconds in shard_seconds.items() + if seen_pipeline == pipeline_id and seen_strategy == strategy + ] + makespans[strategy].append(max(relevant)) + runner_seconds[strategy] += sum(relevant) + clean_success[strategy] += int( + all(status == "pass" for status in pipeline_statuses[(pipeline_id, strategy)]) + ) + retries[strategy] += sum( + max(0, count - 1) + for (seen_pipeline, seen_strategy, _suite, _hash), count in execution_counts.items() + if seen_pipeline == pipeline_id and seen_strategy == strategy + ) + timing_source = "test-sessions" + + def summary(strategy: str) -> dict[str, float]: + values = makespans[strategy] + return { + "median_seconds": round(percentile(values, 0.5), 6), + "p75_seconds": round(percentile(values, 0.75), 6), + "p90_seconds": round(percentile(values, 0.9), 6), + } + + legacy_summary = summary("legacy") + balanced_summary = summary("balanced") + pipeline_count = len(paired) + return { + "schema_version": 1, + "kind": "live-shadow-replay", + "dataset_fingerprint_sha256": _fingerprint(item for item in observations if item.pipeline_id in set(paired)), + "pipeline_count": pipeline_count, + "legacy": legacy_summary, + "balanced": balanced_summary, + "median_improvement_ratio": round(1 - balanced_summary["median_seconds"] / legacy_summary["median_seconds"], 6), + "runner_seconds_change_ratio": round(runner_seconds["balanced"] / runner_seconds["legacy"] - 1, 6), + "clean_success_rate_change": round( + clean_success["balanced"] / pipeline_count - clean_success["legacy"] / pipeline_count, 6 + ), + "retry_rate_change": round(retries["balanced"] / pipeline_count - retries["legacy"] / pipeline_count, 6), + "exact_hash_parity": True, + "timing_source": timing_source, + "queue": { + strategy: { + "p50_seconds": round(percentile(values, 0.5), 6), + "p90_seconds": round(percentile(values, 0.9), 6), + } + for strategy, values in sorted(queue_seconds.items()) + }, + } + + +def ratchet_violations(report: t.Mapping[str, t.Any], policy: t.Mapping[str, t.Any]) -> list[str]: + """Return reasons a historical or live candidate cannot be promoted.""" + kind = report.get("kind") + if kind == "historical-pr-shape-replay": + limits = policy["pr_shape_replay"] + overall = report["overall"] + violations = [] + if int(report.get("modeled_shape_count", 0)) < int(limits["minimum_shapes"]): + violations.append("PR shape count is below the minimum") + if float(overall.get("median_improvement_ratio", -1)) < float(limits["minimum_median_improvement_ratio"]): + violations.append("PR median improvement is below the ratchet") + if float(overall["balanced"]["p75_seconds"]) >= float(overall["legacy"]["p75_seconds"]): + violations.append("PR p75 did not improve") + if float(overall["balanced"]["p90_seconds"]) > float(overall["legacy"]["p90_seconds"]): + violations.append("PR p90 regressed") + if float(overall["runner_seconds_change_ratio"]) > float(limits["maximum_runner_seconds_increase_ratio"]): + violations.append("PR runner seconds exceed the allowed increase") + for cohort, summary in report["cohorts"].items(): + if float(summary["balanced"]["p90_seconds"]) > float(summary["legacy"]["p90_seconds"]): + violations.append(f"PR cohort p90 regressed: {cohort}") + return violations + if kind == "historical-holdout-replay": + limits = policy["historical_replay"] + elif kind == "live-shadow-replay": + limits = policy["live_shadow"] + else: + raise AllocationError(f"unsupported ratchet report kind: {kind}") + + violations = [] + if int(report.get("pipeline_count", 0)) < int(limits["minimum_runs"]): + violations.append("sample count is below the minimum") + if float(report.get("median_improvement_ratio", -1)) < float(limits["minimum_median_improvement_ratio"]): + violations.append("median improvement is below the ratchet") + if float(report["balanced"]["p75_seconds"]) >= float(report["legacy"]["p75_seconds"]): + violations.append("p75 did not improve") + if float(report["balanced"]["p90_seconds"]) > float(report["legacy"]["p90_seconds"]): + violations.append("p90 regressed") + if float(report.get("runner_seconds_change_ratio", 1)) > float(limits["maximum_runner_seconds_increase_ratio"]): + violations.append("runner seconds exceed the allowed increase") + if kind == "live-shadow-replay": + if report.get("exact_hash_parity") is not True: + violations.append("live shadow Riot hash parity was not proven") + if report.get("timing_source") != "ci-jobs": + violations.append("live shadow timing must come from CI job events") + queue = report.get("queue", {}) + if isinstance(queue, dict) and "legacy" in queue and "balanced" in queue: + legacy_queue = float(queue["legacy"]["p90_seconds"]) + balanced_queue = float(queue["balanced"]["p90_seconds"]) + maximum_queue = legacy_queue * (1 + float(limits["maximum_queue_p90_increase_ratio"])) + if balanced_queue > maximum_queue: + violations.append("queue p90 regressed beyond the allowed increase") + else: + violations.append("live shadow queue evidence is missing") + if float(report.get("clean_success_rate_change", -1)) < 0: + violations.append("clean success rate regressed") + if float(report.get("retry_rate_change", 1)) > 0: + violations.append("retry rate regressed") + return violations + + +def load_json(path: Path) -> dict[str, t.Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise AllocationError(f"{path} must contain a JSON object") + return value + + +def write_json(path: Path, value: t.Mapping[str, t.Any]) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/scripts/ci_allocation/jobs.py b/scripts/ci_allocation/jobs.py new file mode 100644 index 00000000000..2ed6c0516a4 --- /dev/null +++ b/scripts/ci_allocation/jobs.py @@ -0,0 +1,141 @@ +"""Normalize GitLab job timing events used by the CI allocation model.""" + +from __future__ import annotations + +from dataclasses import asdict +from dataclasses import dataclass +from datetime import datetime +from datetime import timezone +import json +from pathlib import Path +import typing as t + +from .history import _iso_from_nanoseconds +from .history import load_json_documents +from .history import suite_from_job_name +from .planner import AllocationError + + +@dataclass(frozen=True) +class JobObservation: + pipeline_id: str + job_id: str + job_name: str + stage_name: str + suite: str + strategy: str + shard_index: int + shard_total: int + duration_seconds: float + queue_seconds: float + status: str + timestamp: str + + +def _duration_seconds(attributes: t.Mapping[str, t.Any], seconds_key: str, nanoseconds_key: str) -> float: + seconds = attributes.get(seconds_key) + if isinstance(seconds, (int, float)) and not isinstance(seconds, bool): + return float(seconds) + nanoseconds = attributes.get(nanoseconds_key) + if isinstance(nanoseconds, (int, float)) and not isinstance(nanoseconds, bool): + return float(nanoseconds) / 1_000_000_000 + raise AllocationError(f"CI job {seconds_key} must be numeric") + + +def _strategy(attributes: t.Mapping[str, t.Any], ci: t.Mapping[str, t.Any], job_name: str) -> str: + test = attributes.get("test", {}) + configuration = test.get("configuration", {}) if isinstance(test, dict) else {} + candidates = ( + attributes.get("ci_allocation_strategy"), + attributes.get("test.configuration.ci_allocation_strategy"), + ci.get("allocation_strategy"), + configuration.get("ci_allocation_strategy") if isinstance(configuration, dict) else None, + ) + for candidate in candidates: + if candidate is None: + continue + if candidate not in {"legacy", "balanced"}: + raise AllocationError("Datadog CI job event has an invalid allocation strategy") + return str(candidate) + if "-allocation-shadow" in job_name: + return "balanced" + # AIDEV-NOTE: A normal job name is not evidence of the legacy strategy after + # promotion. The history layer resolves this value from matching test sessions. + return "unknown" + + +def _timestamp(attributes: t.Mapping[str, t.Any]) -> str: + start = attributes.get("start") + if isinstance(start, str): + parsed = datetime.fromisoformat(start.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + return _iso_from_nanoseconds(start) + + +def job_from_datadog(event: t.Mapping[str, t.Any]) -> JobObservation: + outer_attributes = event.get("attributes", event) + if not isinstance(outer_attributes, dict): + raise AllocationError("Datadog CI job event is missing attributes") + attributes = outer_attributes.get("attributes", outer_attributes) + if not isinstance(attributes, dict): + raise AllocationError("Datadog CI job attributes are malformed") + + ci = attributes.get("ci", {}) + if not isinstance(ci, dict): + raise AllocationError("Datadog CI job event has malformed CI metadata") + pipeline = ci.get("pipeline", {}) + job = ci.get("job", {}) + stage = ci.get("stage", {}) + if not isinstance(pipeline, dict) or not isinstance(job, dict) or not isinstance(stage, dict): + raise AllocationError("Datadog CI job event is missing pipeline, job, or stage metadata") + + pipeline_id = pipeline.get("id", attributes.get("pipeline_id", "")) + job_id = job.get("id", attributes.get("job_id", "")) + job_name = job.get("name", attributes.get("job_name")) + stage_name = stage.get("name", attributes.get("stage_name", "")) + if not isinstance(job_name, str) or not job_name: + raise AllocationError("Datadog CI job event is missing its job name") + if not isinstance(pipeline_id, (str, int)) or not str(pipeline_id): + raise AllocationError("Datadog CI job event is missing its pipeline identity") + if not isinstance(stage_name, str) or not stage_name: + raise AllocationError("Datadog CI job event is missing its stage name") + suite, shard_index, shard_total = suite_from_job_name(job_name, stage_name) + strategy = _strategy(attributes, ci, job_name) + duration = _duration_seconds(attributes, "duration_seconds", "duration") + timing_attributes = dict(attributes) + if "ci.queue_time" not in timing_attributes and "queue_time" in ci: + timing_attributes["ci.queue_time"] = ci["queue_time"] + queue = _duration_seconds(timing_attributes, "queue_seconds", "ci.queue_time") + if duration <= 0 or queue < 0: + raise AllocationError("Datadog CI job duration must be positive and queue time cannot be negative") + return JobObservation( + pipeline_id=str(pipeline_id), + job_id=str(job_id), + job_name=job_name, + stage_name=stage_name, + suite=suite, + strategy=strategy, + shard_index=shard_index, + shard_total=shard_total, + duration_seconds=duration, + queue_seconds=queue, + status=str(ci.get("status", attributes.get("status", "unknown"))), + timestamp=_timestamp(attributes), + ) + + +def load_job_observations(path: Path) -> list[JobObservation]: + observations = [] + for item in load_json_documents(path): + if item.get("schema_version") == 1 and "job_name" in item and "duration_seconds" in item: + observations.append(JobObservation(**{key: item[key] for key in JobObservation.__dataclass_fields__})) + else: + observations.append(job_from_datadog(item)) + return observations + + +def write_job_observations(path: Path, observations: t.Iterable[JobObservation]) -> None: + lines = [json.dumps({"schema_version": 1, **asdict(item)}, sort_keys=True) for item in observations] + path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") diff --git a/scripts/ci_allocation/junit.py b/scripts/ci_allocation/junit.py new file mode 100644 index 00000000000..8653a4bddd4 --- /dev/null +++ b/scripts/ci_allocation/junit.py @@ -0,0 +1,125 @@ +"""Verify collected-test and Riot metadata parity across allocation strategies.""" + +from __future__ import annotations + +from collections import Counter +import hashlib +import json +from pathlib import Path +import re +import typing as t +import xml.etree.ElementTree as ET + +from .planner import AllocationError + + +JUNIT_IDENTITY = re.compile( + r"(?:^|/)junit\.(legacy|balanced)\.([^.]+)(?:\.s([1-9][0-9]*)of([1-9][0-9]*))?" + r"(?:\.([0-9a-f]{64}))?\.\d+\.xml$" +) +REQUIRED_EXECUTION_PROPERTIES = {"riot.python.version"} +PARTITION_PROPERTIES = {"riot.test.shard_index", "riot.test.shard_total"} + + +def _execution_digest(execution: t.Mapping[str, str]) -> str: + encoded = json.dumps(dict(sorted(execution.items())), sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _test_suites(root: ET.Element) -> list[ET.Element]: + if root.tag == "testsuite": + return [root] + return list(root.iter("testsuite")) + + +def collect_junit(paths: t.Iterable[Path], expected_strategy: str) -> tuple[Counter[tuple[str, ...]], dict[str, str]]: + """Collect test identities and execution metadata from JUnit XML artifacts.""" + identities: Counter[tuple[str, ...]] = Counter() + metadata: dict[str, str] = {} + seen_files = 0 + for path in paths: + seen_files += 1 + try: + root = ET.parse(path).getroot() + except (ET.ParseError, OSError) as exc: + raise AllocationError(f"cannot read JUnit artifact {path}: {exc}") from exc + filename_identity = JUNIT_IDENTITY.search(path.as_posix()) + if filename_identity and filename_identity.group(1) != expected_strategy: + raise AllocationError(f"JUnit filename strategy does not match {expected_strategy}: {path}") + for suite in _test_suites(root): + properties = { + str(item.get("name")): str(item.get("value", "")) + for item in suite.findall("./properties/property") + if item.get("name") + } + riot_hash = properties.get("riot.hash") or (filename_identity.group(2) if filename_identity else None) + if not riot_hash: + raise AllocationError(f"JUnit suite in {path} is missing riot.hash") + embedded_strategy = properties.get("riot.ci.allocation_strategy") + if embedded_strategy and embedded_strategy != expected_strategy: + raise AllocationError(f"JUnit strategy does not match {expected_strategy}: {path}") + embedded_index = properties.get("riot.test.shard_index") + embedded_total = properties.get("riot.test.shard_total") + if bool(embedded_index) != bool(embedded_total): + raise AllocationError(f"JUnit runtime shard identity is incomplete: {path}") + if filename_identity and filename_identity.group(3): + if (embedded_index, embedded_total) != (filename_identity.group(3), filename_identity.group(4)): + raise AllocationError(f"JUnit runtime shard identity does not match {path}") + execution = { + key: value + for key, value in properties.items() + if key.startswith("riot.") + and key not in {"riot.hash", "riot.ci.allocation_strategy", *PARTITION_PROPERTIES} + } + filename_digest = filename_identity.group(5) if filename_identity else None + if execution: + missing_properties = REQUIRED_EXECUTION_PROPERTIES - set(execution) + if missing_properties: + raise AllocationError( + f"JUnit suite in {path} is missing Riot execution metadata: {sorted(missing_properties)}" + ) + execution_digest = _execution_digest(execution) + if filename_digest and filename_digest != execution_digest: + raise AllocationError(f"JUnit filename execution metadata does not match {path}") + elif filename_digest: + execution_digest = filename_digest + else: + raise AllocationError(f"JUnit suite in {path} has no Riot execution metadata evidence") + if riot_hash in metadata and metadata[riot_hash] != execution_digest: + raise AllocationError(f"JUnit execution metadata is inconsistent for Riot hash {riot_hash}") + metadata[riot_hash] = execution_digest + for case in suite.findall("./testcase"): + identities[ + ( + riot_hash, + str(case.get("classname", "")), + str(case.get("name", "")), + str(case.get("file", "")), + ) + ] += 1 + if not seen_files: + raise AllocationError(f"no {expected_strategy} JUnit artifacts were provided") + return identities, metadata + + +def verify_junit_parity(legacy_paths: t.Iterable[Path], balanced_paths: t.Iterable[Path]) -> dict[str, t.Any]: + legacy, legacy_metadata = collect_junit(legacy_paths, "legacy") + balanced, balanced_metadata = collect_junit(balanced_paths, "balanced") + if legacy != balanced: + missing = list((legacy - balanced).elements())[:10] + unexpected = list((balanced - legacy).elements())[:10] + raise AllocationError(f"JUnit test identity parity failed: missing={missing}, unexpected={unexpected}") + if legacy_metadata != balanced_metadata: + raise AllocationError("JUnit Riot execution metadata parity failed") + + normalized = sorted((identity, count) for identity, count in legacy.items()) + digest = hashlib.sha256(json.dumps(normalized, separators=(",", ":")).encode()).hexdigest() + return { + "schema_version": 1, + "kind": "junit-allocation-parity", + "test_identity_count": sum(legacy.values()), + "riot_hash_count": len(legacy_metadata), + "test_identity_sha256": digest, + "exact_multiset_parity": True, + "execution_metadata_parity": True, + } diff --git a/scripts/ci_allocation/manifest.py b/scripts/ci_allocation/manifest.py new file mode 100644 index 00000000000..a7db2384387 --- /dev/null +++ b/scripts/ci_allocation/manifest.py @@ -0,0 +1,157 @@ +"""Create and validate the generated legacy-versus-balanced plan artifact.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import typing as t + +from .history import runtime_estimates +from .history import validate_runtime_model +from .planner import AllocationError +from .planner import build_suite_plan +from .planner import verify_assignments +from .planner import verify_runtime_assignments +from .suites import SuiteVenvInfo +from .suites import runtime_setup_seconds +from .suites import runtime_test_item_counts + + +def _json_value(value: t.Any) -> t.Any: + if isinstance(value, dict): + return {str(key): _json_value(item) for key, item in sorted(value.items())} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + if isinstance(value, (set, frozenset)): + return sorted(_json_value(item) for item in value) + if value is None or isinstance(value, (str, int, float, bool)): + return value + raise AllocationError(f"execution metadata contains an unsupported value: {type(value).__name__}") + + +def _digest(value: t.Mapping[str, t.Any]) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def build_allocation_manifest( + *, + suite_venv_info: t.Mapping[str, SuiteVenvInfo], + suite_configs: t.Mapping[str, t.Mapping[str, t.Any]], + legacy_shard_counts: t.Mapping[str, int], + balanced_shard_counts: t.Mapping[str, int], + runtime_model: t.Mapping[str, t.Any], + active_strategy: str, + target_shard_seconds: float = 300.0, + maximum_slices_per_hash: int = 1, +) -> dict[str, t.Any]: + validate_runtime_model(runtime_model) + _estimates, global_fallback = runtime_estimates(runtime_model) + suite_fallbacks = runtime_model["fallbacks"].get("suite_seconds", {}) + + plans = [] + if active_strategy not in {"legacy", "balanced"}: + raise AllocationError("allocation manifest active strategy is invalid") + if set(legacy_shard_counts) != set(balanced_shard_counts): + raise AllocationError("legacy and balanced plans must contain identical semantic suites") + + for suite in sorted(legacy_shard_counts): + info = suite_venv_info.get(suite) + if info is None: + raise AllocationError(f"selected suite has no Riot hashes: {suite}") + config = _json_value(suite_configs[suite]) + fallback = float(suite_fallbacks.get(suite, global_fallback)) + estimates, _unused_fallback = runtime_estimates(runtime_model, suite) + plans.append( + build_suite_plan( + suite=suite, + riot_hashes=info.hashes, + shard_count=legacy_shard_counts[suite], + balanced_shard_count=balanced_shard_counts[suite], + estimates=estimates, + fallback_seconds=fallback, + execution_metadata=config, + overhead_seconds=runtime_setup_seconds(runtime_model, suite), + target_shard_seconds=target_shard_seconds, + test_item_counts=runtime_test_item_counts(info, runtime_model), + maximum_slices_per_hash=maximum_slices_per_hash, + ) + ) + + manifest: dict[str, t.Any] = { + "schema_version": 1, + "planner_version": "runtime-sliced-lpt-v2", + "active_strategy": active_strategy, + "runtime_model_sha256": _digest(runtime_model), + "suites": plans, + } + manifest["manifest_sha256"] = _digest(manifest) + verify_allocation_manifest(manifest) + return manifest + + +def verify_allocation_manifest(manifest: t.Mapping[str, t.Any]) -> None: + if manifest.get("schema_version") != 1 or manifest.get("planner_version") != "runtime-sliced-lpt-v2": + raise AllocationError("unsupported allocation manifest schema or planner version") + if manifest.get("active_strategy") not in {"legacy", "balanced"}: + raise AllocationError("allocation manifest active strategy is invalid") + suites = manifest.get("suites") + if not isinstance(suites, list): + raise AllocationError("allocation manifest suites must be a list") + suite_names = [] + for raw_plan in suites: + if not isinstance(raw_plan, dict): + raise AllocationError("allocation manifest suite plan must be an object") + suite = raw_plan.get("suite") + hashes = raw_plan.get("riot_hashes") + if ( + not isinstance(suite, str) + or not isinstance(hashes, list) + or not all(isinstance(item, str) for item in hashes) + ): + raise AllocationError("allocation manifest suite identity is malformed") + suite_names.append(suite) + for strategy in ("legacy", "balanced"): + strategy_plan = raw_plan.get(strategy) + if not isinstance(strategy_plan, dict) or not isinstance(strategy_plan.get("assignments"), list): + raise AllocationError(f"allocation manifest {suite} {strategy} plan is malformed") + assignments = strategy_plan["assignments"] + if not all( + isinstance(shard, list) and all(isinstance(item, str) for item in shard) for shard in assignments + ): + raise AllocationError(f"allocation manifest {suite} {strategy} assignments are malformed") + if strategy == "legacy": + verify_assignments(hashes, assignments) + else: + verify_runtime_assignments(hashes, assignments) + parity = raw_plan.get("parity") + expected_parity = { + "exact_union", + "no_overlap", + "no_empty_shards", + "execution_metadata_equal", + "complete_runtime_slices", + } + if not isinstance(parity, dict) or set(parity) != expected_parity or not all(parity.values()): + raise AllocationError(f"allocation manifest parity proof failed for {suite}") + if len(suite_names) != len(set(suite_names)): + raise AllocationError("allocation manifest contains duplicate suites") + + expected_digest = manifest.get("manifest_sha256") + unsigned = dict(manifest) + unsigned.pop("manifest_sha256", None) + if expected_digest != _digest(unsigned): + raise AllocationError("allocation manifest digest does not match its content") + + +def load_manifest(path: Path) -> dict[str, t.Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise AllocationError("allocation manifest must contain a JSON object") + verify_allocation_manifest(value) + return value + + +def write_manifest(path: Path, manifest: t.Mapping[str, t.Any]) -> None: + path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/scripts/ci_allocation/planner.py b/scripts/ci_allocation/planner.py new file mode 100644 index 00000000000..e1219f4291d --- /dev/null +++ b/scripts/ci_allocation/planner.py @@ -0,0 +1,294 @@ +"""Build and verify deterministic shard assignments.""" + +from __future__ import annotations + +import hashlib +import heapq +import json +import math +import re +import typing as t + + +class AllocationError(RuntimeError): + pass + + +EXECUTION_UNIT = re.compile(r"^(?P[0-9a-f]+)(?:@(?P[1-9][0-9]*)/(?P[1-9][0-9]*))?$") + + +def _normalize_hashes(riot_hashes: t.Iterable[str]) -> list[str]: + hashes = sorted(riot_hashes) + if not hashes: + raise AllocationError("an allocation requires at least one Riot hash") + if len(hashes) != len(set(hashes)): + raise AllocationError("an allocation cannot contain duplicate Riot hashes") + return hashes + + +def execution_unit_id(riot_hash: str, shard_index: int = 1, shard_total: int = 1) -> str: + """Encode one whole or runtime-sliced Riot environment.""" + if not re.fullmatch(r"[0-9a-f]+", riot_hash): + raise AllocationError(f"invalid Riot hash: {riot_hash}") + if shard_total <= 0 or shard_index <= 0 or shard_index > shard_total: + raise AllocationError("runtime shard index must be within its positive shard total") + return riot_hash if shard_total == 1 else f"{riot_hash}@{shard_index}/{shard_total}" + + +def parse_execution_unit(value: str) -> tuple[str, int, int]: + """Decode an execution unit into Riot hash, runtime index, and runtime total.""" + match = EXECUTION_UNIT.fullmatch(value) + if match is None: + raise AllocationError(f"invalid Riot execution unit: {value}") + shard_index = int(match.group("index") or 1) + shard_total = int(match.group("total") or 1) + if shard_index > shard_total: + raise AllocationError(f"runtime shard index exceeds its total: {value}") + return match.group("riot_hash"), shard_index, shard_total + + +def expand_runtime_units( + riot_hashes: t.Iterable[str], + estimates: t.Mapping[str, float], + fallback_seconds: float, + *, + target_shard_seconds: float, + setup_seconds: float, + test_item_counts: t.Optional[t.Mapping[str, int]] = None, + maximum_slices_per_hash: int = 1, +) -> tuple[list[str], dict[str, float]]: + """Expand measured pytest environments into independently runnable test slices.""" + hashes = _normalize_hashes(riot_hashes) + if not math.isfinite(target_shard_seconds) or target_shard_seconds <= 0: + raise AllocationError("target_shard_seconds must be finite and positive") + if not math.isfinite(setup_seconds) or setup_seconds < 0: + raise AllocationError("setup_seconds must be finite and non-negative") + if maximum_slices_per_hash <= 0: + raise AllocationError("maximum_slices_per_hash must be positive") + if target_shard_seconds <= setup_seconds and maximum_slices_per_hash > 1: + raise AllocationError("target_shard_seconds must exceed setup_seconds for runtime slicing") + + test_item_counts = test_item_counts or {} + units: list[str] = [] + weights: dict[str, float] = {} + for riot_hash in hashes: + estimate = float(estimates.get(riot_hash, fallback_seconds)) + if not math.isfinite(estimate) or estimate <= 0: + raise AllocationError(f"invalid duration estimate for Riot hash {riot_hash}") + + # Only hashes backed by actual test-item observations may be split. A + # fallback duration can describe a one-test or non-pytest environment. + item_count = int(test_item_counts.get(riot_hash, 0)) + slice_count = 1 + if riot_hash in estimates and item_count > 1 and estimate > target_shard_seconds: + test_seconds = max(1.0, estimate - setup_seconds) + available_seconds = target_shard_seconds - setup_seconds + slice_count = min( + item_count, + maximum_slices_per_hash, + max(2, math.ceil(test_seconds / available_seconds)), + ) + + sliced_test_seconds = max(1.0, estimate - setup_seconds) / slice_count + for shard_index in range(1, slice_count + 1): + unit = execution_unit_id(riot_hash, shard_index, slice_count) + units.append(unit) + weights[unit] = estimate if slice_count == 1 else setup_seconds + sliced_test_seconds + return units, weights + + +def verify_runtime_assignments(expected_hashes: t.Iterable[str], assignments: list[list[str]]) -> None: + """Prove runtime slices cover each semantic Riot hash exactly once.""" + expected = _normalize_hashes(expected_hashes) + if not assignments: + raise AllocationError("an allocation must contain at least one shard") + if any(not shard for shard in assignments): + raise AllocationError("an allocation cannot contain an empty shard") + + flattened = [unit for shard in assignments for unit in shard] + if len(flattened) != len(set(flattened)): + raise AllocationError("a Riot execution unit is assigned to more than one shard") + + by_hash: dict[str, list[tuple[int, int]]] = {} + for unit in flattened: + riot_hash, shard_index, shard_total = parse_execution_unit(unit) + by_hash.setdefault(riot_hash, []).append((shard_index, shard_total)) + if sorted(by_hash) != expected: + missing = sorted(set(expected) - set(by_hash)) + unexpected = sorted(set(by_hash) - set(expected)) + raise AllocationError(f"allocation differs from the semantic suite: missing={missing}, unexpected={unexpected}") + for riot_hash, slices in by_hash.items(): + totals = {total for _index, total in slices} + if len(totals) != 1: + raise AllocationError(f"runtime shard totals differ for Riot hash {riot_hash}") + total = next(iter(totals)) + if {index for index, _total in slices} != set(range(1, total + 1)): + raise AllocationError(f"runtime shards are incomplete for Riot hash {riot_hash}") + + +def weighted_runtime_lpt(units: t.Iterable[str], shard_count: int, weights: t.Mapping[str, float]) -> list[list[str]]: + """Pack runtime units by weight while keeping sibling slices in separate jobs.""" + normalized = sorted(units) + if not normalized or len(normalized) != len(set(normalized)): + raise AllocationError("runtime execution units must be non-empty and unique") + if shard_count <= 0 or shard_count > len(normalized): + raise AllocationError("shard_count must be between one and the execution unit count") + + max_slices = max(parse_execution_unit(unit)[2] for unit in normalized) + if shard_count < max_slices: + raise AllocationError("runtime sibling slices require at least one distinct job each") + + shards: list[list[str]] = [[] for _ in range(shard_count)] + loads = [0.0] * shard_count + assigned_hashes: list[set[str]] = [set() for _ in range(shard_count)] + for unit in sorted(normalized, key=lambda item: (-float(weights[item]), item)): + riot_hash, _index, _total = parse_execution_unit(unit) + candidates = [index for index in range(shard_count) if riot_hash not in assigned_hashes[index]] + if not candidates: + raise AllocationError(f"cannot place sibling runtime slice for Riot hash {riot_hash}") + shard_index = min(candidates, key=lambda index: (loads[index], index)) + shards[shard_index].append(unit) + loads[shard_index] += float(weights[unit]) + assigned_hashes[shard_index].add(riot_hash) + return shards + + +def legacy_round_robin(riot_hashes: t.Iterable[str], shard_count: int) -> list[list[str]]: + """Reproduce the current ci-split-input.sh assignment exactly.""" + hashes = _normalize_hashes(riot_hashes) + if shard_count <= 0 or shard_count > len(hashes): + raise AllocationError("shard_count must be between one and the Riot hash count") + shards: list[list[str]] = [[] for _ in range(shard_count)] + for index, riot_hash in enumerate(hashes): + shards[index % shard_count].append(riot_hash) + return shards + + +def weighted_lpt( + riot_hashes: t.Iterable[str], + shard_count: int, + estimates: t.Mapping[str, float], + fallback_seconds: float, +) -> list[list[str]]: + """Assign atomic Riot hashes with deterministic longest-processing-time packing.""" + hashes = _normalize_hashes(riot_hashes) + if shard_count <= 0 or shard_count > len(hashes): + raise AllocationError("shard_count must be between one and the Riot hash count") + if not math.isfinite(fallback_seconds) or fallback_seconds <= 0: + raise AllocationError("fallback_seconds must be finite and positive") + + weights: dict[str, float] = {} + for riot_hash in hashes: + value = float(estimates.get(riot_hash, fallback_seconds)) + if not math.isfinite(value) or value <= 0: + raise AllocationError(f"invalid duration estimate for Riot hash {riot_hash}") + weights[riot_hash] = value + + shards: list[list[str]] = [[] for _ in range(shard_count)] + heap: list[tuple[float, int]] = [(0.0, index) for index in range(shard_count)] + heapq.heapify(heap) + for riot_hash in sorted(hashes, key=lambda item: (-weights[item], item)): + total, shard_index = heapq.heappop(heap) + shards[shard_index].append(riot_hash) + heapq.heappush(heap, (total + weights[riot_hash], shard_index)) + return shards + + +def verify_assignments(expected_hashes: t.Iterable[str], assignments: list[list[str]]) -> None: + expected = _normalize_hashes(expected_hashes) + if not assignments: + raise AllocationError("an allocation must contain at least one shard") + if any(not shard for shard in assignments): + raise AllocationError("an allocation cannot contain an empty shard") + + flattened = [riot_hash for shard in assignments for riot_hash in shard] + if len(flattened) != len(set(flattened)): + raise AllocationError("a Riot hash is assigned to more than one shard") + if sorted(flattened) != expected: + missing = sorted(set(expected) - set(flattened)) + unexpected = sorted(set(flattened) - set(expected)) + raise AllocationError(f"allocation differs from the semantic suite: missing={missing}, unexpected={unexpected}") + + +def predicted_makespan( + assignments: list[list[str]], estimates: t.Mapping[str, float], fallback_seconds: float +) -> float: + return max(sum(float(estimates.get(riot_hash, fallback_seconds)) for riot_hash in shard) for shard in assignments) + + +def metadata_digest(metadata: t.Mapping[str, t.Any]) -> str: + encoded = json.dumps(metadata, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + return hashlib.sha256(encoded).hexdigest() + + +def build_suite_plan( + *, + suite: str, + riot_hashes: t.Iterable[str], + shard_count: int, + balanced_shard_count: t.Optional[int] = None, + estimates: t.Mapping[str, float], + fallback_seconds: float, + execution_metadata: t.Mapping[str, t.Any], + overhead_seconds: float = 0.0, + target_shard_seconds: float = 300.0, + test_item_counts: t.Optional[t.Mapping[str, int]] = None, + maximum_slices_per_hash: int = 1, +) -> dict[str, t.Any]: + """Build legacy and balanced plans with exact-set parity evidence.""" + hashes = _normalize_hashes(riot_hashes) + legacy = legacy_round_robin(hashes, shard_count) + candidate_shard_count = balanced_shard_count if balanced_shard_count is not None else shard_count + runtime_units, runtime_weights = expand_runtime_units( + hashes, + estimates, + fallback_seconds, + target_shard_seconds=target_shard_seconds, + setup_seconds=overhead_seconds, + test_item_counts=test_item_counts, + maximum_slices_per_hash=maximum_slices_per_hash, + ) + balanced = weighted_runtime_lpt(runtime_units, candidate_shard_count, runtime_weights) + verify_assignments(hashes, legacy) + verify_runtime_assignments(hashes, balanced) + if not math.isfinite(overhead_seconds) or overhead_seconds < 0: + raise AllocationError("overhead_seconds must be finite and non-negative") + + # AIDEV-NOTE: Runtime history may change placement or refine a hash into + # slices, but this exact-set proof is the correctness authority. Never + # relax it to make a recommendation pass. + return { + "suite": suite, + "execution_metadata_sha256": metadata_digest(execution_metadata), + "riot_hashes": hashes, + "legacy": { + "algorithm": "sorted-round-robin-v1", + "shard_count": shard_count, + "assignments": legacy, + # Fitted Riot estimates already contain one activation/setup cost. + "predicted_makespan_seconds": predicted_makespan(legacy, estimates, fallback_seconds), + }, + "balanced": { + "algorithm": "runtime-sliced-lpt-v2", + "shard_count": candidate_shard_count, + "assignments": balanced, + "execution_unit_count": len(runtime_units), + "predicted_makespan_seconds": predicted_makespan(balanced, runtime_weights, fallback_seconds), + }, + "parity": { + "exact_union": True, + "no_overlap": True, + "no_empty_shards": True, + "execution_metadata_equal": True, + "complete_runtime_slices": True, + }, + } + + +def selected_shard(plan: dict[str, t.Any], strategy: str, node_index: int) -> list[str]: + if strategy not in {"legacy", "balanced"}: + raise AllocationError(f"unknown allocation strategy: {strategy}") + assignments = plan[strategy]["assignments"] + if node_index <= 0 or node_index > len(assignments): + raise AllocationError("node_index is outside the plan") + return list(assignments[node_index - 1]) diff --git a/scripts/ci_allocation/pr_history.py b/scripts/ci_allocation/pr_history.py new file mode 100644 index 00000000000..59acd82396d --- /dev/null +++ b/scripts/ci_allocation/pr_history.py @@ -0,0 +1,263 @@ +"""Replay current semantic suite selection across historical pull-request shapes.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import asdict +from dataclasses import dataclass +import fnmatch +import hashlib +import json +from pathlib import Path +import re +import subprocess +import typing as t + +from .history import percentile +from .history import runtime_estimates +from .history import suite_stage +from .planner import AllocationError +from .planner import expand_runtime_units +from .planner import legacy_round_robin +from .planner import predicted_makespan +from .planner import weighted_runtime_lpt +from .suites import SuiteVenvInfo +from .suites import compute_parallelism +from .suites import compute_runtime_parallelism +from .suites import runtime_setup_seconds +from .suites import runtime_test_item_counts + + +@dataclass(frozen=True) +class PRShape: + commit_sha: str + timestamp: str + subject: str + changed_files: tuple[str, ...] + selected_suites: tuple[str, ...] + cohort: str + + +def classify_cohort(files: t.Iterable[str]) -> str: + paths = tuple(files) + if paths and all(path.startswith(("docs/", "releasenotes/")) or path.endswith((".md", ".rst")) for path in paths): + return "docs" + if any(path.startswith((".gitlab/", ".github/", "scripts/")) for path in paths): + return "ci" + if any(path.startswith(("ddtrace/appsec/", "tests/appsec/")) for path in paths): + return "appsec" + if any(path.startswith(("ddtrace/contrib/", "tests/contrib/")) for path in paths): + return "integration" + return "core" + + +def select_suites(changed_files: t.Iterable[str], suite_patterns: t.Mapping[str, t.Iterable[str]]) -> tuple[str, ...]: + files = tuple(changed_files) + selected = [] + for suite in sorted(suite_patterns): + patterns = tuple(suite_patterns[suite]) + if not patterns or any(fnmatch.filter(files, pattern) for pattern in patterns): + selected.append(suite) + return tuple(selected) + + +def collect_pr_shapes( + *, + root: Path, + suite_patterns: t.Mapping[str, t.Iterable[str]], + since: str, + max_count: t.Optional[int] = None, +) -> list[PRShape]: + """Collect first-parent PR-shaped commits and their changed paths in one Git call.""" + command = [ + "git", + "log", + "--first-parent", + f"--since={since}", + "--format=%x1e%H%x1f%cI%x1f%s", + "--name-only", + ] + if max_count is not None: + command.insert(3, f"--max-count={max_count}") + result = subprocess.run(command, cwd=root, check=True, capture_output=True, text=True) + shapes = [] + for record in result.stdout.split("\x1e"): + record = record.strip() + if not record: + continue + header, *path_lines = record.splitlines() + fields = header.split("\x1f", 2) + if len(fields) != 3: + raise AllocationError("git history record is malformed") + commit_sha, timestamp, subject = fields + if not re.search(r"\(#\d+\)$", subject): + continue + changed_files = tuple(sorted({line.strip() for line in path_lines if line.strip()})) + shapes.append( + PRShape( + commit_sha=commit_sha, + timestamp=timestamp, + subject=subject, + changed_files=changed_files, + selected_suites=select_suites(changed_files, suite_patterns), + cohort=classify_cohort(changed_files), + ) + ) + return shapes + + +def write_pr_shapes(path: Path, shapes: t.Iterable[PRShape]) -> None: + lines = [json.dumps({"schema_version": 1, **asdict(item)}, sort_keys=True) for item in shapes] + path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") + + +def load_pr_shapes(path: Path) -> list[PRShape]: + shapes = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + value = json.loads(line) + if not isinstance(value, dict) or value.pop("schema_version", None) != 1: + raise AllocationError("PR history contains an unsupported record") + value["changed_files"] = tuple(value["changed_files"]) + value["selected_suites"] = tuple(value["selected_suites"]) + shapes.append(PRShape(**value)) + return shapes + + +def _fingerprint(shapes: t.Iterable[PRShape]) -> str: + value = [asdict(item) for item in shapes] + value.sort(key=lambda item: item["commit_sha"]) + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def replay_pr_shapes( + *, + shapes: list[PRShape], + suite_configs: t.Mapping[str, t.Mapping[str, t.Any]], + suite_venv_info: dict[str, SuiteVenvInfo], + runtime_model: t.Mapping[str, t.Any], + target_jobs: int, + target_shard_seconds: float, + maximum_parallelism_per_suite: int, + maximum_slices_per_hash: int = 1, +) -> dict[str, t.Any]: + """Estimate both planners over historical PR path-selection cohorts.""" + if not shapes: + raise AllocationError("PR replay requires at least one historical shape") + estimates, global_fallback = runtime_estimates(runtime_model) + suite_fallbacks = runtime_model["fallbacks"].get("suite_seconds", {}) + estimates_by_suite = {suite: runtime_estimates(runtime_model, suite)[0] for suite in suite_venv_info} + test_item_counts = {suite: runtime_test_item_counts(info, runtime_model) for suite, info in suite_venv_info.items()} + setup_seconds = {suite: runtime_setup_seconds(runtime_model, suite) for suite in suite_venv_info} + results: list[tuple[str, float, float, float, float]] = [] + unmodeled_shapes = 0 + for shape in shapes: + selected = [suite for suite in shape.selected_suites if suite in suite_venv_info and suite in suite_configs] + legacy_shard_counts = compute_parallelism( + suite_configs, + selected, + suite_venv_info, + target_jobs=target_jobs, + ) + if not legacy_shard_counts: + unmodeled_shapes += 1 + continue + balanced_shard_counts = ( + compute_runtime_parallelism( + suite_venv_info, + selected, + estimates, + suite_fallbacks, + global_fallback, + target_shard_seconds=target_shard_seconds, + maximum_parallelism_per_suite=maximum_parallelism_per_suite, + maximum_total_jobs=sum(legacy_shard_counts.values()), + suite_overheads=setup_seconds, + global_overhead=float(runtime_model["overheads"].get("unit_global_seconds", 0.0)), + test_item_counts_by_suite=test_item_counts, + estimates_by_suite=estimates_by_suite, + maximum_slices_per_hash=maximum_slices_per_hash, + ) + if estimates + else legacy_shard_counts + ) + legacy_stages: dict[str, float] = defaultdict(float) + balanced_stages: dict[str, float] = defaultdict(float) + legacy_runner_seconds = 0.0 + balanced_runner_seconds = 0.0 + for suite, legacy_shard_count in legacy_shard_counts.items(): + hashes = suite_venv_info[suite].hashes + fallback = float(suite_fallbacks.get(suite, global_fallback)) + suite_estimates = estimates_by_suite[suite] + overhead = setup_seconds[suite] + legacy = legacy_round_robin(hashes, legacy_shard_count) + units, weights = expand_runtime_units( + hashes, + suite_estimates, + fallback, + target_shard_seconds=target_shard_seconds, + setup_seconds=overhead, + test_item_counts=test_item_counts[suite], + maximum_slices_per_hash=maximum_slices_per_hash, + ) + balanced = weighted_runtime_lpt(units, balanced_shard_counts[suite], weights) + stage = suite_stage(suite) + legacy_stages[stage] = max(legacy_stages[stage], predicted_makespan(legacy, suite_estimates, fallback)) + balanced_stages[stage] = max(balanced_stages[stage], predicted_makespan(balanced, weights, fallback)) + legacy_runner_seconds += sum(float(suite_estimates.get(riot_hash, fallback)) for riot_hash in hashes) + balanced_runner_seconds += sum(weights.values()) + results.append( + ( + shape.cohort, + max(legacy_stages.values()), + max(balanced_stages.values()), + legacy_runner_seconds, + balanced_runner_seconds, + ) + ) + + if not results: + raise AllocationError("PR replay contains no modeled semantic suites") + + def summarize(items: list[tuple[str, float, float, float, float]]) -> dict[str, t.Any]: + legacy = [item[1] for item in items] + balanced = [item[2] for item in items] + return { + "count": len(items), + "legacy": { + "median_seconds": round(percentile(legacy, 0.5), 6), + "p75_seconds": round(percentile(legacy, 0.75), 6), + "p90_seconds": round(percentile(legacy, 0.9), 6), + }, + "balanced": { + "median_seconds": round(percentile(balanced, 0.5), 6), + "p75_seconds": round(percentile(balanced, 0.75), 6), + "p90_seconds": round(percentile(balanced, 0.9), 6), + }, + "legacy_runner_seconds": round(sum(item[3] for item in items), 6), + "balanced_runner_seconds": round(sum(item[4] for item in items), 6), + } + + overall = summarize(results) + overall["median_improvement_ratio"] = round( + 1 - overall["balanced"]["median_seconds"] / overall["legacy"]["median_seconds"], 6 + ) + overall["runner_seconds_change_ratio"] = round( + overall["balanced_runner_seconds"] / overall["legacy_runner_seconds"] - 1, 6 + ) + by_cohort: dict[str, list[tuple[str, float, float, float, float]]] = defaultdict(list) + for item in results: + by_cohort[item[0]].append(item) + return { + "schema_version": 1, + "kind": "historical-pr-shape-replay", + "dataset_fingerprint_sha256": _fingerprint(shapes), + "input_shape_count": len(shapes), + "modeled_shape_count": len(results), + "unmodeled_shape_count": unmodeled_shapes, + "target_jobs": target_jobs, + "target_shard_seconds": target_shard_seconds, + "overall": overall, + "cohorts": {cohort: summarize(items) for cohort, items in sorted(by_cohort.items())}, + } diff --git a/scripts/ci_allocation/runtime.py b/scripts/ci_allocation/runtime.py new file mode 100644 index 00000000000..648aedc4195 --- /dev/null +++ b/scripts/ci_allocation/runtime.py @@ -0,0 +1,165 @@ +"""Partition collected pytest items and verify runtime-shard inventory artifacts.""" + +from __future__ import annotations + +from collections import Counter +from collections import defaultdict +import hashlib +import json +from pathlib import Path +import typing as t + +from .manifest import load_manifest +from .planner import AllocationError +from .planner import parse_execution_unit + + +SELECTION_ALGORITHM = "sha256-round-robin-v1" + + +def _digest(values: t.Iterable[str]) -> str: + encoded = json.dumps(sorted(values), separators=(",", ":"), ensure_ascii=True).encode() + return hashlib.sha256(encoded).hexdigest() + + +def partition_nodeids(nodeids: t.Iterable[str], shard_total: int) -> dict[str, int]: + """Assign runtime-discovered node IDs evenly and deterministically.""" + values = list(nodeids) + if not values or len(values) != len(set(values)): + raise AllocationError("runtime test inventory must be non-empty and unique") + if shard_total <= 0 or shard_total > len(values): + raise AllocationError("runtime test shard total must be within the collected item count") + + # Hash ordering spreads parametrized families and slow files that are adjacent + # in pytest collection order. Round-robin then guarantees count skew <= 1. + ordered = sorted(values, key=lambda value: (hashlib.sha256(value.encode()).digest(), value)) + return {nodeid: index % shard_total + 1 for index, nodeid in enumerate(ordered)} + + +def build_runtime_inventory( + *, + suite: str, + riot_hash: str, + shard_index: int, + shard_total: int, + collected_nodeids: t.Iterable[str], +) -> dict[str, t.Any]: + """Create compact exact-set evidence for one runtime pytest slice.""" + collected = sorted(collected_nodeids) + assignments = partition_nodeids(collected, shard_total) + selected = sorted(nodeid for nodeid in collected if assignments[nodeid] == shard_index) + if not selected: + raise AllocationError(f"runtime test shard {shard_index}/{shard_total} is empty") + return { + "schema_version": 1, + "kind": "ci-runtime-test-shard-inventory", + "selection_algorithm": SELECTION_ALGORITHM, + "suite": suite, + "riot_hash": riot_hash, + "shard_index": shard_index, + "shard_total": shard_total, + "collection_count": len(collected), + "collection_sha256": _digest(collected), + "selected_count": len(selected), + "selected_sha256": _digest(selected), + "selected_nodeids": selected, + # One full inventory per hash is enough to prove the selected-slice + # union without repeating thousands of node IDs in every artifact. + "collected_nodeids": collected if shard_index == 1 else None, + } + + +def write_runtime_inventory(path: Path, inventory: t.Mapping[str, t.Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(inventory, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8") + + +def _load_inventory(path: Path) -> dict[str, t.Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise AllocationError(f"cannot read runtime test inventory {path}: {exc}") from exc + if not isinstance(value, dict) or value.get("kind") != "ci-runtime-test-shard-inventory": + raise AllocationError(f"runtime test inventory is malformed: {path}") + return value + + +def verify_runtime_inventories( + paths: t.Iterable[Path], plan_path: Path, strategy: str = "balanced" +) -> dict[str, t.Any]: + """Prove every planned sub-hash slice collected an exact disjoint test set.""" + if strategy != "balanced": + raise AllocationError("runtime test inventory verification requires the balanced strategy") + plan = load_manifest(plan_path) + expected: set[tuple[str, str, int, int]] = set() + for suite_plan in plan["suites"]: + suite = suite_plan["suite"] + for assignment in suite_plan[strategy]["assignments"]: + for unit in assignment: + riot_hash, shard_index, shard_total = parse_execution_unit(unit) + if shard_total > 1: + expected.add((suite, riot_hash, shard_index, shard_total)) + + inventories = [_load_inventory(path) for path in paths] + observed: dict[tuple[str, str, int, int], dict[str, t.Any]] = {} + for inventory in inventories: + key = ( + str(inventory.get("suite", "")), + str(inventory.get("riot_hash", "")), + int(inventory.get("shard_index", 0)), + int(inventory.get("shard_total", 0)), + ) + if key in observed: + raise AllocationError(f"duplicate runtime test inventory: {key}") + observed[key] = inventory + if set(observed) != expected: + missing = sorted(expected - set(observed))[:10] + unexpected = sorted(set(observed) - expected)[:10] + raise AllocationError( + f"runtime test inventories differ from the plan: missing={missing}, unexpected={unexpected}" + ) + + grouped: dict[tuple[str, str, int], list[dict[str, t.Any]]] = defaultdict(list) + for (suite, riot_hash, _index, total), inventory in observed.items(): + grouped[(suite, riot_hash, total)].append(inventory) + + selected_total = 0 + collection_total = 0 + for (suite, riot_hash, total), group in grouped.items(): + if {int(item["shard_index"]) for item in group} != set(range(1, total + 1)): + raise AllocationError(f"runtime test shard indices are incomplete for {suite} {riot_hash}") + collection_counts = {int(item["collection_count"]) for item in group} + collection_digests = {str(item["collection_sha256"]) for item in group} + algorithms = {str(item["selection_algorithm"]) for item in group} + if len(collection_counts) != 1 or len(collection_digests) != 1 or algorithms != {SELECTION_ALGORITHM}: + raise AllocationError(f"runtime test collection differs across slices for {suite} {riot_hash}") + if any(int(item["selected_count"]) <= 0 for item in group): + raise AllocationError(f"runtime test shard is empty for {suite} {riot_hash}") + + full_values = [item.get("collected_nodeids") for item in group if item.get("collected_nodeids") is not None] + if len(full_values) != 1 or not isinstance(full_values[0], list): + raise AllocationError(f"runtime test collection inventory is missing for {suite} {riot_hash}") + collected = [str(value) for value in full_values[0]] + selected = [str(value) for item in group for value in item.get("selected_nodeids", [])] + if Counter(selected) != Counter(collected): + raise AllocationError(f"runtime test shard union differs from collection for {suite} {riot_hash}") + if _digest(collected) != next(iter(collection_digests)): + raise AllocationError(f"runtime test collection digest differs for {suite} {riot_hash}") + for item in group: + values = [str(value) for value in item.get("selected_nodeids", [])] + if len(values) != int(item["selected_count"]) or _digest(values) != item["selected_sha256"]: + raise AllocationError(f"runtime test shard digest differs for {suite} {riot_hash}") + selected_total += len(selected) + collection_total += len(collected) + + return { + "schema_version": 1, + "kind": "ci-runtime-test-shard-parity", + "split_hash_count": len(grouped), + "runtime_slice_count": len(observed), + "test_identity_count": selected_total, + "collection_identity_count": collection_total, + "exact_union": True, + "no_overlap": True, + "no_empty_slices": True, + } diff --git a/scripts/ci_allocation/suites.py b/scripts/ci_allocation/suites.py new file mode 100644 index 00000000000..52814b45351 --- /dev/null +++ b/scripts/ci_allocation/suites.py @@ -0,0 +1,306 @@ +"""Resolve semantic suites into their Riot execution units.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from dataclasses import field +import hashlib +import heapq +import math +import re +import typing as t + +from .planner import expand_runtime_units +from .planner import parse_execution_unit +from .planner import predicted_makespan +from .planner import weighted_runtime_lpt + + +@dataclass(frozen=True) +class SuiteVenvInfo: + """The Riot hashes and Python versions selected by one semantic suite.""" + + hashes: tuple[str, ...] + python_versions: frozenset[str] + commands: t.Mapping[str, tuple[str, ...]] = field(default_factory=dict) + python_version_by_hash: t.Mapping[str, str] = field(default_factory=dict) + + @property + def venv_count(self) -> int: + return len(self.hashes) + + +def collect_all_suite_venv_info(suite_patterns: dict[str, str]) -> dict[str, SuiteVenvInfo]: + """Collect Riot hashes and Python versions for many suites in one pass.""" + import riotfile + + compiled: dict[str, re.Pattern[str]] = {} + for suite, pattern in suite_patterns.items(): + try: + compiled[suite] = re.compile(pattern) + except re.error: + continue + + venv_hashes: dict[str, set[str]] = {suite: set() for suite in compiled} + python_versions: dict[str, set[str]] = {suite: set() for suite in compiled} + commands: dict[str, dict[str, set[str]]] = {suite: defaultdict(set) for suite in compiled} + versions_by_hash: dict[str, dict[str, str]] = {suite: {} for suite in compiled} + + for instance in riotfile.venv.instances(): # type: ignore[attr-defined] + if not instance.name: + continue + hint = instance.py._hint + for suite, regex in compiled.items(): + if instance.matches_pattern(regex): + venv_hashes[suite].add(instance.short_hash) + commands[suite][instance.short_hash].add(str(instance.command or "")) + if re.match(r"^3\.\d+$", hint): + python_versions[suite].add(hint) + versions_by_hash[suite][instance.short_hash] = hint + + return { + suite: SuiteVenvInfo( + hashes=tuple(sorted(hashes)), + python_versions=frozenset(python_versions[suite]), + commands={riot_hash: tuple(sorted(values)) for riot_hash, values in sorted(commands[suite].items())}, + python_version_by_hash=dict(sorted(versions_by_hash[suite].items())), + ) + for suite, hashes in venv_hashes.items() + if hashes + } + + +def runtime_test_item_counts(info: SuiteVenvInfo, runtime_model: t.Mapping[str, t.Any]) -> dict[str, int]: + """Resolve command-level Test Visibility evidence to this suite's Riot hashes.""" + test_sharding = runtime_model.get("test_sharding", {}) + command_evidence = test_sharding.get("command_fingerprints", {}) if isinstance(test_sharding, dict) else {} + if not isinstance(command_evidence, dict): + return {} + + result = {} + for riot_hash, commands in info.commands.items(): + commands = (commands,) if isinstance(commands, str) else commands + counts = [] + for command in commands: + fingerprint = hashlib.sha256(command.encode()).hexdigest() + evidence = command_evidence.get(fingerprint) + item_count = evidence.get("minimum_items") if isinstance(evidence, dict) else None + if not isinstance(item_count, int) or isinstance(item_count, bool) or item_count <= 0: + break + counts.append(item_count) + else: + if counts: + result[riot_hash] = min(counts) + return result + + +def runtime_setup_seconds(runtime_model: t.Mapping[str, t.Any], suite: str) -> float: + """Return measured setup/activation time for one new runtime execution unit.""" + overheads = runtime_model.get("overheads", {}) + if not isinstance(overheads, dict): + return 0.0 + suite_values = overheads.get("unit_suite_seconds", {}) + if isinstance(suite_values, dict) and suite in suite_values: + return float(suite_values[suite]) + stage_values = overheads.get("unit_stage_seconds", {}) + stage = suite.split("::", 1)[0] if "::" in suite else "core" + if isinstance(stage_values, dict) and stage in stage_values: + return float(stage_values[stage]) + return float(overheads.get("unit_global_seconds", 0.0)) + + +def calculate_parallelism_from_venvs(venv_count: int, venvs_per_job: int, max_parallelism: int = 25) -> int: + """Calculate the baseline shard count from a suite's Riot environment count.""" + if venv_count <= 0: + raise ValueError("venv_count must be positive") + if venvs_per_job <= 0: + raise ValueError("venvs_per_job must be positive") + if max_parallelism <= 0: + raise ValueError("max_parallelism must be positive") + return min(math.ceil(venv_count / venvs_per_job), max_parallelism) + + +def scale_suites( + suite_venv_info: dict[str, SuiteVenvInfo], + final_jobs: dict[str, int], + scalable_suites: list[str], + venvs_per_job_map: dict[str, int], + target: int, +) -> dict[str, int]: + """Scale selected suites toward a total job target without splitting Riot hashes.""" + if target <= 0: + raise ValueError("target must be positive") + + final_jobs = dict(final_jobs) + current_vpj = dict(venvs_per_job_map) + + while sum(final_jobs.values()) < target: + best_gain = 0 + best_suite: t.Optional[str] = None + + for suite in scalable_suites: + venv_count = suite_venv_info[suite].venv_count + current = final_jobs[suite] + if current >= venv_count: + continue + + if suite in current_vpj: + vpj = current_vpj[suite] + if vpj <= 1: + continue + new_parallelism = math.ceil(venv_count / (vpj - 1)) + else: + new_parallelism = current + 1 + + gain = new_parallelism - current + if gain > best_gain: + best_gain = gain + best_suite = suite + + if best_suite is None or best_gain == 0: + break + + venv_count = suite_venv_info[best_suite].venv_count + if best_suite in current_vpj: + current_vpj[best_suite] -= 1 + final_jobs[best_suite] = math.ceil(venv_count / current_vpj[best_suite]) + else: + final_jobs[best_suite] += 1 + + return final_jobs + + +def compute_parallelism( + suite_configs: t.Mapping[str, t.Mapping[str, t.Any]], + selected_suites: t.Iterable[str], + suite_venv_info: dict[str, SuiteVenvInfo], + *, + target_jobs: int, +) -> dict[str, int]: + """Apply the current baseline and sparse-run scaling policy.""" + selected = [ + suite + for suite in selected_suites + if suite in suite_configs and not suite_configs[suite].get("skip", False) and suite in suite_venv_info + ] + baseline_jobs: dict[str, int] = {} + scalable_suites: list[str] = [] + venvs_per_job_map: dict[str, int] = {} + for suite in selected: + config = suite_configs[suite] + static_parallelism = config.get("parallelism") + venvs_per_job = config.get("venvs_per_job") + if static_parallelism is not None: + baseline_jobs[suite] = int(static_parallelism) + scalable_suites.append(suite) + elif venvs_per_job is not None: + baseline_jobs[suite] = calculate_parallelism_from_venvs( + suite_venv_info[suite].venv_count, int(venvs_per_job) + ) + scalable_suites.append(suite) + venvs_per_job_map[suite] = int(venvs_per_job) + else: + baseline_jobs[suite] = 1 + + if sum(baseline_jobs.values()) < target_jobs and scalable_suites: + return scale_suites( + suite_venv_info, + baseline_jobs, + scalable_suites, + venvs_per_job_map, + target_jobs, + ) + return baseline_jobs + + +def compute_runtime_parallelism( + suite_venv_info: t.Mapping[str, SuiteVenvInfo], + selected_suites: t.Iterable[str], + estimates: t.Mapping[str, float], + suite_fallbacks: t.Mapping[str, float], + global_fallback: float, + *, + target_shard_seconds: float, + maximum_parallelism_per_suite: int, + maximum_total_jobs: t.Optional[int] = None, + suite_overheads: t.Optional[t.Mapping[str, float]] = None, + global_overhead: float = 0.0, + test_item_counts_by_suite: t.Optional[t.Mapping[str, t.Mapping[str, int]]] = None, + estimates_by_suite: t.Optional[t.Mapping[str, t.Mapping[str, float]]] = None, + maximum_slices_per_hash: int = 1, +) -> dict[str, int]: + """Size suites from modeled work within an optional global job budget.""" + if target_shard_seconds <= 0: + raise ValueError("target_shard_seconds must be positive") + if maximum_parallelism_per_suite <= 0: + raise ValueError("maximum_parallelism_per_suite must be positive") + + result = {} + suite_overheads = suite_overheads or {} + test_item_counts_by_suite = test_item_counts_by_suite or {} + estimates_by_suite = estimates_by_suite or {} + runtime_units: dict[str, tuple[list[str], dict[str, float]]] = {} + minimum_jobs: dict[str, int] = {} + for suite in selected_suites: + info = suite_venv_info.get(suite) + if info is None: + continue + fallback = float(suite_fallbacks.get(suite, global_fallback)) + suite_estimates = estimates_by_suite.get(suite, estimates) + overhead = float(suite_overheads.get(suite, global_overhead)) + if overhead < 0: + raise ValueError("suite overhead cannot be negative") + units, weights = expand_runtime_units( + info.hashes, + suite_estimates, + fallback, + target_shard_seconds=target_shard_seconds, + setup_seconds=overhead, + test_item_counts=test_item_counts_by_suite.get(suite, {}), + maximum_slices_per_hash=maximum_slices_per_hash, + ) + runtime_units[suite] = (units, weights) + minimum_jobs[suite] = max(parse_execution_unit(unit)[2] for unit in units) + modeled_work = sum(weights.values()) + result[suite] = min( + len(units), + maximum_parallelism_per_suite, + max(minimum_jobs[suite], math.ceil(modeled_work / target_shard_seconds)), + ) + if result[suite] < minimum_jobs[suite]: + raise ValueError(f"maximum_parallelism_per_suite cannot place runtime slices for {suite}") + + if maximum_total_jobs is None or sum(result.values()) <= maximum_total_jobs: + return result + if maximum_total_jobs < sum(minimum_jobs.values()): + raise ValueError("maximum_total_jobs cannot place every runtime slice") + + def removal_penalty(suite: str, shard_count: int) -> float: + units, weights = runtime_units[suite] + current = weighted_runtime_lpt(units, shard_count, weights) + reduced = weighted_runtime_lpt(units, shard_count - 1, weights) + return predicted_makespan(reduced, weights, global_fallback) - predicted_makespan( + current, weights, global_fallback + ) + + # AIDEV-NOTE: The legacy total is a cost ceiling, not a per-suite ceiling. + # Remove the shard with the smallest modeled critical-path penalty so capacity + # can move from over-sharded suites to measured long poles deterministically. + candidates = [ + (removal_penalty(suite, count), suite, count) for suite, count in result.items() if count > minimum_jobs[suite] + ] + heapq.heapify(candidates) + while sum(result.values()) > maximum_total_jobs: + if not candidates: + raise ValueError("maximum_total_jobs cannot be satisfied") + _penalty, suite, expected_count = heapq.heappop(candidates) + if result[suite] != expected_count: + continue + result[suite] -= 1 + if result[suite] > minimum_jobs[suite]: + heapq.heappush( + candidates, + (removal_penalty(suite, result[suite]), suite, result[suite]), + ) + return result diff --git a/scripts/ci_allocation_cli.py b/scripts/ci_allocation_cli.py new file mode 100755 index 00000000000..2252c99116d --- /dev/null +++ b/scripts/ci_allocation_cli.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "riot>=0.22.0", +# "ruamel.yaml>=0.17.21", +# ] +# /// +"""Build, verify, replay, and select dd-trace-py CI shard allocations.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import typing as t + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +for path in (ROOT, SCRIPTS, ROOT / "tests"): + if str(path) not in sys.path: + sys.path.append(str(path)) + +from ci_allocation.history import build_runtime_model # noqa: E402 +from ci_allocation.history import live_shadow_report # noqa: E402 +from ci_allocation.history import load_json # noqa: E402 +from ci_allocation.history import load_observations # noqa: E402 +from ci_allocation.history import ratchet_violations # noqa: E402 +from ci_allocation.history import replay_observations # noqa: E402 +from ci_allocation.history import runtime_estimates # noqa: E402 +from ci_allocation.history import validate_runtime_model # noqa: E402 +from ci_allocation.history import write_json # noqa: E402 +from ci_allocation.history import write_observations # noqa: E402 +from ci_allocation.jobs import load_job_observations # noqa: E402 +from ci_allocation.jobs import write_job_observations # noqa: E402 +from ci_allocation.junit import verify_junit_parity # noqa: E402 +from ci_allocation.manifest import load_manifest # noqa: E402 +from ci_allocation.planner import AllocationError # noqa: E402 +from ci_allocation.planner import build_suite_plan # noqa: E402 +from ci_allocation.planner import selected_shard # noqa: E402 +from ci_allocation.pr_history import collect_pr_shapes # noqa: E402 +from ci_allocation.pr_history import load_pr_shapes # noqa: E402 +from ci_allocation.pr_history import replay_pr_shapes # noqa: E402 +from ci_allocation.pr_history import write_pr_shapes # noqa: E402 +from ci_allocation.runtime import verify_runtime_inventories # noqa: E402 +from ci_allocation.suites import collect_all_suite_venv_info # noqa: E402 + + +DEFAULT_POLICY = ROOT / "ci" / "ci-allocation-policy.json" +DEFAULT_MODEL = ROOT / "ci" / "ci-allocation-runtime-model.json" +DEFAULT_PLAN = ROOT / ".gitlab" / "ci-allocation-plan.json" +UV_REQUIRED_COMMANDS = {"export-pr-history", "replay-pr-history"} + + +def _suite_catalog() -> tuple[dict[str, dict[str, t.Any]], dict[str, t.Any]]: + from tests.suitespec import get_suites + + suites = {name: dict(config) for name, config in get_suites().items() if config.get("type", "test") == "test"} + patterns = { + name: str(config.get("pattern", name)) for name, config in suites.items() if not config.get("skip", False) + } + return suites, collect_all_suite_venv_info(patterns) + + +def _policy(path: Path) -> dict[str, t.Any]: + policy = load_json(path) + if policy.get("schema_version") != 1: + raise AllocationError("unsupported CI allocation policy schema") + for section in ("allocation", "model", "ratchets"): + if not isinstance(policy.get(section), dict): + raise AllocationError(f"CI allocation policy is missing {section}") + for ratchet in ("historical_replay", "pr_shape_replay", "live_shadow"): + if not isinstance(policy["ratchets"].get(ratchet), dict): + raise AllocationError(f"CI allocation policy is missing {ratchet}") + return policy + + +def command_select(args: argparse.Namespace) -> None: + hashes = [line.strip() for line in sys.stdin if line.strip()] + strategy = args.strategy or _policy(args.policy)["allocation"]["active_strategy"] + if args.plan.exists(): + manifest = load_manifest(args.plan) + suite_plans = [item for item in manifest["suites"] if item["suite"] == args.suite] + if len(suite_plans) == 1: + suite_plan = suite_plans[0] + if sorted(hashes) != suite_plan["riot_hashes"]: + raise AllocationError(f"runtime Riot hashes differ from the generated plan for {args.suite}") + if int(suite_plan[strategy]["shard_count"]) != args.node_total: + raise AllocationError(f"runtime node total differs from the generated plan for {args.suite}") + for unit in selected_shard(suite_plan, strategy, args.node_index): + print(unit) + return + + model = load_json(args.model) + estimates, fallback = runtime_estimates(model) + plan = build_suite_plan( + suite=args.suite, + riot_hashes=hashes, + shard_count=args.node_total, + estimates=estimates, + fallback_seconds=fallback, + execution_metadata={"suite": args.suite}, + ) + for riot_hash in selected_shard(plan, strategy, args.node_index): + print(riot_hash) + + +def command_ingest(args: argparse.Namespace) -> None: + observations = load_observations(args.input) + write_observations(args.output, observations) + print(f"wrote {len(observations)} observations to {args.output}") + + +def command_ingest_jobs(args: argparse.Namespace) -> None: + observations = load_job_observations(args.input) + write_job_observations(args.output, observations) + print(f"wrote {len(observations)} CI job observations to {args.output}") + + +def command_build_model(args: argparse.Namespace) -> None: + policy = _policy(args.policy) + observations = load_observations(args.observations) + jobs = load_job_observations(args.jobs) + model = build_runtime_model(observations, policy["model"], jobs) + write_json(args.output, model) + print( + f"wrote {len(model['estimates'])} Riot hash estimates from " + f"{model['dataset']['training_observations']} observations to {args.output}" + ) + if args.report is not None: + report = replay_observations( + observations, + model, + target_shard_seconds=float(policy["allocation"]["target_shard_seconds"]), + maximum_parallelism_per_suite=int(policy["allocation"]["maximum_parallelism_per_suite"]), + ) + write_json(args.report, report) + print(f"wrote holdout replay to {args.report}") + + +def command_replay_observations(args: argparse.Namespace) -> None: + policy = _policy(args.policy) + report = replay_observations( + load_observations(args.observations), + load_json(args.model), + target_shard_seconds=float(policy["allocation"]["target_shard_seconds"]), + maximum_parallelism_per_suite=int(policy["allocation"]["maximum_parallelism_per_suite"]), + ) + write_json(args.output, report) + print(f"wrote historical replay to {args.output}") + + +def command_live_report(args: argparse.Namespace) -> None: + report = live_shadow_report(load_observations(args.observations), load_job_observations(args.jobs)) + write_json(args.output, report) + print(f"wrote live shadow replay to {args.output}") + + +def command_check_ratchet(args: argparse.Namespace) -> None: + policy = _policy(args.policy) + report = load_json(args.report) + violations = ratchet_violations(report, policy["ratchets"]) + if violations: + raise AllocationError("; ".join(violations)) + print(f"CI allocation ratchet passed for {report['kind']}") + + +def command_verify_plan(args: argparse.Namespace) -> None: + manifest = load_manifest(args.plan) + print(f"CI allocation plan verified: {len(manifest['suites'])} semantic suites") + + +def command_verify_junit(args: argparse.Namespace) -> None: + report = verify_junit_parity(args.legacy, args.balanced) + if args.output is not None: + write_json(args.output, report) + print( + f"JUnit allocation parity verified: {report['test_identity_count']} tests, " + f"{report['riot_hash_count']} Riot hashes" + ) + + +def command_verify_runtime_shards(args: argparse.Namespace) -> None: + report = verify_runtime_inventories(args.manifests, args.plan) + if args.output is not None: + write_json(args.output, report) + print( + f"Runtime test shard parity verified: {report['test_identity_count']} tests, " + f"{report['split_hash_count']} split Riot hashes" + ) + + +def command_check_contract(args: argparse.Namespace) -> None: + policy = _policy(args.policy) + model = load_json(args.model) + validate_runtime_model(model) + if model["parameters"] != policy["model"]: + raise AllocationError("runtime model parameters do not match the allocation policy") + if int(policy["allocation"].get("target_jobs", 0)) <= 0: + raise AllocationError("allocation target_jobs must be positive") + if float(policy["allocation"].get("target_shard_seconds", 0)) <= 0: + raise AllocationError("allocation target_shard_seconds must be positive") + if int(policy["allocation"].get("maximum_parallelism_per_suite", 0)) <= 0: + raise AllocationError("allocation maximum_parallelism_per_suite must be positive") + if int(policy["allocation"].get("maximum_slices_per_hash", 0)) <= 1: + raise AllocationError("allocation maximum_slices_per_hash must be greater than one") + maximum_model_bytes = int(policy["allocation"].get("maximum_runtime_model_bytes", 0)) + if maximum_model_bytes <= 0: + raise AllocationError("allocation maximum_runtime_model_bytes must be positive") + if args.model.stat().st_size > maximum_model_bytes: + raise AllocationError(f"runtime model is {args.model.stat().st_size} bytes; maximum is {maximum_model_bytes}") + if policy["allocation"].get("active_strategy") not in {"legacy", "balanced"}: + raise AllocationError("allocation active_strategy must be legacy or balanced") + if policy["allocation"]["active_strategy"] != "legacy" and not model["estimates"]: + raise AllocationError("balanced allocation cannot be active with an empty runtime model") + print( + f"CI allocation contract verified: strategy={policy['allocation']['active_strategy']}, " + f"estimates={len(model['estimates'])}" + ) + + +def command_export_pr_history(args: argparse.Namespace) -> None: + from tests.suitespec import get_patterns + from tests.suitespec import get_suites + + patterns = { + suite: get_patterns(suite) for suite, config in get_suites().items() if config.get("type", "test") == "test" + } + shapes = collect_pr_shapes( + root=ROOT, + suite_patterns=patterns, + since=args.since, + max_count=args.max_count, + ) + write_pr_shapes(args.output, shapes) + print(f"wrote {len(shapes)} PR workload shapes to {args.output}") + + +def command_replay_pr_history(args: argparse.Namespace) -> None: + policy = _policy(args.policy) + suites, suite_info = _suite_catalog() + report = replay_pr_shapes( + shapes=load_pr_shapes(args.pr_history), + suite_configs=suites, + suite_venv_info=suite_info, + runtime_model=load_json(args.model), + target_jobs=int(policy["allocation"]["target_jobs"]), + target_shard_seconds=float(policy["allocation"]["target_shard_seconds"]), + maximum_parallelism_per_suite=int(policy["allocation"]["maximum_parallelism_per_suite"]), + maximum_slices_per_hash=int(policy["allocation"]["maximum_slices_per_hash"]), + ) + write_json(args.output, report) + print(f"wrote PR-shape replay to {args.output}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + select = subparsers.add_parser("select", help="select hashes for one CI node") + select.add_argument("--suite", required=True) + select.add_argument("--strategy", choices=("legacy", "balanced")) + select.add_argument("--node-index", type=int, required=True) + select.add_argument("--node-total", type=int, required=True) + select.add_argument("--model", type=Path, default=DEFAULT_MODEL) + select.add_argument("--policy", type=Path, default=DEFAULT_POLICY) + select.add_argument("--plan", type=Path, default=DEFAULT_PLAN) + select.set_defaults(func=command_select) + + ingest = subparsers.add_parser("ingest-datadog", help="normalize Test Visibility session exports") + ingest.add_argument("--input", type=Path, required=True) + ingest.add_argument("--output", type=Path, required=True) + ingest.set_defaults(func=command_ingest) + + ingest_jobs = subparsers.add_parser("ingest-jobs", help="normalize Datadog CI job event exports") + ingest_jobs.add_argument("--input", type=Path, required=True) + ingest_jobs.add_argument("--output", type=Path, required=True) + ingest_jobs.set_defaults(func=command_ingest_jobs) + + build_model = subparsers.add_parser("build-model", help="fit a model and optionally replay its holdout") + build_model.add_argument("--observations", type=Path, required=True) + build_model.add_argument("--output", type=Path, required=True) + build_model.add_argument("--jobs", type=Path, required=True) + build_model.add_argument("--report", type=Path) + build_model.add_argument("--policy", type=Path, default=DEFAULT_POLICY) + build_model.set_defaults(func=command_build_model) + + replay = subparsers.add_parser("replay-observations", help="replay a model over its historical holdout") + replay.add_argument("--observations", type=Path, required=True) + replay.add_argument("--model", type=Path, default=DEFAULT_MODEL) + replay.add_argument("--policy", type=Path, default=DEFAULT_POLICY) + replay.add_argument("--output", type=Path, required=True) + replay.set_defaults(func=command_replay_observations) + + live_report = subparsers.add_parser("build-live-report", help="compare paired same-pipeline shadow runs") + live_report.add_argument("--observations", type=Path, required=True) + live_report.add_argument("--jobs", type=Path, required=True) + live_report.add_argument("--output", type=Path, required=True) + live_report.set_defaults(func=command_live_report) + + ratchet = subparsers.add_parser("check-ratchet", help="enforce historical or live promotion thresholds") + ratchet.add_argument("--report", type=Path, required=True) + ratchet.add_argument("--policy", type=Path, default=DEFAULT_POLICY) + ratchet.set_defaults(func=command_check_ratchet) + + verify = subparsers.add_parser("verify-plan", help="verify an allocation manifest") + verify.add_argument("--plan", type=Path, required=True) + verify.set_defaults(func=command_verify_plan) + + verify_junit = subparsers.add_parser("verify-junit", help="compare collected tests from legacy and shadow jobs") + verify_junit.add_argument("--legacy", type=Path, nargs="+", required=True) + verify_junit.add_argument("--balanced", type=Path, nargs="+", required=True) + verify_junit.add_argument("--output", type=Path) + verify_junit.set_defaults(func=command_verify_junit) + + verify_runtime = subparsers.add_parser( + "verify-runtime-shards", help="verify exact pytest inventory coverage across sub-hash slices" + ) + verify_runtime.add_argument("--manifests", type=Path, nargs="+", required=True) + verify_runtime.add_argument("--plan", type=Path, default=DEFAULT_PLAN) + verify_runtime.add_argument("--output", type=Path) + verify_runtime.set_defaults(func=command_verify_runtime_shards) + + contract = subparsers.add_parser("check-contract", help="validate the checked-in policy and runtime model") + contract.add_argument("--policy", type=Path, default=DEFAULT_POLICY) + contract.add_argument("--model", type=Path, default=DEFAULT_MODEL) + contract.set_defaults(func=command_check_contract) + + export_prs = subparsers.add_parser("export-pr-history", help="export historical PR path-selection shapes") + export_prs.add_argument("--since", default="2 years ago") + export_prs.add_argument("--max-count", type=int) + export_prs.add_argument("--output", type=Path, required=True) + export_prs.set_defaults(func=command_export_pr_history) + + replay_prs = subparsers.add_parser("replay-pr-history", help="replay the planner over PR path-selection history") + replay_prs.add_argument("--pr-history", type=Path, required=True) + replay_prs.add_argument("--model", type=Path, default=DEFAULT_MODEL) + replay_prs.add_argument("--policy", type=Path, default=DEFAULT_POLICY) + replay_prs.add_argument("--output", type=Path, required=True) + replay_prs.set_defaults(func=command_replay_pr_history) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.command in UV_REQUIRED_COMMANDS and os.getenv("CI_ALLOCATION_UV_REEXEC") != "1": + environment = dict(os.environ) + environment["CI_ALLOCATION_UV_REEXEC"] = "1" + runner = ROOT / "scripts" / "uv-run-script" + os.execve(str(runner), [str(runner), str(Path(__file__)), *sys.argv[1:]], environment) + try: + args.func(args) + except (AllocationError, FileNotFoundError, ValueError, json.JSONDecodeError, subprocess.CalledProcessError) as exc: + print(f"CI allocation error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index e2d390c9d53..778db247c57 100755 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -32,6 +32,25 @@ BENCHMARK_SCENARIO_REGEX = re.compile(" +- name: ([a-z0-9]+)-.+") +_SUITE_PIP_CACHE_KEYS: dict[str, str] = {} + + +def _riot_pip_cache_key(suite_name: str) -> str: + if suite_name in _SUITE_PIP_CACHE_KEYS: + return _SUITE_PIP_CACHE_KEYS[suite_name] + return subprocess.check_output([".gitlab/scripts/get-riot-pip-cache-key.sh", suite_name]).decode().strip() + + +def _pip_cache_key_for_hashes(riot_hashes: t.Iterable[str]) -> str: + lines = [] + for riot_hash in sorted(riot_hashes): + requirements = ROOT / ".riot" / "requirements" / f"{riot_hash}.txt" + if requirements.is_file(): + lines.extend(requirements.read_text().splitlines()) + payload = "".join(f"{line}\n" for line in sorted(lines)).encode() + return hashlib.sha256(payload).hexdigest() + + def _get_bool_env(name: str) -> str: """Return "true"/"false" for a boolean environment variable. @@ -133,16 +152,13 @@ def __str__(self) -> str: if wait_for: lines.append(f" - riot -v run -s --pass-env wait -- {' '.join(wait_for)}") - env = self.env + env = dict(self.env or {}) if not env or "SUITE_NAME" not in env: - env = env or {} env["SUITE_NAME"] = self.pattern or self.name suite_name = env["SUITE_NAME"] env["PIP_CACHE_DIR"] = "${CI_PROJECT_DIR}/.cache/pip" - env["PIP_CACHE_KEY"] = ( - subprocess.check_output([".gitlab/scripts/get-riot-pip-cache-key.sh", suite_name]).decode().strip() - ) + env["PIP_CACHE_KEY"] = _riot_pip_cache_key(suite_name) if not self.skip_pip_cache: lines.append(" cache:") lines.append(f" key: v1-pip-${'{PIP_CACHE_KEY}'}-{TESTRUNNER_IMAGE_HASH}-cache") @@ -173,152 +189,13 @@ def __str__(self) -> str: return "\n".join(lines) -@dataclass -class SuiteVenvInfo: - venv_count: int - python_versions: set[str] - - # Module-level state: populated by gen_required_suites, consumed by gen_build_base_venvs _global_python_versions: set[str] = set() -# Target minimum number of GitLab job instances for a CI run (used to scale up sparse runs) -TARGET_JOBS = 200 - # All supported Python versions (fallback when no venv info is available) ALL_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] -def collect_all_suite_venv_info(suite_patterns: dict[str, str]) -> dict[str, SuiteVenvInfo]: - """Collect venv count and Python versions for multiple suites in a single pass. - - Iterates riotfile.venv.instances() once and matches each instance against all - suite patterns simultaneously, which is much more efficient than per-suite iteration. - - Args: - suite_patterns: mapping of suite name -> regex pattern string - - Returns: - mapping of suite name -> SuiteVenvInfo for suites that have matching venvs - """ - # Importing will load/evaluate the whole riotfile.py - import riotfile - - compiled: dict[str, re.Pattern] = {} - for suite, pattern in suite_patterns.items(): - try: - compiled[suite] = re.compile(pattern) - except re.error: - LOGGER.warning("Invalid pattern for suite %s: %s", suite, pattern) - - venv_hashes: dict[str, set] = {s: set() for s in compiled} - python_versions: dict[str, set] = {s: set() for s in compiled} - - for inst in riotfile.venv.instances(): # type: ignore[attr-defined] - if not inst.name: - continue - hint = inst.py._hint # type: ignore[attr-defined] - for suite, regex in compiled.items(): - if inst.matches_pattern(regex): # type: ignore[attr-defined] - venv_hashes[suite].add(inst.short_hash) # type: ignore[attr-defined] - # Only collect properly versioned hints (e.g. "3.10"), skip bare "3" - if re.match(r"^3\.\d+$", hint): - python_versions[suite].add(hint) - - result: dict[str, SuiteVenvInfo] = {} - for suite in compiled: - if venv_hashes[suite]: - result[suite] = SuiteVenvInfo( - venv_count=len(venv_hashes[suite]), - python_versions=python_versions[suite], - ) - else: - LOGGER.warning("No riot venvs found for suite %s with pattern %s", suite, suite_patterns[suite]) - return result - - -def calculate_parallelism_from_venvs(venv_count: int, venvs_per_job: int, max_parallelism: int = 25) -> int: - """Calculate parallelism given a venv count and venvs_per_job packing density.""" - import math - - return min(math.ceil(venv_count / venvs_per_job), max_parallelism) - - -def _scale_suites( - suite_venv_info: dict[str, SuiteVenvInfo], - final_jobs: dict[str, int], - scalable_suites: list[str], - venvs_per_job_map: dict[str, int], - target: int, -) -> dict[str, int]: - """Scale up parallelism for scalable suites to approach the target total job count. - - Works for both venvs_per_job suites (reduces vpj by 1 per step) and static - parallelism suites (increments parallelism by 1 per step). Each iteration picks - the suite that yields the largest gain until the target is reached. - - Args: - suite_venv_info: venv info per suite (from collect_all_suite_venv_info) - final_jobs: current parallelism per suite (a copy is returned) - scalable_suites: all suites eligible for scaling (with venv info) - venvs_per_job_map: current venvs_per_job value for dynamic suites (others absent) - target: desired minimum total job count - - Returns: - Updated parallelism mapping - """ - import math - - final_jobs = dict(final_jobs) - current_vpj = dict(venvs_per_job_map) - - while sum(final_jobs.values()) < target: - best_gain = 0 - best_suite = None - - for suite in scalable_suites: - venv_count = suite_venv_info[suite].venv_count - current = final_jobs[suite] - # Allow up to 1 job per venv when scaling (no parallelism cap during scale-up). - # The cap in calculate_parallelism_from_venvs only applies to baseline. - if current >= venv_count: - continue - - if suite in current_vpj: - # Dynamic (venvs_per_job) suite: compute gain from reducing vpj by 1 - vpj = current_vpj[suite] - if vpj <= 1: - continue - new_parallelism = math.ceil(venv_count / (vpj - 1)) - else: - # Static parallelism suite: gain is always 1 - new_parallelism = current + 1 - - gain = new_parallelism - current - if gain > best_gain: - best_gain = gain - best_suite = suite - - if best_suite is None or best_gain == 0: - break - - venv_count = suite_venv_info[best_suite].venv_count - if best_suite in current_vpj: - current_vpj[best_suite] -= 1 - final_jobs[best_suite] = math.ceil(venv_count / current_vpj[best_suite]) - else: - final_jobs[best_suite] += 1 - - LOGGER.debug( - "Scaled suite %s: parallelism %d -> %d", - best_suite, - final_jobs[best_suite] - best_gain, - final_jobs[best_suite], - ) - - return final_jobs - - def gen_required_suites() -> None: """Generate the list of test and benchmark suites that need to be run.""" import suitespec @@ -493,48 +370,134 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: non_skipped = [s for s in required_suites if not suites[s].get("skip", False)] suite_patterns = {s: suites[s].get("pattern", s) for s in non_skipped} suite_venv_info = collect_all_suite_venv_info(suite_patterns) + missing_suites = sorted(set(non_skipped) - set(suite_venv_info)) + if missing_suites: + raise ValueError(f"selected suites have no Riot environments: {missing_suites}") + for suite, pattern in suite_patterns.items(): + cache_key = _pip_cache_key_for_hashes(suite_venv_info[suite].hashes) + previous = _SUITE_PIP_CACHE_KEYS.setdefault(pattern, cache_key) + if previous != cache_key: + raise ValueError(f"suite pattern has inconsistent Riot requirements: {pattern}") # Populate the module-level global so gen_build_base_venvs can use it _global_python_versions = set() for info in suite_venv_info.values(): _global_python_versions.update(info.python_versions) - # Compute baseline parallelism. Track scalable suites (those with venv info, eligible - # for scaling up) and the vpj map for dynamic suites. - baseline_jobs: dict[str, int] = {} - scalable_suites: list[str] = [] # all suites with venv info (both static and dynamic) - venvs_per_job_map: dict[str, int] = {} # only for venvs_per_job suites - - for suite in non_skipped: - config = suites[suite] - static_parallelism = config.get("parallelism") - venvs_per_job = config.get("venvs_per_job") - - if static_parallelism is not None: - baseline_jobs[suite] = static_parallelism - if suite in suite_venv_info: - scalable_suites.append(suite) - elif venvs_per_job is not None and suite in suite_venv_info: - parallelism = calculate_parallelism_from_venvs(suite_venv_info[suite].venv_count, venvs_per_job) - baseline_jobs[suite] = parallelism - scalable_suites.append(suite) - venvs_per_job_map[suite] = venvs_per_job - else: - baseline_jobs[suite] = 1 - - # Scale up suites if total job count is below the target - total_baseline = sum(baseline_jobs.values()) - if total_baseline < TARGET_JOBS and scalable_suites: - LOGGER.info( - "Total baseline jobs (%d) below target (%d), scaling up %d suite(s)", - total_baseline, - TARGET_JOBS, - len(scalable_suites), + allocation_policy = load_json(CI_ALLOCATION_POLICY) + allocation_config = allocation_policy["allocation"] + legacy_jobs = compute_parallelism( + suites, + non_skipped, + suite_venv_info, + target_jobs=int(allocation_config["target_jobs"]), + ) + + runtime_model = load_json(CI_ALLOCATION_MODEL) + estimates, global_fallback = runtime_estimates(runtime_model) + estimates_by_suite = {suite: runtime_estimates(runtime_model, suite)[0] for suite in non_skipped} + setup_seconds = {suite: runtime_setup_seconds(runtime_model, suite) for suite in non_skipped} + test_item_counts = {suite: runtime_test_item_counts(suite_venv_info[suite], runtime_model) for suite in non_skipped} + balanced_jobs = ( + compute_runtime_parallelism( + suite_venv_info, + non_skipped, + estimates, + runtime_model["fallbacks"].get("suite_seconds", {}), + global_fallback, + target_shard_seconds=float(allocation_config["target_shard_seconds"]), + maximum_parallelism_per_suite=int(allocation_config["maximum_parallelism_per_suite"]), + maximum_total_jobs=sum(legacy_jobs.values()), + suite_overheads=setup_seconds, + global_overhead=float(runtime_model["overheads"]["unit_global_seconds"]), + test_item_counts_by_suite=test_item_counts, + estimates_by_suite=estimates_by_suite, + maximum_slices_per_hash=int(allocation_config["maximum_slices_per_hash"]), + ) + if estimates + else legacy_jobs + ) + active_strategy = allocation_config["active_strategy"] + if active_strategy not in {"legacy", "balanced"}: + raise ValueError(f"unsupported CI allocation strategy: {active_strategy}") + if active_strategy == "balanced" and not estimates: + raise ValueError("balanced CI allocation requires a populated runtime model") + final_jobs = legacy_jobs if active_strategy == "legacy" else balanced_jobs + LOGGER.info("Selected %s suite jobs: %d", active_strategy, sum(final_jobs.values())) + shadow_enabled = _get_bool_env("CI_ALLOCATION_SHADOW") == "true" + if shadow_enabled and (active_strategy != "legacy" or not estimates): + raise ValueError("allocation shadowing requires a populated model with the legacy strategy active") + plan_configs = { + suite: {key: value for key, value in suites[suite].items() if not key.startswith("_")} for suite in non_skipped + } + allocation_manifest = build_allocation_manifest( + suite_venv_info=suite_venv_info, + suite_configs=plan_configs, + legacy_shard_counts=legacy_jobs, + balanced_shard_counts=balanced_jobs, + runtime_model=runtime_model, + active_strategy=active_strategy, + target_shard_seconds=float(allocation_config["target_shard_seconds"]), + maximum_slices_per_hash=int(allocation_config["maximum_slices_per_hash"]), + ) + write_manifest(CI_ALLOCATION_PLAN, allocation_manifest) + plans = {plan["suite"]: plan for plan in allocation_manifest["suites"]} + + def emit_jobs( + output: t.TextIO, + *, + suite: str, + clean_name: str, + stage: str, + suite_config: dict, + strategy: str, + shadow: bool, + ) -> None: + assignments = plans[suite][strategy]["assignments"] + config = suite_config.copy() + config["parallelism"] = len(assignments) if len(assignments) > 1 else None + config["env"] = dict(config.get("env") or {}) + config["env"].update( + { + "SUITE_NAME": config.get("pattern", clean_name), + "CI_ALLOCATION_SUITE": suite, + "CI_ALLOCATION_STRATEGY": strategy, + "CI_ALLOCATION_ASSIGNMENTS": ";".join(",".join(assignment) for assignment in assignments), + } + ) + if shadow: + config["allow_failure"] = True + job_name = clean_name + ("-allocation-shadow" if shadow else "") + print( + JobSpec( + job_name, + stage=stage, + python_versions=set(suite_venv_info[suite].python_versions), + **config, + ), + file=output, + ) + + def emit_legacy_matrix(output: t.TextIO, *, suite: str, clean_name: str, stage: str, suite_config: dict) -> None: + config = suite_config.copy() + config["parallelism"] = legacy_jobs[suite] if legacy_jobs[suite] > 1 else None + config["env"] = dict(config.get("env") or {}) + config["env"].update( + { + "SUITE_NAME": config.get("pattern", clean_name), + "CI_ALLOCATION_SUITE": suite, + "CI_ALLOCATION_STRATEGY": "legacy", + } + ) + print( + JobSpec( + clean_name, + stage=stage, + python_versions=set(suite_venv_info[suite].python_versions), + **config, + ), + file=output, ) - final_jobs = _scale_suites(suite_venv_info, baseline_jobs, scalable_suites, venvs_per_job_map, TARGET_JOBS) - LOGGER.info("Scaled total jobs: %d", sum(final_jobs.values())) - else: - final_jobs = baseline_jobs # === PASS 2: Emit YAML === with TESTS_GEN.open("a") as f: @@ -542,23 +505,39 @@ def _gen_tests(suites: dict, required_suites: list[str]) -> None: suite_config = suites[suite].copy() stage = suite_config.pop("_stage", "core") clean_name = suite_config.pop("_clean_name", suite) - - py_versions = suite_venv_info[suite].python_versions if suite in suite_venv_info else None - jobspec = JobSpec(clean_name, stage=stage, python_versions=py_versions, **suite_config) - if jobspec.skip: + if suite_config.get("skip", False): LOGGER.debug("Skipping suite %s", suite) continue - - # Apply final parallelism (may be higher than baseline if scaling was applied) - final_parallelism = final_jobs.get(suite) - if final_parallelism is not None and final_parallelism > 1: - if jobspec.parallelism != final_parallelism: - LOGGER.info("Suite %s: parallelism=%d", suite, final_parallelism) - jobspec.parallelism = final_parallelism - elif jobspec.parallelism is None and (final_parallelism is None or final_parallelism <= 1): - pass # leave as None (GitLab default: single job) - - print(str(jobspec), file=f) + LOGGER.info("Suite %s: %s jobs=%d", suite, active_strategy, final_jobs[suite]) + if active_strategy == "legacy": + emit_legacy_matrix( + f, + suite=suite, + clean_name=clean_name, + stage=stage, + suite_config=suite_config, + ) + else: + emit_jobs( + f, + suite=suite, + clean_name=clean_name, + stage=stage, + suite_config=suite_config, + strategy=active_strategy, + shadow=False, + ) + + if shadow_enabled: + emit_jobs( + f, + suite=suite, + clean_name=clean_name, + stage=stage, + suite_config=suite_config, + strategy="balanced", + shadow=True, + ) def gen_build_docs() -> None: @@ -649,6 +628,19 @@ def check(name: str, command: str, paths: set[str]) -> None: command="scripts/lint suitespec-check", paths={"*"}, ) + check( + name="Check CI allocation contract", + command="scripts/ci_allocation_cli.py check-contract", + paths={ + ".gitlab/*", + "ci/ci-allocation-*.json", + "scripts/ci_allocation/*", + "scripts/ci_allocation_cli.py", + "scripts/gen_gitlab_config.py", + "tests/suitespec.py", + "tests/**/suitespec.yml", + }, + ) check( name="Check ddtrace error logs", command="scripts/lint error-log-check", @@ -817,6 +809,9 @@ def gen_build_base_venvs() -> None: GITLAB = ROOT / ".gitlab" TESTS = ROOT / "tests" TESTS_GEN = GITLAB / "tests-gen.yml" +CI_ALLOCATION_PLAN = GITLAB / "ci-allocation-plan.json" +CI_ALLOCATION_MODEL = ROOT / "ci" / "ci-allocation-runtime-model.json" +CI_ALLOCATION_POLICY = ROOT / "ci" / "ci-allocation-policy.json" MICROBENCHMARKS_GEN = GITLAB / "benchmarks/microbenchmarks-gen.yml" MICROBENCHMARKS_SLOS = GITLAB / "benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml" MICROBENCHMARKS_SLOS_TEMPLATE = GITLAB / "benchmarks/bp-runner.microbenchmarks.fail-on-breach.template.yml" @@ -833,6 +828,18 @@ def gen_build_base_venvs() -> None: sys.path.append(str(ROOT / "scripts")) sys.path.append(str(ROOT / "tests")) +from ci_allocation.history import load_json # noqa: E402 +from ci_allocation.history import runtime_estimates # noqa: E402 +from ci_allocation.manifest import build_allocation_manifest # noqa: E402 +from ci_allocation.manifest import write_manifest # noqa: E402 +from ci_allocation.suites import calculate_parallelism_from_venvs # noqa: E402,F401 +from ci_allocation.suites import collect_all_suite_venv_info # noqa: E402 +from ci_allocation.suites import compute_parallelism # noqa: E402 +from ci_allocation.suites import compute_runtime_parallelism # noqa: E402 +from ci_allocation.suites import runtime_setup_seconds # noqa: E402 +from ci_allocation.suites import runtime_test_item_counts # noqa: E402 +from ci_allocation.suites import scale_suites as _scale_suites # noqa: E402,F401 + def template(name: str, **params): """Render a template file with the given parameters.""" diff --git a/tests/appsec/appsec_utils.py b/tests/appsec/appsec_utils.py index 352abda5f6d..7956131a926 100644 --- a/tests/appsec/appsec_utils.py +++ b/tests/appsec/appsec_utils.py @@ -22,6 +22,50 @@ FILE_PATH = Path(__file__).resolve().parent +def _port_is_available(port: int) -> bool: + """Whether a server could bind the port right now. + + Binding is the question that matters, since it is what the next server does. Probing with + connect() instead reports a port as free once a bound server's listen backlog fills, and + opens real connections to a live server. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + sock.bind(("0.0.0.0", int(port))) + return True + except OSError: + return False + + +def _wait_for_port_release(port: int, timeout: float = 10.0) -> bool: + """Wait until the port can be bound again, returning False if it never can. + + Server teardown is best effort and gunicorn workers can outlive it, so without this the + next test to use the same port fails to bind. 31 tests share port 8050. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if _port_is_available(port): + return True + time.sleep(0.1) + return _port_is_available(port) + + +def _server_diagnostics(server_process, port: int, cmd: list) -> str: + """Facts that are not in the captured server output but explain most startup failures.""" + if isinstance(server_process, multiprocessing.Process): + exit_code = server_process.exitcode + else: + exit_code = server_process.poll() + return ( + f"port={port} port_still_bound={not _port_is_available(port)} pid={server_process.pid} " + f"exit_code={exit_code} (None means it was still running, so it was too slow rather " + f"than dead; a non-zero code with the port bound means another server still holds it)\n" + f"command={cmd}" + ) + + @contextmanager def gunicorn_flask_server( use_ddtrace_cmd: bool = True, @@ -335,6 +379,10 @@ def appsec_application_server( if preexec is not None: subprocess_kwargs["preexec_fn"] = preexec # type: ignore[assignment] + # A previous test's server may still hold the port, which would make this one fail to bind. + if not _wait_for_port_release(port): + print(f"WARNING: port {port} was still bound when starting the server") + if use_multiprocess: # Run the server command by replacing the child Python process with the target binary (exec), # ensuring signals/termination behave like the subprocess.Popen path. @@ -374,17 +422,13 @@ def appsec_application_server( print("Server started") except RetryError: raise AssertionError( - "Server failed to start, see stdout and stderr logs" - "\n=== Captured STDOUT ===\n%s=== End of captured STDOUT ===" - "\n=== Captured STDERR ===\n%s=== End of captured STDERR ===" - % (getattr(server_process, "stdout", None), getattr(server_process, "stderr", None)) + "Server failed to start; its output is in the captured stdout/stderr above.\n" + + _server_diagnostics(server_process, port, cmd) ) except Exception: raise AssertionError( - "Server FAILED, see stdout and stderr logs" - "\n=== Captured STDOUT ===\n%s=== End of captured STDOUT ===" - "\n=== Captured STDERR ===\n%s=== End of captured STDERR ===" - % (getattr(server_process, "stdout", None), getattr(server_process, "stderr", None)) + "Server FAILED; its output is in the captured stdout/stderr above.\n" + + _server_diagnostics(server_process, port, cmd) ) # If we run a Gunicorn application, we want to get the child's pid, see test_flask_remoteconfig.py @@ -399,9 +443,8 @@ def appsec_application_server( pass except Exception: raise AssertionError( - "\n=== Captured STDOUT ===\n%s=== End of captured STDOUT ===" - "\n=== Captured STDERR ===\n%s=== End of captured STDERR ===" - % (getattr(server_process, "stdout", None), getattr(server_process, "stderr", None)) + "Server shutdown request failed; its output is in the captured stdout/stderr above.\n" + + _server_diagnostics(server_process, port, cmd) ) finally: try: @@ -433,7 +476,9 @@ def appsec_application_server( assert "Return value is tainted" in stderr_output assert "Tainted arguments:" in stderr_output finally: - pass + # Do not hand the port to the next test while a worker still holds it. + if not _wait_for_port_release(port): + print(f"WARNING: port {port} still bound after server teardown") def _mp_target(_cmd: list[str], _env: dict) -> None: diff --git a/tests/appsec/contrib_appsec/test_django.py b/tests/appsec/contrib_appsec/test_django.py index b422c0787f1..de6d329665b 100644 --- a/tests/appsec/contrib_appsec/test_django.py +++ b/tests/appsec/contrib_appsec/test_django.py @@ -1,5 +1,7 @@ import importlib import os +from pathlib import Path +import shutil import django from django.conf import settings @@ -15,6 +17,25 @@ _FLAT_URLCONF = "tests.appsec.contrib_appsec.django_app.urls" _SUBAPP_URLCONF = "tests.appsec.contrib_appsec.django_app.urls_subapps" +_DATABASE_TEMPLATE = Path(__file__).with_name("db.sqlite3") + + +@pytest.fixture(scope="module", autouse=True) +def isolated_database(tmp_path_factory): + """Use a worker-local copy of the Django database template.""" + database_path = tmp_path_factory.mktemp("appsec-django") / "db.sqlite3" + shutil.copyfile(_DATABASE_TEMPLATE, database_path) + + os.environ["DJANGO_SETTINGS_MODULE"] = "tests.appsec.contrib_appsec.django_app.settings" + original_database_name = settings.DATABASES["default"]["NAME"] + settings.DATABASES["default"]["NAME"] = str(database_path) + try: + yield + finally: + from django.db import connections + + connections.close_all() + settings.DATABASES["default"]["NAME"] = original_database_name class _Test_Django_Base: diff --git a/tests/appsec/iast/fixtures/taint_sinks/path_traversal.py b/tests/appsec/iast/fixtures/taint_sinks/path_traversal.py index 1420b47d0e3..2b98d17822c 100644 --- a/tests/appsec/iast/fixtures/taint_sinks/path_traversal.py +++ b/tests/appsec/iast/fixtures/taint_sinks/path_traversal.py @@ -8,6 +8,7 @@ import pickle import shutil import tarfile +import tempfile from zipfile import ZipFile @@ -58,8 +59,9 @@ def path_os_remove(origin_string): def path_os_rename(origin_string): try: - # label path_os_rename - os.rename(origin_string, "test.txt") + with tempfile.TemporaryDirectory() as tmp_dir: + # label path_os_rename + os.rename(origin_string, os.path.join(tmp_dir, "test.txt")) except Exception: pass @@ -90,24 +92,27 @@ def path_os_listdir(origin_string): def path_shutil_copy(origin_string): try: - # label path_shutil_copy - shutil.copy(origin_string, "not_exists.txt2") + with tempfile.TemporaryDirectory() as tmp_dir: + # label path_shutil_copy + shutil.copy(origin_string, os.path.join(tmp_dir, "copied.txt")) except Exception: pass def path_shutil_copytree(origin_string): try: - # label path_shutil_copytree - shutil.copytree(origin_string, "not_exists.txt2") + with tempfile.TemporaryDirectory() as tmp_dir: + # label path_shutil_copytree + shutil.copytree(origin_string, os.path.join(tmp_dir, "copied")) except Exception: pass def path_shutil_move(origin_string): try: - # label path_shutil_move - shutil.move(origin_string, "not_exists.txt2") + with tempfile.TemporaryDirectory() as tmp_dir: + # label path_shutil_move + shutil.move(origin_string, os.path.join(tmp_dir, "moved.txt")) except Exception: pass diff --git a/tests/appsec/iast/test_product_inspect_regression.py b/tests/appsec/iast/test_product_inspect_regression.py index ddc6d480795..3da12af04a5 100644 --- a/tests/appsec/iast/test_product_inspect_regression.py +++ b/tests/appsec/iast/test_product_inspect_regression.py @@ -17,7 +17,7 @@ import sys -from tests.utils import override_env +from tests.utils import override_global_config def sample_function_with_many_params( @@ -84,16 +84,9 @@ def test_iast_post_preload_does_not_drop_inspect(self): inspect_id_before = id(sys.modules.get("inspect")) inspect_module_before = sys.modules.get("inspect") - # Call post_preload directly (this is what happens in production) - # We need to ensure IAST is enabled for post_preload to run its logic - with override_env({"DD_IAST_ENABLED": "true"}): - # Force reload of asm_config to pick up the environment variable - import importlib - - from ddtrace.internal.settings import asm - - importlib.reload(asm) - + # Enabled in place: reloading the asm module would rebind its config to a new object, + # leaving every module that already imported the old one reading stale settings. + with override_global_config(dict(_iast_enabled=True)): from ddtrace.internal.iast import product # Call post_preload - this should NOT drop inspect diff --git a/tests/appsec/iast/test_telemetry.py b/tests/appsec/iast/test_telemetry.py index 0e672483ea6..a2ce1c498e9 100644 --- a/tests/appsec/iast/test_telemetry.py +++ b/tests/appsec/iast/test_telemetry.py @@ -35,6 +35,23 @@ from tests.utils import override_global_config +@pytest.fixture(autouse=True) +def empty_telemetry_session(request): + """Start every telemetry test with an empty session. + + These tests assert on metric names, counts and exact lists, so a metric emitted by + whatever ran before is enough to fail them. Draining the native worker first pushes out + anything it still holds, so clear() actually leaves nothing behind. + + Only for tests that already use the session: requesting it unconditionally would make the + pure metric_verbosity cases xfail when no test agent is running. + """ + if "test_agent_session" in request.fixturenames: + request.getfixturevalue("telemetry_writer").periodic(force_flush=True) + request.getfixturevalue("test_agent_session").clear() + yield + + def _get_iast_metrics(test_agent_session, telemetry_writer): """Flush the native worker and return the iast-namespace generate-metrics series.""" telemetry_writer.periodic(force_flush=True) @@ -56,11 +73,14 @@ def _get_iast_logs(test_agent_session, telemetry_writer): def _assert_instrumented_sink(test_agent_session, telemetry_writer, vuln_type): generate_metrics = _get_iast_metrics(test_agent_session, telemetry_writer) - assert len(generate_metrics) == 1, "Expected 1 generate_metrics" - assert [metric["metric"] for metric in generate_metrics] == ["instrumented.sink"] - assert [metric["tags"] for metric in generate_metrics] == [[f"vulnerability_type:{vuln_type.lower()}"]] - assert [metric["points"][0][1] for metric in generate_metrics][0] >= 1 - assert [metric["type"] for metric in generate_metrics] == ["count"] + assert generate_metrics, "Expected an instrumented.sink metric" + # Check every series rather than how many: the native worker can flush mid-patching and + # split one metric over several, but each of them still has to be well formed. + for metric in generate_metrics: + assert metric["metric"] == "instrumented.sink" + assert metric["tags"] == [f"vulnerability_type:{vuln_type.lower()}"] + assert metric["type"] == "count" + assert sum(point[1] for metric in generate_metrics for point in metric["points"]) >= 1 @pytest.mark.parametrize( @@ -185,10 +205,6 @@ def test_metric_instrumented_vulnerability(no_request_sampling, telemetry_writer def test_metric_instrumented_propagation(no_request_sampling, telemetry_writer, test_agent_session): - # Drain metrics emitted before this test so the session below holds only our own. - _get_iast_metrics(test_agent_session, telemetry_writer) - test_agent_session.clear() - with override_global_config(dict(_iast_enabled=True, _iast_telemetry_report_lvl=TELEMETRY_INFORMATION_NAME)): _iast_patched_module("benchmarks.bm.iast_fixtures.str_methods") @@ -234,6 +250,8 @@ def test_metric_request_tainted(no_request_sampling, telemetry_writer, test_agen generate_metrics = _get_iast_metrics(test_agent_session, telemetry_writer) # Remove potential sinks from internal usage of the lib (like http.client, used to communicate with # the agent) + # Remove potential sinks from internal usage of the lib (like http.client, used to communicate with + # the agent) filtered_metrics = [metric["metric"] for metric in generate_metrics if metric["metric"] != "executed.sink"] assert filtered_metrics == ["executed.source", "request.tainted"] assert len(filtered_metrics) == 2, "Expected 2 generate_metrics" @@ -347,7 +365,10 @@ def test_django_instrumented_metrics(telemetry_writer, test_agent_session): _on_django_patch() generate_metrics = _get_iast_metrics(test_agent_session, telemetry_writer) - metrics_source_tags_result = [metric["tags"][0] for metric in generate_metrics] + # Only instrumented.source carries a source_type tag; instrumented.propagation has none. + metrics_source_tags_result = [ + metric["tags"][0] for metric in generate_metrics if metric["metric"] == "instrumented.source" + ] assert len(metrics_source_tags_result) == 9 assert f"source_type:{origin_to_str(OriginType.HEADER_NAME)}" in metrics_source_tags_result diff --git a/tests/appsec/suitespec.yml b/tests/appsec/suitespec.yml index 4cc6976ee4b..12ed6fbe08f 100644 --- a/tests/appsec/suitespec.yml +++ b/tests/appsec/suitespec.yml @@ -74,6 +74,7 @@ suites: paths: - '@appsec_iast' - tests/appsec/iast_packages/* + - tests/appsec/appsec_utils.py timeout: 50m iast_tdd_propagation: venvs_per_job: 1 @@ -84,6 +85,7 @@ suites: - '@appsec_iast' - '@remoteconfig' - tests/appsec/iast_tdd_propagation/* + - tests/appsec/appsec_utils.py retry: 2 snapshot: true appsec_integrations_pygoat: @@ -148,6 +150,7 @@ suites: - '@appsec_iast' - tests/appsec/integrations/flask_tests/test_iast_flask.py - tests/appsec/integrations/flask_tests/test_appsec_flask_telemetry.py + - tests/appsec/appsec_utils.py retry: 2 # test_appsec_flask_telemetry.py asserts on payloads received by the test agent. snapshot: true @@ -162,6 +165,7 @@ suites: - '@appsec_iast' - '@remoteconfig' - tests/appsec/integrations/flask_tests/* + - tests/appsec/appsec_utils.py retry: 2 services: - testagent @@ -176,6 +180,7 @@ suites: - '@appsec_iast' - '@remoteconfig' - tests/appsec/integrations/django_tests/* + - tests/appsec/appsec_utils.py retry: 2 services: - testagent @@ -190,6 +195,7 @@ suites: - '@appsec_iast' - '@remoteconfig' - tests/appsec/integrations/fastapi_tests/* + - tests/appsec/appsec_utils.py retry: 2 services: - testagent diff --git a/tests/contrib/flask/test_appsec_flask_snapshot.py b/tests/contrib/flask/test_appsec_flask_snapshot.py index f8b10eefe5f..2c58e2a9d29 100644 --- a/tests/contrib/flask/test_appsec_flask_snapshot.py +++ b/tests/contrib/flask/test_appsec_flask_snapshot.py @@ -2,7 +2,6 @@ import signal import subprocess import sys -import time from typing import Callable # noqa:F401 from typing import Generator # noqa:F401 @@ -89,11 +88,8 @@ def flask_client( client.get_ignored("/shutdown") except Exception: pass - # At this point the traces have been sent to the test agent - # but the test agent hasn't necessarily finished processing - # the traces (race condition) so wait just a bit for that - # processing to complete. - time.sleep(0.2) + # The test agent may not have finished processing the traces yet. Each test declares + # wait_for_num_traces so the snapshot polls for them rather than racing a fixed sleep. finally: os.killpg(proc.pid, signal.SIGKILL) stdout, stderr = proc.communicate() @@ -103,6 +99,7 @@ def flask_client( @pytest.mark.snapshot( + wait_for_num_traces=1, ignores=[ "error", "type", @@ -135,6 +132,7 @@ def test_flask_ipblock_match_403(flask_client): @pytest.mark.snapshot( + wait_for_num_traces=1, ignores=[ "error", "type", @@ -167,6 +165,7 @@ def test_flask_ipblock_match_403_json(flask_client): @pytest.mark.snapshot( + wait_for_num_traces=1, ignores=[ "error", "type", @@ -198,6 +197,7 @@ def test_flask_userblock_match_403_json(flask_client): @pytest.mark.snapshot( + wait_for_num_traces=1, ignores=[ "error", "type", @@ -229,6 +229,7 @@ def test_flask_userblock_match_200_json(flask_client): @pytest.mark.snapshot( + wait_for_num_traces=1, ignores=[ "error", "type", @@ -261,6 +262,7 @@ def test_flask_processexec_ossystem(flask_client): @pytest.mark.snapshot( + wait_for_num_traces=1, ignores=[ "error", "type", @@ -294,6 +296,7 @@ def test_flask_processexec_osspawn(flask_client): @pytest.mark.snapshot( + wait_for_num_traces=1, ignores=[ "error", "type", @@ -326,6 +329,7 @@ def test_flask_processexec_subprocesscommunicateshell(flask_client): @pytest.mark.snapshot( + wait_for_num_traces=1, ignores=[ "error", "type", diff --git a/tests/contrib/pydantic_ai/test_pydantic_ai_llmobs.py b/tests/contrib/pydantic_ai/test_pydantic_ai_llmobs.py index 21ad4bb7fe2..b1801edb40b 100644 --- a/tests/contrib/pydantic_ai/test_pydantic_ai_llmobs.py +++ b/tests/contrib/pydantic_ai/test_pydantic_ai_llmobs.py @@ -1,15 +1,28 @@ import json +import sys +from typing import Optional +from typing import Union import mock +from pydantic import BaseModel import pydantic_ai import pytest from typing_extensions import TypedDict from ddtrace.internal.utils.version import parse_version from ddtrace.llmobs._utils import _get_llmobs_data_metastruct -from ddtrace.llmobs._utils import load_data_value from ddtrace.llmobs._utils import safe_json +from ddtrace.llmobs.types import AgentManifest +from tests.contrib.pydantic_ai.utils import ABSENT +from tests.contrib.pydantic_ai.utils import MANIFEST_FIELD_CASES +from tests.contrib.pydantic_ai.utils import MANIFEST_LEAK_CASES from tests.contrib.pydantic_ai.utils import PYDANTIC_AI_TAGS +from tests.contrib.pydantic_ai.utils import _assert_contains +from tests.contrib.pydantic_ai.utils import _function_model +from tests.contrib.pydantic_ai.utils import _manifest_of +from tests.contrib.pydantic_ai.utils import _tenant_toolset +from tests.contrib.pydantic_ai.utils import _test_model +from tests.contrib.pydantic_ai.utils import _UnserializableSentinel from tests.contrib.pydantic_ai.utils import calculate_square_tool from tests.contrib.pydantic_ai.utils import expected_agent_metadata from tests.contrib.pydantic_ai.utils import expected_calculate_square_tool @@ -53,7 +66,7 @@ async def test_agent_run(self, pydantic_ai, request_vcr, pydantic_ai_llmobs, tes metadata=expected_agent_metadata( instructions=instructions, system_prompt=system_prompt, - model_settings=model_settings, + model_params=model_settings, tools=expected_calculate_square_tool(), ), tags=PYDANTIC_AI_TAGS, @@ -252,7 +265,12 @@ class Output(TypedDict): name="test_agent", input_value="What is the square of 2?", output_value=safe_json(output[0].parts[0].args, ensure_ascii=False), - metadata=expected_agent_metadata(instructions=instructions, tools=expected_calculate_square_tool()), + metadata=expected_agent_metadata( + instructions=instructions, + tools=expected_calculate_square_tool(), + # A TypedDict output yields a name but no schema, since only a pydantic model has one. + data_contracts={"output": {"name": "Output"}}, + ), tags=PYDANTIC_AI_TAGS, ) assert_llmobs_span_data( @@ -443,33 +461,25 @@ async def test_agent_run_with_user_prompt_and_message_history( ) async def test_agent_run_with_unserializable_model_settings(self, pydantic_ai, pydantic_ai_llmobs, test_spans): - """Regression test: agent.model_settings containing non-JSON-serializable provider - sentinel values must not crash span submission. + """A non-serializable provider sentinel in model_settings must not crash span submission. - Uses FunctionModel to avoid OpenAI SDK serialization, which would reject the - sentinel before our span-tagging code ever runs. + FunctionModel avoids OpenAI SDK serialization, which would reject it before span tagging. """ - from pydantic_ai.messages import ModelResponse - from pydantic_ai.messages import TextPart - from pydantic_ai.models.function import FunctionModel - - def model_func(messages, info): - return ModelResponse(parts=[TextPart(content="Hello!")]) - agent = pydantic_ai.Agent( - model=FunctionModel(model_func), + model=_function_model(), name="test_agent", model_settings={"temperature": _UnserializableSentinel(), "max_tokens": 100}, ) await agent.run("Hello, world!") spans = [s for trace in test_spans.pop_traces() for s in trace] assert len(spans) == 1 - span_data = _get_llmobs_data_metastruct(spans[0]) - recorded_settings = span_data["meta"]["metadata"]["_dd"]["agent_manifest"]["model_settings"] - # Coerced values must be JSON-serializable. - json.dumps(recorded_settings) - assert recorded_settings["max_tokens"] == 100 - assert recorded_settings["temperature"] == "Omit()" + settings = _manifest_of(spans[0])["model_settings"] + # The whole manifest has to survive the encoder, which is what the sentinel used to break. + json.dumps(_manifest_of(spans[0])) + assert settings["max_tokens"] == 100 + # A sentinel stands for "not set", so the field is absent rather than holding the string + # "Omit()" in a slot that is meant to hold a number. + assert "temperature" not in settings class TestLLMObsPydanticAISpanLinks: @@ -501,30 +511,625 @@ async def test_agent_calls_tool(self, pydantic_ai, request_vcr, pydantic_ai_llmo assert second_llm_span_data["span_links"][0]["attributes"] == {"from": "output", "to": "input"} -class _UnserializableSentinel: - """Stand-in for provider sentinels such as OpenAI's ``Omit`` / ``NOT_GIVEN``.""" +@pytest.mark.parametrize( + "ddtrace_global_config", + [dict(_llmobs_enabled=True, _llmobs_ml_app="")], +) +class TestPydanticAIAgentManifest: + """The agent manifest as a contract: shape, omissions, and what must never ship. - def __repr__(self): - return "Omit()" + These go through a real agent.run() so the manifest is read off the span the customer would get, + not off a direct builder call. + """ + # Every top-level key the shared schema allows. A key outside this set is either a typo or an + # invention, and both are caught by test_shape_is_one_flat_document rather than by review. + # Derived from the type rather than restated, so AgentManifest is the one definition of the + # schema. mypy already rejects an unknown key at the assignment; this catches the same mistake + # from the wire side, where a key could be introduced by a nested dict it cannot see. + SCHEMA_KEYS = frozenset(AgentManifest.__annotations__) -def test_model_settings_unserializable_values_are_coerced(): - """Regression test: ``agent.model_settings`` may hold provider sentinels (e.g. - OpenAI's ``Omit``/``NOT_GIVEN`` for unset params). The agent manifest is written to - the span's meta_struct, so storing these raw crashed the agentless trace encoder at - span finish (``TypeError: Object of type Omit is not JSON serializable``). They must - be coerced to JSON-safe values while serializable settings are preserved. - """ - raw = {"temperature": _UnserializableSentinel(), "max_tokens": 100} - # This is what used to be stored raw on the span and crash encoding. - with pytest.raises(TypeError): - json.dumps(raw) + async def _run(self, pydantic_ai, test_spans, **agent_kwargs): + agent_kwargs.setdefault("model", _function_model()) + agent = pydantic_ai.Agent(**agent_kwargs) + await agent.run("Hello, world!") + # The agent span is the root of the trace. A structured output can add a tool span underneath + # it on some versions, so index rather than asserting a span count. + trace = test_spans.pop_traces()[0] + return agent, _manifest_of(trace[0]) + + @pytest.mark.parametrize("make_kwargs,expected,min_version", MANIFEST_FIELD_CASES) + async def test_field_mapping_cases( + self, pydantic_ai, pydantic_ai_llmobs, test_spans, make_kwargs, expected, min_version + ): + """One case per field mapping, asserted as a subset so unrelated keys do not couple.""" + if min_version and PYDANTIC_AI_VERSION < min_version: + pytest.skip("pydantic-ai < {} does not support this field".format(min_version)) + + _, manifest = await self._run(pydantic_ai, test_spans, name="test_agent", **make_kwargs()) + + if isinstance(expected, ABSENT): + for key in expected.keys: + assert key not in manifest, "manifest should not contain {}".format(key) + else: + _assert_contains(manifest, expected) + + @pytest.mark.parametrize("make_kwargs,forbidden,expected,min_version", MANIFEST_LEAK_CASES) + async def test_secrets_never_ship_cases( + self, pydantic_ai, pydantic_ai_llmobs, test_spans, make_kwargs, forbidden, expected, min_version + ): + """The security contract, one case per carrier. expected is asserted so a case cannot pass empty.""" + if min_version and PYDANTIC_AI_VERSION < min_version: + pytest.skip("pydantic-ai < {} does not support this field".format(min_version)) + + _, manifest = await self._run(pydantic_ai, test_spans, name="test_agent", **make_kwargs()) + + blob = safe_json(manifest) + for canary in forbidden: + assert canary not in blob, "{} reached the manifest".format(canary) + _assert_contains(manifest, expected) + + async def test_shape_is_one_flat_document(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """One flat document whose keys all come from the shared schema, driven off a configured agent.""" + + class Deps: + tenant: str + + _, manifest = await self._run( + pydantic_ai, + test_spans, + name="test_agent", + instructions="Stay terse.", + system_prompt="Cite sources.", + tools=[calculate_square_tool], + deps_type=Deps, + model_settings={"temperature": 0.5, "logit_bias": {50256: -100}, "timeout": 30.0}, + ) + + unknown = set(manifest) - self.SCHEMA_KEYS + assert not unknown, "manifest emits keys outside the shared schema: {}".format(sorted(unknown)) + # No key is a dotted path: the flat schema has no prefixed names to parse apart. + assert not [key for key in manifest if "." in key] + + async def test_framework_is_the_display_name(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """framework keeps the value it has always had, so a consumer filtering on it does not break.""" + _, manifest = await self._run(pydantic_ai, test_spans, name="test_agent") + assert manifest["framework"] == "PydanticAI" + + async def test_field_mapping(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """The whole document compared exactly, which catches a field moving or gaining a wrapper.""" + + class Deps: + tenant: str + + class Resolution(BaseModel): + answer: str + + agent, manifest = await self._run( + pydantic_ai, + test_spans, + name="support_orchestrator", + instructions="You orchestrate specialists.", + system_prompt="Cite sources.", + tools=[calculate_square_tool], + output_type=Resolution, + deps_type=Deps, + # A plain-text model cannot satisfy a structured output_type; it exhausts output retries. + model=_test_model(), + retries=3, + end_strategy="exhaustive", + model_settings={"temperature": 0.2, "max_tokens": 1024}, + ) + + actual = manifest + assert actual["framework"] == "PydanticAI" + assert actual["name"] == "support_orchestrator" + assert actual["instructions"] == "You orchestrate specialists." + assert actual["system_prompts"] == ["Cite sources."] + assert actual["model_settings"] == {"temperature": 0.2, "max_tokens": 1024} + assert actual["tools"] == expected_calculate_square_tool() + assert actual["data_contracts"] == {"output": {"name": "Resolution"}} + assert actual["agent_settings"]["end_strategy"] == "exhaustive" + assert actual["agent_settings"]["deps_type"] == "Deps" + assert agent.model_settings["max_tokens"] == 1024, "the caller's own dict was mutated" + + async def test_model_settings_is_an_allowlist(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A key the allowlist does not name drops, even though nothing denies it by name. + + A denylist would have to enumerate an open set; providers keep adding passthroughs. + """ + _, manifest = await self._run( + pydantic_ai, + test_spans, + name="test_agent", + model_settings={"temperature": 0.2, "some_future_provider_blob": {"token": "sk-unknown"}}, + ) + + assert manifest["model_settings"] == {"temperature": 0.2} + assert "sk-unknown" not in safe_json(manifest) + + async def test_allowlisted_key_still_drops_a_blob_value(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """The allowlist protects the key; a shape check protects the value. + + logit_bias and tool_choice take a caller mapping, the same shape a credential travels in. + """ + _, manifest = await self._run( + pydantic_ai, + test_spans, + name="test_agent", + model_settings={ + "temperature": 0.2, + # A real logit_bias is token id to bias, so a string value here is already invalid. + "logit_bias": {"tok": "sk-canary-in-a-value"}, + "tool_choice": {"function": {"nested": "sk-canary-nested"}}, + }, + ) + + assert manifest["model_settings"] == {"temperature": 0.2} + blob = safe_json(manifest) + assert "sk-canary-in-a-value" not in blob + assert "sk-canary-nested" not in blob + + async def test_legitimate_container_values_survive(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """The shape check must not swallow genuine container config, or the previous test passes trivially.""" + _, manifest = await self._run( + pydantic_ai, + test_spans, + name="test_agent", + model_settings={"stop_sequences": ["END", "STOP"], "logit_bias": {50256: -100}}, + ) + + assert manifest["model_settings"]["stop_sequences"] == ["END", "STOP"] + assert manifest["model_settings"]["logit_bias"] == {"50256": -100} + + async def test_unnamed_agent_reports_the_placeholder_name(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """An agent with no recoverable name still reports one, matching the span name fallback. + + Two unnamed agents therefore share it, which is why name is not identity for versioning. + """ + # Held in a list so pydantic-ai's name inference, which scans the frame for a bound variable, + # finds nothing. + agents = [pydantic_ai.Agent(model=_function_model())] + await agents[0].run("Hello, world!") + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + assert manifest["name"] == "PydanticAI Agent" + + async def test_agent_name_may_be_inferred_by_the_framework(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """pydantic-ai infers a name from the calling frame, and the builder cannot tell it apart. + + Pinned rather than fixed, so the limitation is not rediscovered. + """ + agent = pydantic_ai.Agent(model=_function_model()) + await agent.run("Hello, world!") + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + assert manifest["name"] == "agent" + + async def test_minimal_agent_emits_nothing_empty(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """No key ships as null, "", [] or {}, so absence means not configured. + + origin/main emitted instructions: null for exactly this agent. Walks the whole document. + """ + _, manifest = await self._run(pydantic_ai, test_spans) + + def walk(node, path): + if isinstance(node, dict): + for key, value in node.items(): + assert value is not None, "{}.{} is null".format(path, key) + assert value != "" and value != [] and value != {}, "{}.{} is empty".format(path, key) + walk(value, "{}.{}".format(path, key)) + elif isinstance(node, list): + for index, item in enumerate(node): + walk(item, "{}[{}]".format(path, index)) + + walk(manifest, "manifest") + + async def test_function_tool_appears_once_per_key(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A function tool appears once in tools and once in capabilities, never twice within either. + + Across the two keys is deliberate: tools is compatibility, capabilities the typed superset. + """ + _, manifest = await self._run( + pydantic_ai, test_spans, name="test_agent", tools=[calculate_square_tool, foo_tool] + ) + + tool_names = [tool["name"] for tool in manifest["tools"]] + assert sorted(tool_names) == sorted(set(tool_names)) + assert "capabilities" not in manifest, "a function tool is reported once, under tools" + + async def test_capabilities_are_typed(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """Every capability carries a name and a type from the closed set.""" + _, manifest = await self._run(pydantic_ai, test_spans, name="test_agent", toolsets=[_tenant_toolset]) + + assert manifest["capabilities"] + for capability in manifest["capabilities"]: + assert capability["name"] + assert capability["type"] in {"mcp", "builtin", "custom", "tool_preparation"} + + async def test_extra_instructions_carry_type_and_name(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A dynamic resolver ships flat as {type, name}, naming which text is decided at run time.""" + agent = pydantic_ai.Agent(model=_function_model(), name="test_agent") + + @agent.instructions + def per_tenant_policy() -> str: + """Inject the tenant policy at run time.""" + return "policy" + + await agent.run("Hello, world!") + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + assert manifest["extra_instructions"] == [{"type": "dynamic_instructions", "name": "per_tenant_policy"}] + + async def test_function_source_never_reaches_the_wire(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A callable is recorded by name. A function body can hold a literal secret.""" + agent = pydantic_ai.Agent(model=_function_model(), name="test_agent") + + @agent.output_validator + def reject_ungrounded(value): + """A distinctive marker: SENTINEL_SOURCE_MUST_NOT_SHIP.""" + return value + + await agent.run("Hello, world!") + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + blob = safe_json(manifest) + assert "SENTINEL_SOURCE_MUST_NOT_SHIP" not in blob + assert "def reject_ungrounded" not in blob + assert manifest["guardrails"] == ["reject_ungrounded"] + + async def test_repeated_output_validator_is_reported_twice(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """The same validator registered twice runs twice, so both are reported rather than collapsed.""" + agent = pydantic_ai.Agent(model=_function_model(), name="test_agent") + + def reject_ungrounded(value): + return value + + agent.output_validator(reject_ungrounded) + agent.output_validator(reject_ungrounded) + + await agent.run("Hello, world!") + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + assert manifest["guardrails"] == ["reject_ungrounded", "reject_ungrounded"] + + async def test_data_contracts_carry_the_output_type(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """The declared output type lands under data_contracts.output by name.""" + + class Resolution(BaseModel): + answer: str + confidence: float + + _, manifest = await self._run( + pydantic_ai, test_spans, name="test_agent", output_type=Resolution, model=_test_model() + ) + + assert manifest["data_contracts"] == {"output": {"name": "Resolution"}} + + async def test_output_type_name_omits_the_defining_module(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A generic's arguments are named bare, so the value does not move with the import path. + + str() would report list[__main__.Resolution] as a script, list[app.models.Resolution] imported. + """ + + class Resolution(BaseModel): + answer: str + + _, manifest = await self._run( + pydantic_ai, test_spans, name="test_agent", output_type=list[Resolution], model=_test_model() + ) + + assert manifest["data_contracts"] == {"output": {"name": "list[Resolution]"}} + assert Resolution.__module__ not in manifest["data_contracts"]["output"]["name"] + + async def test_output_type_name_is_one_value_per_union_spelling(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """The three ways to declare the same union report one name, so they cannot fingerprint as three.""" + + class Answer(BaseModel): + answer: str + + class Refusal(BaseModel): + reason: str + + spellings = [Union[Answer, Refusal], [Answer, Refusal]] + if sys.version_info >= (3, 10): + # PEP 604 unions on classes raise TypeError below 3.10, and this suite pins 3.9. + spellings.append(Answer.__or__(Refusal)) + + for output_type in spellings: + _, manifest = await self._run( + pydantic_ai, test_spans, name="test_agent", output_type=output_type, model=_test_model() + ) + assert manifest["data_contracts"] == {"output": {"name": "Answer | Refusal"}} + + _, manifest = await self._run( + pydantic_ai, test_spans, name="test_agent", output_type=Optional[Answer], model=_test_model() + ) + assert manifest["data_contracts"] == {"output": {"name": "Answer | None"}} + + async def test_agent_settings_carry_the_loop_knobs(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """retries is the output-validation budget and tool_retries the per-tool one. + + Agent(retries=3, output_retries=2) reports retries 2 with tool_retries 3, not one number. + """ + _, manifest = await self._run( + pydantic_ai, test_spans, name="test_agent", retries=3, output_retries=2, end_strategy="exhaustive" + ) + + settings = manifest["agent_settings"] + assert settings["retries"] == 2 + assert settings["tool_retries"] == 3 + assert settings["end_strategy"] == "exhaustive" + + async def test_no_handoffs_for_pydantic_ai(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """pydantic-ai declares no handoff parameter, so the key is absent rather than synthesized. + + Delegation here is one agent called inside another's tool: code, not declared config. + """ + _, manifest = await self._run(pydantic_ai, test_spans, name="test_agent", tools=[calculate_square_tool]) + + assert "handoffs" not in manifest + + async def test_declared_string_model_is_read(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A deferred model check leaves model as a string; origin/main dropped it for these agents.""" + agent = pydantic_ai.Agent("openai:gpt-4o", name="test_agent", defer_model_check=True) + # The manifest must report the declared value, not whatever served this one call. + await agent.run("Hello, world!", model=_function_model()) + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + assert manifest["model"] == "gpt-4o" + + async def test_non_string_model_name_never_ships(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """model_name is annotated str, but a custom Model returns whatever it likes. + + The encoder reprs what it cannot encode, and a model object's repr is a plausible place for a + connection string, so this mirrors the str-only read on the tool description. + """ + model = _function_model() + hostile = property(lambda self: _UnserializableSentinel()) + + with mock.patch.object(type(model), "model_name", hostile): + agent = pydantic_ai.Agent(model=model, name="test_agent") + await agent.run("Hello, world!") + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + assert "model" not in manifest + assert "Omit()" not in safe_json(manifest) + + @pytest.mark.parametrize( + "declared,expected", + [ + ("openai:gpt-4o", "gpt-4o"), + # A bedrock or azure model name contains its own colon. Splitting on the last one reports + # the version suffix as the model, which is wrong data rather than missing data. + ("bedrock:anthropic.claude-v1:0", "anthropic.claude-v1:0"), + ("gpt-4o", "gpt-4o"), + ], + ) + async def test_declared_model_string_splits_on_the_first_colon( + self, pydantic_ai, pydantic_ai_llmobs, test_spans, declared, expected + ): + agent = pydantic_ai.Agent(declared, name="test_agent", defer_model_check=True) + await agent.run("Hello, world!", model=_function_model()) + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + assert manifest["model"] == expected + + async def test_non_string_system_prompts_never_ship(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """pydantic-ai does not validate system_prompt, so a non-string would ship as a leaking repr.""" + agent = pydantic_ai.Agent(model=_function_model(), name="test_agent", system_prompt="real prompt") + agent._system_prompts = ("real prompt", _UnserializableSentinel()) + # Built directly rather than through agent.run(). pydantic-ai itself cannot run an agent whose + # system prompts are not all strings, so the span path can never reach the builder with one. + # The guard is defense in depth: the attribute is public and unvalidated, so a framework change + # or a caller reaching in gets filtered rather than shipping a repr with a memory address. + # The integration instance is the one the patch installed, so no config has to be synthesized. + manifest = pydantic_ai._datadog_integration._build_agent_manifest(agent) + + assert manifest["system_prompts"] == ["real prompt"] + # The sentinel's own repr, not a memory-address pattern: a provider sentinel like NOT_GIVEN + # reprs as a bare name, so checking for "object at 0x" would pass without filtering anything. + assert "Omit()" not in safe_json(manifest) + + async def test_wire_values_are_json_native(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """Only JSON-native values reach the wire. + + The manifest travels in meta_struct, so an unencodable object fails the whole span. + """ + _, manifest = await self._run( + pydantic_ai, + test_spans, + name="test_agent", + model_settings={"temperature": 0.2, "logit_bias": {50256: -100}}, + ) + + json.dumps(manifest) + assert manifest["model_settings"]["logit_bias"] == {"50256": -100}, "an int key is coerced to str" + + async def test_non_finite_floats_never_ship(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """NaN and Infinity are valid Python floats and invalid JSON, so the key drops rather than nulls. + + An earlier version of this test passed while NaN was landing as an explicit null. + """ + _, manifest = await self._run( + pydantic_ai, + test_spans, + name="test_agent", + model_settings={"temperature": float("nan"), "top_p": 0.9}, + ) + + json.dumps(manifest, allow_nan=False) + assert manifest["model_settings"] == {"top_p": 0.9} + + def test_mcp_servers_are_named_but_never_addressed(self, pydantic_ai): + """MCP capture, which no other test reaches: the mcp extra is in none of the riot venvs. + + No URI is emitted, so a server address cannot carry a credential onto the wire. + """ + from ddtrace.llmobs._integrations.pydantic_ai import PydanticAIIntegration + + class FakeMCPServer: + def __init__(self, server_id=None, url=None): + self.id = server_id + self.url = url + + class NotAnMCPServer: + pass + + agent = mock.Mock() + agent._user_toolsets = [ + FakeMCPServer(server_id="billing", url="https://user:sk-secret@mcp.example.com:8443/sse?token=abc"), + FakeMCPServer(server_id=None, url=None), + NotAnMCPServer(), + ] + integration = PydanticAIIntegration(integration_config=mock.Mock()) + + with mock.patch.object(PydanticAIIntegration, "_mcp_server_classes", staticmethod(lambda: (FakeMCPServer,))): + names = integration._mcp_server_names(agent) + + assert names == ["billing", "FakeMCPServer"], "the non-MCP toolset is filtered out" + assert "sk-secret" not in safe_json(names) + + async def test_non_string_agent_name_falls_back_to_the_placeholder( + self, pydantic_ai, pydantic_ai_llmobs, test_spans + ): + """A name that is not a str must not be printed onto the wire, since the encoder reprs it.""" + + class Leaky: + def __repr__(self): + return "Leaky(token=sk-not-a-real-key)" + + agents = [pydantic_ai.Agent(model=_test_model(), name=Leaky())] + await agents[0].run("Hello, world!") + manifest = _manifest_of(test_spans.pop_traces()[0][0]) + + assert "sk-not-a-real-key" not in safe_json(manifest) + assert manifest["name"] == "PydanticAI Agent" + + async def test_non_string_tool_description_is_dropped(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A tool description that is not a str must not be printed onto the wire. + + pydantic-ai accepts Tool(fn, description=) and the encoder reprs what it cannot encode. + """ + + class SecretHolder: + def __init__(self): + self.api_key = "sk-not-a-real-key" + + def __repr__(self): + return "SecretHolder(api_key={!r})".format(self.api_key) + + def mytool(x: str) -> str: + """real docstring""" + return "ok" + + _, manifest = await self._run( + pydantic_ai, + test_spans, + name="test_agent", + tools=[pydantic_ai.Tool(mytool, description=SecretHolder())], + model=_test_model(), + ) + + assert "sk-not-a-real-key" not in safe_json(manifest) + assert "description" not in manifest["tools"][0] + + async def test_non_string_tool_parameter_key_is_coerced(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A non-str parameter key is coerced rather than left for the encoder. + + Tool.from_schema takes a caller json_schema, so a non-str key reaches the manifest. + """ + + def mytool(**kwargs) -> str: + """Takes whatever the declared schema names, since the model does call it.""" + return "ok" + + tool = pydantic_ai.Tool.from_schema( + mytool, + name="schema_tool", + description="d", + json_schema={"type": "object", "properties": {"alpha": {"type": "string"}, 7: {"type": "string"}}}, + ) + + _, manifest = await self._run(pydantic_ai, test_spans, name="test_agent", tools=[tool], model=_test_model()) + + parameters = manifest["tools"][0]["parameters"] + assert set(parameters) == {"alpha", "7"}, "both parameters survive, with keys coerced to str" + assert safe_json(manifest) is not None, "the payload must still encode" + + async def test_non_finite_agent_settings_never_ship(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """agent_settings does not go through the value coercer, so it needs its own guard. + + A bare Infinity token is not valid JSON, and spans ship batched, so one agent can invalidate + a whole payload. + """ + # output_retries, not tool_timeout: the hazard is reachable on every pin, tool_timeout is 1.63.0+. + _, manifest = await self._run( + pydantic_ai, test_spans, name="test_agent", output_retries=float("inf"), model=_test_model() + ) + + json.dumps(manifest, allow_nan=False) + assert "retries" not in manifest["agent_settings"], "a non-finite retry budget must drop" + + @pytest.mark.skipif(PYDANTIC_AI_VERSION < (1, 63, 0), reason="pydantic-ai < 1.63.0 has no agent metadata") + async def test_cyclic_metadata_does_not_cost_the_section(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A self-referential value terminates instead of recursing until the interpreter gives up.""" + cyclic: dict = {"team": "cx"} + cyclic["self"] = cyclic + + _, manifest = await self._run(pydantic_ai, test_spans, name="test_agent", metadata=cyclic) + + json.dumps(manifest) + assert manifest["name"] == "test_agent" + + async def test_callable_metadata_emits_no_metadata(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A metadata resolver is not captured, and is never called to find out what it returns. + + The negative is the point: building a manifest must not execute caller code. + """ + if PYDANTIC_AI_VERSION < (1, 39, 0): + pytest.skip("pydantic-ai < 1.39.0 does not accept a callable metadata") + + called = [] + + def tenant_metadata(ctx=None): + """Compute metadata for the current run.""" + called.append(True) + return {"tier": "gold"} + + agent = pydantic_ai.Agent(model=_test_model(), name="test_agent", metadata=tenant_metadata) + # Built directly rather than through a run: pydantic-ai resolves metadata itself during a run, + # so a run-scoped call counter cannot tell its calls apart from ours. + manifest = pydantic_ai._datadog_integration._build_agent_manifest(agent) + + assert "metadata" not in manifest + assert not called, "building the manifest must not evaluate a caller-supplied resolver" + + async def test_tool_preparation_is_recorded_as_a_capability(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """A prepare hook rewrites the tool list per step, so a change to it must move the manifest. + + It records that a transformation exists; tools still reports what was declared. + """ + + async def drop_destructive(ctx, defs): + """Withhold the destructive tool.""" + return [d for d in defs if d.name != "delete_account"] + + _, manifest = await self._run( + pydantic_ai, test_spans, name="test_agent", prepare_tools=drop_destructive, model=_test_model() + ) + + prep = [cap for cap in manifest["capabilities"] if cap.get("type") == "tool_preparation"] + assert prep == [{"name": "drop_destructive", "type": "tool_preparation"}] - coerced = load_data_value(raw) - json.dumps(coerced) # must not raise - assert coerced["max_tokens"] == 100 - assert coerced["temperature"] == "Omit()" + async def test_section_failure_is_isolated(self, pydantic_ai, pydantic_ai_llmobs, test_spans): + """One section raising costs that section only, not the whole manifest.""" + from ddtrace.llmobs._integrations.pydantic_ai import PydanticAIIntegration + with mock.patch.object(PydanticAIIntegration, "_manifest_model", side_effect=ValueError("boom")): + _, manifest = await self._run(pydantic_ai, test_spans, name="test_agent", instructions="Stay terse.") -def test_model_settings_none_is_preserved(): - assert load_data_value(None) is None + assert "model" not in manifest + assert "model_settings" not in manifest + assert manifest["name"] == "test_agent" + assert manifest["instructions"] == "Stay terse." diff --git a/tests/contrib/pydantic_ai/utils.py b/tests/contrib/pydantic_ai/utils.py index 9629fc95479..da2f393c3ff 100644 --- a/tests/contrib/pydantic_ai/utils.py +++ b/tests/contrib/pydantic_ai/utils.py @@ -1,3 +1,9 @@ +import mock +import pytest + +from ddtrace.llmobs._utils import _get_llmobs_data_metastruct + + PYDANTIC_AI_TAGS = { "ml_app": "", "service": "tests.contrib.pydantic_ai", @@ -5,6 +11,14 @@ } +# pydantic-ai's own defaults for an agent that configures none of these, at the versions riotfile.py +# pins. They are framework defaults rather than caller choices, which is why they are asserted here +# once instead of being repeated per test. They are NOT a framework invariant: end_strategy defaults +# to "graceful" at 2.x, so adding a 2.x pin will fail here on purpose. +DEFAULT_DATA_CONTRACTS = {"output": {"name": "str"}} +DEFAULT_AGENT_SETTINGS = {"retries": 1, "tool_retries": 1, "end_strategy": "early"} + + def expected_calculate_square_tool(): return [ { @@ -16,29 +30,56 @@ def expected_calculate_square_tool(): def expected_foo_tool(): + # No parameters key: a tool that takes no arguments has nothing to say, and an empty dict is + # exactly what the manifest must not emit. return [ { "name": "foo_tool", "description": "Return foo string", - "parameters": {}, } ] -def expected_agent_metadata(instructions=None, system_prompt=None, model_settings=None, tools=None) -> dict: - return { - "_dd": { - "agent_manifest": { - "framework": "PydanticAI", - "name": "test_agent", - "model": "gpt-4o", - "model_settings": model_settings, - "instructions": instructions, - "system_prompts": (system_prompt,) if system_prompt else (), - "tools": tools if tools is not None else [], - } - } - } +def expected_agent_manifest( + name="test_agent", + model="gpt-4o", + model_provider=None, + instructions=None, + system_prompt=None, + model_params=None, + tools=None, + **extra_fields, +) -> dict: + """Build the agent manifest a test expects. + + One flat document. A field with no value does not appear at all, which is what omit-when-absent + means on the wire, so a test that omits an argument is asserting the key is absent. + + model_provider is accepted and ignored: pydantic-ai has never emitted it. + """ + manifest = {"framework": "PydanticAI"} + if name: + manifest["name"] = name + if instructions: + manifest["instructions"] = instructions + if system_prompt: + manifest["system_prompts"] = [system_prompt] + if model: + manifest["model"] = model + if model_params: + # Reported under pydantic-ai's own spelling: the integration allowlists keys but does not + # rename them, so what a test configures is what it expects on the wire. + manifest["model_settings"] = dict(model_params) + if tools: + manifest["tools"] = tools + manifest["data_contracts"] = {"output": dict(DEFAULT_DATA_CONTRACTS["output"])} + manifest["agent_settings"] = dict(DEFAULT_AGENT_SETTINGS) + manifest.update(extra_fields) + return manifest + + +def expected_agent_metadata(**kwargs) -> dict: + return {"_dd": {"agent_manifest": expected_agent_manifest(**kwargs)}} def calculate_square_tool(x: int) -> int: @@ -49,3 +90,206 @@ def calculate_square_tool(x: int) -> int: def foo_tool() -> str: """Return foo string""" return "foo" + + +class _UnserializableSentinel: + """Stand-in for provider sentinels such as OpenAI's ``Omit`` / ``NOT_GIVEN``.""" + + def __repr__(self): + return "Omit()" + + +def _test_model(): + """A model that synthesises schema-valid output, needed when output_type is a function.""" + from pydantic_ai.models.test import TestModel + + return TestModel() + + +def _function_model(): + """A model that answers locally, so a manifest test needs no cassette and no network.""" + from pydantic_ai.messages import ModelResponse + from pydantic_ai.messages import TextPart + from pydantic_ai.models.function import FunctionModel + + def model_func(messages, info): + return ModelResponse(parts=[TextPart(content="Hello!")]) + + return FunctionModel(model_func) + + +def _manifest_of(span): + return _get_llmobs_data_metastruct(span)["meta"]["metadata"]["_dd"]["agent_manifest"] + + +class ABSENT: + """Marks keys a case asserts are missing. An empty expected dict asserts nothing at all.""" + + def __init__(self, *keys): + self.keys = keys + + +def _assert_contains(manifest, expected, path=""): + """Assert every field in expected matches, ignoring anything not mentioned. + + Lets a case skip builtin_tools' name, which is "WebSearchTool" below 1.63.0, "web_search" after. + """ + for key, want in expected.items(): + assert key in manifest, "manifest is missing {}{}".format(path, key) + got = manifest[key] + if isinstance(want, dict) and isinstance(got, dict): + _assert_contains(got, want, "{}{}.".format(path, key)) + elif isinstance(want, list) and isinstance(got, list): + assert len(got) == len(want), "{}{}: expected {} entries, got {}".format(path, key, len(want), len(got)) + for index, (want_entry, got_entry) in enumerate(zip(want, got)): + if isinstance(want_entry, dict) and isinstance(got_entry, dict): + _assert_contains(got_entry, want_entry, "{}{}[{}].".format(path, key, index)) + else: + assert got_entry == want_entry, "{}{}[{}]".format(path, key, index) + else: + assert got == want, "{}{}: expected {!r}, got {!r}".format(path, key, want, got) + + +# What must never reach the wire, one case per carrier: (kwargs factory, forbidden substrings, +# manifest subset that must still be present, minimum pydantic-ai version). Collected in one table so +# the security contract is reviewable in a single place. +MANIFEST_LEAK_CASES = [ + pytest.param( + lambda: dict( + model_settings={ + "temperature": 0.5, + "extra_headers": {"Authorization": "Bearer sk-leak-canary"}, + "extra_body": {"credential": "sk-leak-canary-2"}, + "openai_user": "end-user-4711", + } + ), + ["sk-leak-canary", "Authorization", "extra_headers", "extra_body", "end-user-4711"], + {"model_settings": {"temperature": 0.5}}, + None, + id="transport_params_never_ship", + ), + pytest.param( + lambda: dict( + model_settings={ + "temperature": 0.5, + "anthropic_metadata": {"user_id": "user-42-pii"}, + "bedrock_request_metadata": {"trace_token": "sk-leak-canary"}, + "openai_user": "end-user-99", + } + ), + ["sk-leak-canary", "user-42-pii", "end-user-99"], + {"model_settings": {"temperature": 0.5}}, + None, + id="provider_blobs_never_ship", + ), + pytest.param( + # The shared schema has no field for validation_context, so it drops entirely rather than + # shipping key names. It accepts Any, and a dict there routinely holds a live client or a key. + lambda: dict(validation_context={"tenant": "acme", "api_key": "sk-leak-canary"}), + ["sk-leak-canary", "validation_context"], + {}, + (1, 63, 0), + id="validation_context_never_ships", + ), +] + + +class _Deps: + tenant: str + + +def _redact_history(messages): + """Strip personal data from history.""" + return messages + + +def _tenant_toolset(ctx): + """Load the tenant's toolset.""" + return None + + +def _escalate(reason: str) -> str: + """Hand the ticket to a human.""" + return reason + + +def _builtin_web_search(): + from pydantic_ai.builtin_tools import WebSearchTool + + return WebSearchTool(search_context_size="high", max_uses=3) + + +# One field mapping per case: (kwargs factory, expected manifest subset, minimum pydantic-ai version). +# A factory rather than a literal so version-gated imports happen only when the case runs. +MANIFEST_FIELD_CASES = [ + pytest.param( + lambda: dict(model_settings={"temperature": 0, "parallel_tool_calls": False, "max_tokens": 0}), + # Falsy is not absent. Filtering on truthiness is what loses a deliberate temperature of 0. + {"model_settings": {"temperature": 0, "parallel_tool_calls": False, "max_tokens": 0}}, + None, + id="falsy_model_params_survive", + ), + pytest.param( + lambda: dict(model_settings={"temperature": 0.5, "stop_sequences": ["END"], "timeout": 30.0}), + {"model_settings": {"temperature": 0.5, "stop_sequences": ["END"], "timeout": 30.0}}, + None, + id="allowlisted_params_pass_through_unrenamed", + ), + pytest.param( + # A provider-prefixed key is not on the allowlist, so it drops rather than being promoted. + lambda: dict(model_settings={"openai_reasoning_effort": "high", "temperature": 0.5}), + {"model_settings": {"temperature": 0.5}}, + None, + id="provider_prefixed_param_drops", + ), + pytest.param( + lambda: dict(history_processors=[_redact_history]), + {"memory_policies": ["_redact_history"]}, + None, + id="history_processors_land_in_memory_policies", + ), + pytest.param( + # A processor listed twice runs twice, so it is reported twice: collapsing the repeat would + # describe a pipeline the agent does not run, the same way reordering it would. + lambda: dict(history_processors=[_redact_history, _redact_history]), + {"memory_policies": ["_redact_history", "_redact_history"]}, + None, + id="repeated_history_processor_is_reported_twice", + ), + pytest.param( + lambda: dict(toolsets=[_tenant_toolset]), + {"capabilities": [{"name": "_tenant_toolset", "type": "custom"}]}, + (0, 4, 4), + id="dynamic_toolset_is_a_custom_capability", + ), + pytest.param( + lambda: dict(builtin_tools=[_builtin_web_search()]), + {"capabilities": [{"name": mock.ANY, "type": "builtin"}]}, + None, + id="builtin_tool_is_a_capability", + ), + pytest.param( + lambda: dict(tool_timeout=12.5, max_concurrency=4), + {"agent_settings": {"tool_timeout": 12.5, "max_concurrency": 4}}, + (1, 63, 0), + id="tool_timeout_and_max_concurrency", + ), + pytest.param( + lambda: dict(metadata={"suite": "manifest", "owner": "llmobs"}), + {"metadata": {"suite": "manifest", "owner": "llmobs"}}, + (1, 63, 0), + id="metadata_is_top_level", + ), + pytest.param( + lambda: dict(deps_type=_Deps, end_strategy="exhaustive"), + {"agent_settings": {"deps_type": "_Deps", "end_strategy": "exhaustive"}}, + None, + id="deps_type_and_end_strategy_land_in_agent_settings", + ), + pytest.param( + lambda: dict(model=_test_model(), output_type=[_escalate]), + ABSENT("handoffs", "data_contracts"), + None, + id="output_function_does_not_become_a_handoff", + ), +] diff --git a/tests/internal/test_ci_allocation.py b/tests/internal/test_ci_allocation.py new file mode 100644 index 00000000000..44dbabfb03d --- /dev/null +++ b/tests/internal/test_ci_allocation.py @@ -0,0 +1,613 @@ +"""Tests for duration-aware CI allocation contracts.""" + +from dataclasses import replace +from datetime import datetime +from datetime import timedelta +from datetime import timezone +import hashlib +import json +from pathlib import Path + +import pytest + +from scripts.ci_allocation.history import Observation +from scripts.ci_allocation.history import build_runtime_model +from scripts.ci_allocation.history import live_shadow_report +from scripts.ci_allocation.history import observation_from_datadog +from scripts.ci_allocation.history import ratchet_violations +from scripts.ci_allocation.history import replay_observations +from scripts.ci_allocation.history import runtime_estimates +from scripts.ci_allocation.jobs import JobObservation +from scripts.ci_allocation.jobs import job_from_datadog +from scripts.ci_allocation.junit import verify_junit_parity +from scripts.ci_allocation.manifest import build_allocation_manifest +from scripts.ci_allocation.manifest import verify_allocation_manifest +from scripts.ci_allocation.manifest import write_manifest +from scripts.ci_allocation.planner import AllocationError +from scripts.ci_allocation.planner import expand_runtime_units +from scripts.ci_allocation.planner import legacy_round_robin +from scripts.ci_allocation.planner import predicted_makespan +from scripts.ci_allocation.planner import verify_assignments +from scripts.ci_allocation.planner import verify_runtime_assignments +from scripts.ci_allocation.planner import weighted_lpt +from scripts.ci_allocation.planner import weighted_runtime_lpt +from scripts.ci_allocation.runtime import build_runtime_inventory +from scripts.ci_allocation.runtime import verify_runtime_inventories +from scripts.ci_allocation.runtime import write_runtime_inventory +from scripts.ci_allocation.suites import SuiteVenvInfo +from scripts.ci_allocation.suites import compute_runtime_parallelism + + +def test_legacy_round_robin_preserves_current_assignment(): + assert legacy_round_robin(["f", "a", "e", "c", "b", "d"], 3) == [ + ["a", "d"], + ["b", "e"], + ["c", "f"], + ] + + +def test_weighted_lpt_is_exact_and_reduces_the_predicted_long_pole(): + hashes = ["h1", "h2", "h3", "h4"] + estimates = {"h1": 10, "h2": 9, "h3": 8, "h4": 1} + legacy = legacy_round_robin(hashes, 2) + balanced = weighted_lpt(hashes, 2, estimates, 60) + + verify_assignments(hashes, balanced) + assert predicted_makespan(balanced, estimates, 60) < predicted_makespan(legacy, estimates, 60) + + +def test_assignment_verifier_fails_on_overlap(): + with pytest.raises(AllocationError, match="more than one shard"): + verify_assignments(["a", "b"], [["a"], ["a", "b"]]) + + +def test_runtime_slices_break_one_measured_hash_across_distinct_jobs(): + units, weights = expand_runtime_units( + ["abc"], + {"abc": 650}, + 60, + target_shard_seconds=300, + setup_seconds=50, + test_item_counts={"abc": 100}, + maximum_slices_per_hash=5, + ) + + assignments = weighted_runtime_lpt(units, 3, weights) + + assert units == ["abc@1/3", "abc@2/3", "abc@3/3"] + assert assignments == [["abc@1/3"], ["abc@2/3"], ["abc@3/3"]] + verify_runtime_assignments(["abc"], assignments) + assert predicted_makespan(assignments, weights, 60) < 300 + + +def test_runtime_slices_require_real_test_item_evidence(): + units, _weights = expand_runtime_units( + ["abc"], + {"abc": 650}, + 60, + target_shard_seconds=300, + setup_seconds=50, + maximum_slices_per_hash=5, + ) + + assert units == ["abc"] + + +def test_runtime_parallelism_targets_modeled_shard_duration(): + suites = {"core": SuiteVenvInfo(("a", "b", "c", "d"), frozenset({"3.13"}))} + + result = compute_runtime_parallelism( + suites, + ["core"], + {"a": 100, "b": 100, "c": 100, "d": 100}, + {}, + 60, + target_shard_seconds=150, + maximum_parallelism_per_suite=2, + ) + + assert result == {"core": 2} + + +def test_runtime_parallelism_reallocates_within_legacy_job_budget(): + suites = { + "long": SuiteVenvInfo(("a", "b", "c", "d"), frozenset({"3.13"})), + "short": SuiteVenvInfo(("e", "f", "g", "h"), frozenset({"3.13"})), + } + + result = compute_runtime_parallelism( + suites, + suites, + {"a": 100, "b": 100, "c": 100, "d": 100, "e": 40, "f": 40, "g": 40, "h": 40}, + {}, + 60, + target_shard_seconds=150, + maximum_parallelism_per_suite=4, + maximum_total_jobs=4, + ) + + assert result == {"long": 2, "short": 2} + + +def test_runtime_parallelism_can_allocate_more_jobs_than_hashes_for_measured_tests(): + suites = {"core": SuiteVenvInfo(("a",), frozenset({"3.13"}))} + + result = compute_runtime_parallelism( + suites, + ["core"], + {"a": 650}, + {}, + 60, + target_shard_seconds=300, + maximum_parallelism_per_suite=5, + suite_overheads={"core": 50}, + test_item_counts_by_suite={"core": {"a": 100}}, + maximum_slices_per_hash=5, + ) + + assert result == {"core": 3} + + +def test_datadog_session_normalization_uses_riot_hash_as_atomic_identity(): + event = { + "attributes": { + "attributes": { + "duration": 12_500_000_000, + "start": 1_800_000_000_000_000_000, + "test": {"status": "pass", "configuration": {"riot_hash": "abc123"}}, + "ci": { + "pipeline": {"id": "pipeline-1"}, + "stage": {"name": "appsec"}, + "job": {"name": "appsec/threats 4/6"}, + }, + "git": {"commit": {"sha": "deadbeef"}}, + } + } + } + + observation = observation_from_datadog(event) + + assert observation.riot_hash == "abc123" + assert observation.suite == "appsec::threats" + assert observation.shard_total == 6 + assert observation.duration_seconds == 12.5 + + +def test_datadog_job_normalization_captures_total_and_queue_time(): + event = { + "attributes": { + "attributes": { + "start": "2026-01-01T00:00:00Z", + "duration_seconds": 90, + "queue_seconds": 5, + "test": {"configuration": {"ci_allocation_strategy": "balanced"}}, + "ci": { + "status": "success", + "pipeline": {"id": "pipeline-1"}, + "stage": {"name": "core"}, + "job": {"id": "job-1", "name": "core/internal 2/4"}, + }, + } + } + } + + observation = job_from_datadog(event) + + assert observation.suite == "internal" + assert observation.shard_index == 2 + assert observation.duration_seconds == 90 + assert observation.queue_seconds == 5 + assert observation.strategy == "balanced" + + +def test_datadog_job_normalization_rejects_missing_queue_time(): + event = { + "attributes": { + "attributes": { + "start": "2026-01-01T00:00:00Z", + "duration_seconds": 90, + "ci": { + "status": "success", + "pipeline": {"id": "pipeline-1"}, + "stage": {"name": "core"}, + "job": {"id": "job-1", "name": "core/internal 2/4"}, + }, + } + } + } + + with pytest.raises(AllocationError, match="queue_seconds must be numeric"): + job_from_datadog(event) + + +def _observation( + riot_hash: str, + duration: float, + when: datetime, + pipeline: str, + *, + shard_total: int = 2, +) -> Observation: + return Observation( + riot_hash=riot_hash, + suite="core", + duration_seconds=duration, + timestamp=when.isoformat().replace("+00:00", "Z"), + status="pass", + pipeline_id=pipeline, + commit_sha=pipeline, + shard_index=1, + shard_total=shard_total, + job_name=f"core {pipeline}", + ) + + +def test_runtime_model_reserves_holdout_and_replays_observed_durations(): + start = datetime(2026, 1, 1, tzinfo=timezone.utc) + durations = {"h1": 10, "h2": 9, "h3": 8, "h4": 1} + observations = [] + for day in (0, 1, 2): + observations.extend( + _observation(riot_hash, duration, start + timedelta(days=day), f"train-{day}") + for riot_hash, duration in durations.items() + ) + for day in (25, 26): + observations.extend( + _observation(riot_hash, duration, start + timedelta(days=day), f"holdout-{day}") + for riot_hash, duration in durations.items() + ) + observations.append(replace(observations[-1], status="fail", riot_hash="failed-hash")) + jobs = [ + JobObservation( + pipeline_id=f"train-{day}", + job_id=f"job-{day}", + job_name=f"core train-{day}", + stage_name="core", + suite="core", + strategy="legacy", + shard_index=1, + shard_total=2, + duration_seconds=sum(durations.values()) + 4, + queue_seconds=2, + status="success", + timestamp=(start + timedelta(days=day)).isoformat().replace("+00:00", "Z"), + ) + for day in (0, 1, 2) + ] + + model = build_runtime_model( + observations, + { + "estimate_quantile": 0.9, + "half_life_days": 30, + "history_window_days": 90, + "holdout_days": 7, + "minimum_samples": 2, + "sparse_safety_factor": 1.25, + }, + jobs, + ) + report = replay_observations( + observations, + model, + target_shard_seconds=12, + maximum_parallelism_per_suite=3, + ) + + assert model["dataset"]["training_observations"] == 12 + assert model["dataset"]["holdout_observations"] == 8 + assert model["dataset"]["censored_observations"] == 1 + assert model["overheads"]["global_seconds"] == 0 + assert model["overheads"]["unit_global_seconds"] == 1 + assert model["overheads"]["matched_session_count"] == 12 + assert model["estimates"]["h1"] == 11 + assert report["pipeline_count"] == 2 + assert report["balanced"]["median_seconds"] < report["legacy"]["median_seconds"] + + +def test_runtime_model_rejects_session_only_timings(): + start = datetime(2026, 1, 1, tzinfo=timezone.utc) + observations = [ + _observation("h1", 10, start, "train"), + _observation("h1", 10, start + timedelta(days=2), "holdout"), + ] + + with pytest.raises(AllocationError, match="missing CI job timing"): + build_runtime_model( + observations, + { + "estimate_quantile": 0.9, + "half_life_days": 30, + "history_window_days": 90, + "holdout_days": 1, + "minimum_samples": 1, + "sparse_safety_factor": 1.25, + }, + [], + ) + + +def test_manifest_proves_both_topologies_cover_the_same_suite(): + model = { + "schema_version": 1, + "planner_version": "weighted-lpt-v1", + "dataset": {"source": "test"}, + "parameters": {}, + "overheads": { + "global_seconds": 0, + "suite_seconds": {}, + "unit_global_seconds": 1, + "unit_suite_seconds": {}, + "sample_count": 1, + "matched_session_count": 2, + }, + "fallbacks": {"global_seconds": 60, "suite_seconds": {}}, + "estimates": {"a": 10, "b": 5}, + } + manifest = build_allocation_manifest( + suite_venv_info={"core": SuiteVenvInfo(("a", "b"), frozenset({"3.13"}))}, + suite_configs={"core": {"pattern": "core"}}, + legacy_shard_counts={"core": 1}, + balanced_shard_counts={"core": 2}, + runtime_model=model, + active_strategy="legacy", + ) + + verify_allocation_manifest(manifest) + assert manifest["suites"][0]["legacy"]["shard_count"] == 1 + assert manifest["suites"][0]["balanced"]["shard_count"] == 2 + + tampered = json.loads(json.dumps(manifest)) + tampered["suites"][0]["balanced"]["assignments"][0].append("b") + with pytest.raises(AllocationError, match="more than one shard"): + verify_allocation_manifest(tampered) + + +def test_manifest_and_runtime_inventories_prove_exact_sub_hash_coverage(tmp_path): + command = "pytest tests/example" + fingerprint = hashlib.sha256(command.encode()).hexdigest() + model = { + "schema_version": 1, + "planner_version": "weighted-lpt-v1", + "dataset": {"source": "test"}, + "parameters": {}, + "overheads": { + "global_seconds": 0, + "suite_seconds": {}, + "unit_global_seconds": 50, + "unit_suite_seconds": {}, + "sample_count": 1, + "matched_session_count": 1, + }, + "fallbacks": {"global_seconds": 60, "suite_seconds": {}}, + "estimates": {"abc": 650}, + "test_sharding": {"command_fingerprints": {fingerprint: {"minimum_items": 6}}}, + } + manifest = build_allocation_manifest( + suite_venv_info={ + "core": SuiteVenvInfo( + ("abc",), + frozenset({"3.13"}), + commands={"abc": (command,)}, + python_version_by_hash={"abc": "3.13"}, + ) + }, + suite_configs={"core": {"pattern": "core"}}, + legacy_shard_counts={"core": 1}, + balanced_shard_counts={"core": 3}, + runtime_model=model, + active_strategy="legacy", + target_shard_seconds=300, + maximum_slices_per_hash=5, + ) + plan_path = tmp_path / "plan.json" + write_manifest(plan_path, manifest) + nodeids = [f"tests/example/test_values.py::test_value[{index}]" for index in range(6)] + paths: list[Path] = [] + for shard_index in range(1, 4): + inventory = build_runtime_inventory( + suite="core", + riot_hash="abc", + shard_index=shard_index, + shard_total=3, + collected_nodeids=nodeids, + ) + path = tmp_path / f"inventory-{shard_index}.json" + write_runtime_inventory(path, inventory) + paths.append(path) + + report = verify_runtime_inventories(paths, plan_path) + + assert report["runtime_slice_count"] == 3 + assert report["test_identity_count"] == 6 + assert report["exact_union"] is True + tampered = json.loads(paths[1].read_text()) + tampered["selected_nodeids"] = tampered["selected_nodeids"][:-1] + paths[1].write_text(json.dumps(tampered)) + with pytest.raises(AllocationError, match="union differs"): + verify_runtime_inventories(paths, plan_path) + + +def test_live_shadow_report_requires_exact_hash_parity_and_compares_actual_shards(): + when = datetime(2026, 1, 1, tzinfo=timezone.utc) + observations = [] + for strategy, durations in (("legacy", {"a": 10, "b": 9, "c": 8}), ("balanced", {"a": 10, "b": 9, "c": 8})): + for shard_index, (riot_hash, duration) in enumerate(durations.items(), 1): + observation = _observation(riot_hash, duration, when, "pipeline", shard_total=3) + observations.append( + Observation( + **{ + **observation.__dict__, + "shard_index": shard_index, + "strategy": strategy, + "job_name": f"core/core{'-allocation-shadow' if strategy == 'balanced' else ''} " + f"{shard_index}/3", + } + ) + ) + + jobs = [] + for strategy, durations in (("legacy", (18, 10, 8)), ("balanced", (10, 9, 8))): + for shard_index, duration in enumerate(durations, 1): + suffix = "-allocation-shadow" if strategy == "balanced" else "" + jobs.append( + JobObservation( + pipeline_id="pipeline", + job_id=f"{strategy}-{shard_index}", + job_name=f"core/core{suffix} {shard_index}/3", + stage_name="core", + suite="core", + strategy=strategy, + shard_index=shard_index, + shard_total=3, + duration_seconds=duration, + queue_seconds=1, + status="success", + timestamp=when.isoformat().replace("+00:00", "Z"), + ) + ) + + report = live_shadow_report(observations, jobs) + + assert report["pipeline_count"] == 1 + assert report["exact_hash_parity"] is True + assert report["timing_source"] == "ci-jobs" + assert report["balanced"]["median_seconds"] < report["legacy"]["median_seconds"] + assert report["legacy"]["median_seconds"] == 18 + assert report["balanced"]["median_seconds"] == 10 + + unresolved_jobs = [replace(job, strategy="unknown") for job in jobs] + assert live_shadow_report(observations, unresolved_jobs)["timing_source"] == "ci-jobs" + + with pytest.raises(AllocationError, match="CI job timings are missing shards"): + live_shadow_report(observations, jobs[:-1]) + + observations[-1] = Observation(**{**observations[-1].__dict__, "riot_hash": "different"}) + with pytest.raises(AllocationError, match="hash parity failed"): + live_shadow_report(observations) + + +def test_junit_parity_compares_test_multisets_and_execution_metadata(tmp_path): + template = """\ + + + + + + +""" + legacy = tmp_path / "junit.legacy.abc.100.xml" + balanced = tmp_path / "junit.balanced.abc.200.xml" + legacy.write_text(template.format(strategy="legacy")) + balanced.write_text(template.format(strategy="balanced")) + + report = verify_junit_parity([legacy], [balanced]) + + assert report["test_identity_count"] == 1 + assert report["execution_metadata_parity"] is True + + legacy.write_text(template.format(strategy="legacy").replace('', "")) + balanced.write_text(template.format(strategy="balanced").replace('', "")) + assert verify_junit_parity([legacy], [balanced])["riot_hash_count"] == 1 + + without_metadata = template.format(strategy="{strategy}").replace( + '', "" + ) + legacy.write_text(without_metadata.format(strategy="legacy")) + balanced.write_text(without_metadata.format(strategy="balanced")) + with pytest.raises(AllocationError, match="no Riot execution metadata evidence"): + verify_junit_parity([legacy], [balanced]) + + digest = "a" * 64 + fallback_legacy = tmp_path / f"junit.legacy.abc.{digest}.300.xml" + fallback_balanced = tmp_path / f"junit.balanced.abc.{digest}.400.xml" + fallback_legacy.write_text(without_metadata.format(strategy="legacy")) + fallback_balanced.write_text(without_metadata.format(strategy="balanced")) + assert verify_junit_parity([fallback_legacy], [fallback_balanced])["execution_metadata_parity"] is True + + legacy.write_text(template.format(strategy="legacy")) + balanced.write_text(template.format(strategy="balanced").replace("test_value", "test_other")) + with pytest.raises(AllocationError, match="test identity parity failed"): + verify_junit_parity([legacy], [balanced]) + + +def test_junit_parity_combines_runtime_slices_into_the_legacy_multiset(tmp_path): + def xml(strategy, cases, shard_index=None): + partition = "" + if shard_index is not None: + partition = ( + f'' + '' + ) + testcases = "".join( + f'' for case in cases + ) + return ( + '' + '' + f'' + '' + f"{partition}{testcases}" + ) + + legacy = tmp_path / "junit.legacy.abc.100.xml" + balanced_1 = tmp_path / "junit.balanced.abc.s1of2.200.xml" + balanced_2 = tmp_path / "junit.balanced.abc.s2of2.300.xml" + legacy.write_text(xml("legacy", ["test_one", "test_two"])) + balanced_1.write_text(xml("balanced", ["test_one"], 1)) + balanced_2.write_text(xml("balanced", ["test_two"], 2)) + + report = verify_junit_parity([legacy], [balanced_1, balanced_2]) + + assert report["test_identity_count"] == 2 + assert report["exact_multiset_parity"] is True + + +def test_suite_scoped_runtime_estimates_override_shared_riot_hash(): + model = { + "schema_version": 1, + "planner_version": "weighted-lpt-v1", + "dataset": {}, + "parameters": {}, + "overheads": { + "global_seconds": 0, + "unit_global_seconds": 0, + "sample_count": 1, + "matched_session_count": 1, + }, + "fallbacks": {"global_seconds": 60}, + "estimates": {"abc": 600}, + "suite_estimates": {"short": {"abc": 10}}, + } + + assert runtime_estimates(model, "long")[0]["abc"] == 600 + assert runtime_estimates(model, "short")[0]["abc"] == 10 + + +def test_live_ratchet_requires_real_job_timing_and_exact_hash_parity(): + report = { + "kind": "live-shadow-replay", + "pipeline_count": 15, + "legacy": {"median_seconds": 100, "p75_seconds": 120, "p90_seconds": 130}, + "balanced": {"median_seconds": 80, "p75_seconds": 100, "p90_seconds": 130}, + "median_improvement_ratio": 0.2, + "runner_seconds_change_ratio": 0.01, + "clean_success_rate_change": 0, + "retry_rate_change": 0, + "exact_hash_parity": True, + "timing_source": "ci-jobs", + "queue": {"legacy": {"p90_seconds": 5}, "balanced": {"p90_seconds": 5}}, + } + policy = { + "live_shadow": { + "minimum_runs": 15, + "minimum_median_improvement_ratio": 0.15, + "maximum_runner_seconds_increase_ratio": 0.05, + "maximum_queue_p90_increase_ratio": 0.05, + } + } + + assert ratchet_violations(report, policy) == [] + + report["exact_hash_parity"] = False + assert "live shadow Riot hash parity was not proven" in ratchet_violations(report, policy) diff --git a/tests/llmobs/test_integrations_utils.py b/tests/llmobs/test_integrations_utils.py index 696650e0055..393917fb5d2 100644 --- a/tests/llmobs/test_integrations_utils.py +++ b/tests/llmobs/test_integrations_utils.py @@ -2,6 +2,10 @@ from types import SimpleNamespace from ddtrace.ext import SpanTypes +from ddtrace.llmobs._integrations.agent_manifest import MAX_WIRE_DEPTH +from ddtrace.llmobs._integrations.agent_manifest import is_number +from ddtrace.llmobs._integrations.agent_manifest import prune_empty +from ddtrace.llmobs._integrations.agent_manifest import wire_value from ddtrace.llmobs._integrations.audio_utils import audio_mime_type_from_format from ddtrace.llmobs._integrations.audio_utils import concat_base64_audio from ddtrace.llmobs._integrations.audio_utils import format_audio_part @@ -22,6 +26,7 @@ from ddtrace.llmobs._integrations.utils import openai_set_meta_tags_from_chat from ddtrace.llmobs._utils import _annotate_llmobs_span_data from ddtrace.llmobs._utils import get_llmobs_input_messages +from ddtrace.llmobs._utils import safe_json def test_format_audio_part_from_bytes(): @@ -672,3 +677,89 @@ def test_none_then_value_arguments_accumulate(self): stored, tool_call_chunk=SimpleNamespace(index=0, id=None, type=None, function=later, custom=None) ) assert stored[0]["function"]["arguments"] == '{"city": "NYC"}' + + +class TestAgentManifestPrimitives: + """The coercion every integration's manifest needs on the way out. + + These exist because an unencodable value does not fail politely: the span encoder reprs it, and a + bare NaN or Infinity token is not valid JSON. Spans ship batched, so one bad value discards every + span batched with it. + """ + + def test_prune_empty_drops_what_means_not_configured(self): + """Sections assign unconditionally so mypy can check key names; this is what drops the blanks.""" + assert prune_empty( + { + "framework": "PydanticAI", + "instructions": "", + "system_prompts": [], + "capabilities": [], + "metadata": {}, + } + ) == {"framework": "PydanticAI"} + + def test_prune_empty_keeps_false_and_zero(self): + """A configured temperature of 0 is not an absent one, which truthiness filtering loses.""" + assert prune_empty({"temperature": 0, "parallel_tool_calls": False, "top_p": 0.0}) == { + "temperature": 0, + "parallel_tool_calls": False, + "top_p": 0.0, + } + + def test_prune_empty_is_depth_first(self): + """A container emptied by its own children has to drop too, or an empty husk ships.""" + assert prune_empty({"agent_settings": {"retries": None}, "tools": [{"name": "x", "description": ""}]}) == { + "tools": [{"name": "x"}] + } + + def test_is_number_rejects_bool_and_non_finite(self): + assert is_number(0) and is_number(1.5) and is_number(-3) + assert not is_number(True), "bool is an int subclass and would otherwise ship as true" + assert not is_number(float("nan")) + assert not is_number(float("inf")) + assert not is_number(float("-inf")) + assert not is_number("1") and not is_number(None) + assert is_number(10**400), "a huge int is finite, and converting it to float to check would raise" + + def test_wire_value_drops_non_finite_and_unencodable(self): + assert wire_value(float("nan")) is None + assert wire_value(float("inf")) is None + assert wire_value(object()) is None + assert wire_value({"good": 1, "bad": object()}) == {"good": 1} + assert wire_value([1, object()]) is None, "one unencodable element costs the list" + + def test_wire_value_coerces_keys_and_terminates(self): + assert wire_value({1: "a"}) == {"1": "a"} + cyclic = {"k": 1} + cyclic["self"] = cyclic + assert wire_value(cyclic) == {"k": 1} + deep = current = {} + for _ in range(MAX_WIRE_DEPTH + 10): + current["n"] = {} + current = current["n"] + wire_value(deep) + + def test_wire_value_bounds_shared_subtrees(self): + """Depth alone does not bound the work: shared children expand into a tree. + + Twenty dicts each referencing the same child twice is 2**20 nodes, which took seconds and + tens of megabytes before the node budget. Cycle detection cannot catch it, because a shared + child is a second visit rather than an ancestor. + """ + node = {"leaf": 1} + for _ in range(20): + node = {"a": node, "b": node} + + wired = wire_value(node) + + assert len(safe_json(wired)) < 200_000, "the node budget is what keeps this off the wire" + + def test_wire_value_keeps_a_shared_subtree_that_fits(self): + """A repeated child is legitimate, so the budget must not turn sharing into a drop.""" + child = {"region": "us1", "tier": "gold"} + + assert wire_value({"primary": child, "replica": child}) == { + "primary": {"region": "us1", "tier": "gold"}, + "replica": {"region": "us1", "tier": "gold"}, + } diff --git a/tests/profiling/test_gunicorn.py b/tests/profiling/test_gunicorn.py index b1f501358ea..5e6da48cae2 100644 --- a/tests/profiling/test_gunicorn.py +++ b/tests/profiling/test_gunicorn.py @@ -107,7 +107,10 @@ def _test_gunicorn( debug_print("Making request to gunicorn server") try: - with urllib.request.urlopen("http://127.0.0.1:7644", timeout=5) as f: + # The handler computes fib(35), which takes over 3s on py3.10 and has been measured at + # 5s on a loaded CI runner. This timeout only guards against a hung worker, so it needs + # margin over that rather than to be tight. + with urllib.request.urlopen("http://127.0.0.1:7644", timeout=20) as f: status_code = f.getcode() assert status_code == 200, status_code response = f.read().decode()