Add bazel coverage job for linux x86 (#54613)

<!--Please give us some feedback on your experience writing this PR ! https://app.datadoghq.com/forms/43db4c02-6837-400c-8083-692e141b1b88 !-->

### What does this PR do?

- Drop an old and unused hybrid test job that combined both legacy `go test` and `bazel test` executions
- Introduce a new coverage job for Linux x86 driven by Bazel.

This is one of the chunks that is needed to ensure parity in CI functionality between legacy and Bazel driven activities. With this change we can now collect and compare coverage reports between `tests_linux-x64-py3` and `bazel:coverage` to identify blindspots and address them.

### Motivation

This is one more step towards fully migrating unit testing to Bazel.



Co-authored-by: joseph.gette <joseph.gette@datadoghq.com>

Author: JSGette

.gitlab/build/bazel/test.yml

@@ -32,6 +32,22 @@
 .bazel:test:reporting:ci_links:
   - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
 
+bazel:coverage:linux-amd64:
+  extends: [.bazel:test:reporting, .bazel:runner:linux-amd64, .bazel:test]
+  script:
+    - !reference [.bazel:test:reporting:ci_links]
+    - bazel coverage --config=go --config=gorace --config=dd-agent-go-tests-only --jobs=$KUBERNETES_CPU_REQUEST --build_tests_only --keep_going --remote_download_outputs=toplevel --build_event_json_file=$BEP_FILE //...
+  after_script:
+    - !reference [.bazel:test:reporting, after_script]
+    - cp "$(bazel info output_path)/_coverage/_coverage_report.dat" "$CI_PROJECT_DIR/coverage-bazel-$CI_JOB_NAME_SLUG.out"
+  artifacts:
+    paths:
+      - junit-*.tgz
+      - bazel-bep-*.json
+      - coverage-bazel-*.out
+  variables:
+    KUBERNETES_CPU_REQUEST: 16
+
 bazel:test:linux-amd64:
   extends: [.bazel:test:reporting, .bazel:runner:linux-amd64, .bazel:test]
   script:

.gitlab/build/source_test/linux.yml

@@ -60,32 +60,6 @@ tests_linux-x64-py3:
   variables:
     CONDA_ENV: ddpy3
 
-tests_linux-x64-py3_hybrid:
-  extends:
-    - .bazel:runner:linux-arm64
-    - .linux_tests
-    - .linux_x64
-  after_script:
-    # In the hybrid test, bazel gets run, and that leaves a symlink to junit-out-base.xml. The uploader chokes on symlinks.
-    - find $(bazel info execution_root) -type l -name 'junit-*.xml' -delete
-    - !reference [.upload_junit_source]
-    # skip coverage upload for several reasons
-    # 1. It would conflate with coverate from the regular job
-    # 2. the bazel tests are not doing coverage yet
-    # 3. when bazel starts it makes the symlink .cache/bazel/…/execroot/_main/junit-out-AgentFlavor.base.xml
-    #    pointing to <top>/junit-out-AgentFlavor.base.xml. Then the coverage uploader fails on the symlink.
-    #    A possible fix for that is to change `datadog-ci coverage upload` so that it can gracefully
-    #    skip symlinks rather than choking. Or, we move XDG_CACHE out of the CI root.
-  variables:
-    CONDA_ENV: ddpy3
-    EXPERIMENTAL_USE_BAZEL_TESTS: "1"
-  allow_failure: true  # This is for measurement during migration. It is not the actual test job.
-  retry: 0  # allowed to fail, ~35 min exec time
-  rules:
-    - !reference [.except_mergequeue]  # allowed to fail, not a gate at this stage
-    - !reference [.except_disable_unit_tests]
-    - !reference [.fast_on_dev_branch_only]
-
 tests_flavor_iot_linux-x64:
   extends:
     - .linux_tests

bazel/configs/go_tests.bazelrc

@@ -3,6 +3,8 @@
 # last-wins rather than additive, so any config or command line that sets it has to
 # repeat this exclusion.
 test --test_tag_filters=-tagset_linux_bpf
+# To produce verbose test.xml files
+test --test_env=GO_TEST_WRAP_TESTV=1
 
 # Go race detector for test invocations. Usage: bazel test --config=gorace //...
 test:gorace --@rules_go//go/config:race