Automatic retry ASM integration tests under failure (#8011)

## Summary of changes

The following jobs have been found flaky, mostly due to a already
reported crash condition:

```
integration_tests_windows.Win x86_net9.0_ASM  7 failures / last month
integration_tests_linux.Test alpine_net7.0_ASM  5 failures / last month
integration_tests_linux.Test alpine_net9.0_ASM  5 failures / last month
integration_tests_linux.Test debian_net9.0_ASM  3 failures / last month
```

The complete flakiness report can be found here:

https://docs.google.com/spreadsheets/d/1Gftmhb-66Dag4qFEXw9tyXp7U0fOQsdCE2gI-1voWyA/edit?gid=1708590243#gid=1708590243

The affected jobs will automatically be retried once to avoid CI
flakiness.

For other teams, the flaky tests have been marked as flaky, which causes
automatic retry. In the case of ASM, the failure does not occur on
especific tests, so marking some of them would not solve the issue. This
change can be reverted once the jobs are more stable.

This PR is part of an initiative of marking the most flaky tests or jobs
of all the teams.

## Reason for change

## Implementation details

## Test coverage

## Other details
<!-- Fixes #{issue} -->


<!--  ⚠️ Note:

Where possible, please obtain 2 approvals prior to merging. Unless
CODEOWNERS specifies otherwise, for external teams it is typically best
to have one review from a team member, and one review from apm-dotnet.
Trivial changes do not require 2 reviews.

MergeQueue is NOT enabled in this repository. If you have write access
to the repo, the PR has 1-2 approvals (see above), and all of the
required checks have passed, you can use the Squash and Merge button to
merge the PR. If you don't have write access, or you need help, reach
out in the #apm-dotnet channel in Slack.
-->

Author: NachoEchevarria

.azure-pipelines/ultimate-pipeline.yml

@@ -1710,12 +1710,25 @@ stages:
         SampleName: $(IntegrationTestSampleName)
 
     - script: tracer\build.cmd RunIntegrationTests RunWindowsRegressionTests -Framework $(framework) --code-coverage-enabled $(CodeCoverageEnabled)
-      displayName: Run integration tests
+      displayName: Run integration tests (Tracer)
+      condition: ne(variables['area'], 'ASM')
+      env:
+        DD_LOGGER_DD_API_KEY: $(ddApiKey)
+        enable_crash_dumps: true
+        Filter: $(IntegrationTestFilter)
+        SampleName: $(IntegrationTestSampleName)
+        Area: $(area)
+
+    - script: tracer\build.cmd RunIntegrationTests RunWindowsRegressionTests -Framework $(framework) --code-coverage-enabled $(CodeCoverageEnabled)
+      displayName: Run integration tests (ASM)
+      condition: eq(variables['area'], 'ASM')
+      retryCountOnTaskFailure: 2
       env:
         DD_LOGGER_DD_API_KEY: $(ddApiKey)
         enable_crash_dumps: true
         Filter: $(IntegrationTestFilter)
         SampleName: $(IntegrationTestSampleName)
+        Area: $(area)
 
 #    TODO: Re-enable test-agent.windows once VM images are updated (hitting rate limits for docker)
 #    - script: docker compose -f docker-compose.windows.yml down
@@ -2272,7 +2285,26 @@ stages:
           -e SampleName=$(IntegrationTestSampleName) \
           -e Area=$(area) \
           IntegrationTests
-      displayName: docker-compose run IntegrationTests
+      displayName: docker-compose run IntegrationTests (Tracer)
+      condition: ne(variables['area'], 'ASM')
+      env:
+        DD_LOGGER_DD_API_KEY: $(ddApiKey)
+        baseImage: $(baseImage) # for interpolation in the docker-compose file
+
+    - script: |
+        docker-compose -f docker-compose.yml -p $(DockerComposeProjectName) \
+          run --no-deps --rm \
+          -e baseImage=$(baseImage) \
+          -e framework=$(publishTargetFramework) \
+          -e CodeCoverageEnabled=$(CodeCoverageEnabled) \
+          -e IncludeTestsRequiringDocker=false  \
+          -e Filter=$(IntegrationTestFilter) \
+          -e SampleName=$(IntegrationTestSampleName) \
+          -e Area=$(area) \
+          IntegrationTests
+      displayName: docker-compose run IntegrationTests (ASM)
+      condition: eq(variables['area'], 'ASM')
+      retryCountOnTaskFailure: 2
       env:
         DD_LOGGER_DD_API_KEY: $(ddApiKey)
         baseImage: $(baseImage) # for interpolation in the docker-compose file

tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetWebApi.cs

@@ -87,7 +87,7 @@ public AspNetWebApi(IisFixture iisFixture, ITestOutputHelper output, bool classi
         public Task TestSecurity(string test, string url, string body)
         {
             // if blocking is enabled, request stops before reaching asp net mvc integrations intercepting before action methods, so no more spans are generated
-            // NOTE: by integrating the latest version of the WAF, blocking was disabled, as it does not support blocking yet
+            // NOTE: by integrating the latest version of the WAF, blocking was disabled, as it does not support blocking yet.
             var sanitisedUrl = VerifyHelper.SanitisePathsForVerify(url);
             var settings = VerifyHelper.GetSpanVerifierSettings(test, sanitisedUrl, body);
             FilterConnectionHeader(settings);