[CI] Add preferCached for docker-startup commands (elastic#244766)

## Summary
Add `preferCached` as an option in the test util for starting docker
images, turn it on for all EPR image pulls, in order to prevent costly
docker pulls in FTRs/integration tests.

After elastic#244517 we still have a small temporal zone in between an agent
being provisioned and the tests being ran, where a promotion would
trigger a docker pull, that depletes disk space, it also would make
pulls slow, and potentially time out tests.

This PR suggests, we should always use the cached variant (for EPR at
least). With this, the above risk zone is no longer causing pulls, it's
just that we are still testing the previously promoted and cached image,
probably not risky.

(cherry picked from commit 5336f0d)

Author: delanni

src/platform/packages/private/kbn-journeys/journey/journey_ftr_config.ts

@@ -87,6 +87,7 @@ export function makeFtrConfigProvider(
           args: dockerArgs,
           waitForLogLine: 'package manifests loaded',
           waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
+          preferCached: true,
         },
       }),
 

src/platform/packages/shared/kbn-scout/src/config/schema/schema.ts

@@ -53,6 +53,7 @@ const dockerServerSchema = () =>
       image: requiredWhenEnabled(Joi.string()),
       port: requiredWhenEnabled(Joi.number()),
       portInContainer: requiredWhenEnabled(Joi.number()),
+      preferCached: Joi.boolean().optional(),
       waitForLogLine: Joi.alternatives(Joi.object().instance(RegExp), Joi.string()).optional(),
       waitForLogLineTimeoutMs: Joi.number().integer().optional(),
       waitFor: Joi.func().optional(),

src/platform/packages/shared/kbn-scout/src/config/serverless/serverless.base.config.ts

@@ -63,6 +63,7 @@ export const defaultConfig: ScoutServerConfig = {
       args: dockerArgs,
       waitForLogLine: 'package manifests loaded',
       waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
+      preferCached: true,
     },
   }),
   esTestCluster: {

src/platform/packages/shared/kbn-scout/src/config/stateful/base.config.ts

@@ -67,7 +67,8 @@ export const defaultConfig: ScoutServerConfig = {
       port: dockerRegistryPort,
       args: dockerArgs,
       waitForLogLine: 'package manifests loaded',
-      waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
+      waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes,
+      preferCached: true,
     },
   }),
   esTestCluster: {

src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -71,6 +71,7 @@ const dockerServerSchema = () =>
       image: requiredWhenEnabled(Joi.string()),
       port: requiredWhenEnabled(Joi.number()),
       portInContainer: requiredWhenEnabled(Joi.number()),
+      preferCached: Joi.boolean().optional(),
       waitForLogLine: Joi.alternatives(Joi.object().instance(RegExp), Joi.string()).optional(),
       waitForLogLineTimeoutMs: Joi.number().integer().optional(),
       waitFor: Joi.func().optional(),

src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/docker_servers/define_docker_servers_config.ts

@@ -14,6 +14,7 @@ export interface DockerServerSpec {
   portInContainer: number;
   port: number;
   image: string;
+  preferCached?: boolean;
   waitForLogLine?: RegExp | string;
   waitForLogLineTimeoutMs?: number;
   /** a function that should return an observable that will allow the tests to execute as soon as it emits anything */

src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/docker_servers/docker_servers_service.ts

@@ -105,8 +105,12 @@ export class DockerServersService {
     const { image, name, waitFor, waitForLogLine, waitForLogLineTimeoutMs } = server;
 
     // pull image from registry
-    log.info(`[docker:${name}] pulling docker image "${image}"`);
-    await execa('docker', ['pull', image]);
+    if (server.preferCached && (await this.isImageAvailableLocally(image))) {
+      log.info(`[docker:${name}] skipping pull of docker image "${image}"`);
+    } else {
+      log.info(`[docker:${name}] pulling docker image "${image}"`);
+      await execa('docker', ['pull', image]);
+    }
 
     // run the image that we just pulled
     const containerId = await this.dockerRun(server);
@@ -207,6 +211,15 @@ export class DockerServersService {
     ).toPromise();
   }
 
+  private async isImageAvailableLocally(imageUrl: string) {
+    try {
+      const { stdout } = await execa('docker', ['images', '-q', imageUrl]);
+      return stdout.trim().length > 0;
+    } catch {
+      return false;
+    }
+  }
+
   private async startServers() {
     await Promise.all(
       this.servers.map(async (server) => {

x-pack/platform/plugins/shared/fleet/server/integration_tests/helpers/docker_registry_helper.ts

@@ -18,7 +18,7 @@ import pRetry from 'p-retry';
 const BEFORE_SETUP_TIMEOUT = 30 * 60 * 1000; // 30 minutes;
 
 const DOCKER_START_TIMEOUT = 6 * 60 * 1000; // 6 minutes
-// This image comes from the latest successful build of https://buildkite.com/elastic/kibana-package-registry-promote
+// This image comes from the latest successful build of https://buildkite.com/elastic/kibana-package-registry-verify-and-promote
 // which is promoted after acceptance tests succeed against docker.elastic.co/package-registry/distribution:lite
 const DOCKER_IMAGE =
   process.env.FLEET_PACKAGE_REGISTRY_DOCKER_IMAGE ||

x-pack/platform/test/api_integration_deployment_agnostic/default_configs/feature_flag.serverless.config.base.ts

@@ -101,7 +101,8 @@ export function createServerlessFeatureFlagTestConfig<T extends DeploymentAgnost
           port: dockerRegistryPort,
           args: dockerArgs,
           waitForLogLine: 'package manifests loaded',
-          waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
+          waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes,
+          preferCached: true,
         },
       }),
       esTestCluster: {

x-pack/platform/test/api_integration_deployment_agnostic/default_configs/feature_flag.stateful.config.base.ts

@@ -97,7 +97,8 @@ export function createStatefulFeatureFlagTestConfig<T extends DeploymentAgnostic
           port: dockerRegistryPort,
           args: dockerArgs,
           waitForLogLine: 'package manifests loaded',
-          waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes
+          waitForLogLineTimeoutMs: 60 * 6 * 1000, // 6 minutes,
+          preferCached: true,
         },
       }),
       testFiles: options.testFiles,