test: Refactor docker-compose files in container tests (#3524)

* Initial refactoring

* Add the base file to the solution

* Remove unnecessary networking from smoketest compose file

Author: nr-ahemsath

…ontainerBuild/Dockerfile.AmazonBaseImage …ontainerBuild/Dockerfile.CustomBaseImagetests/Agent/IntegrationTests/ContainerApplications/CustomBaseContainerBuild/Dockerfile.AmazonBaseImage renamed to tests/Agent/IntegrationTests/ContainerApplications/CustomBaseContainerBuild/Dockerfile.CustomBaseImage tests/Agent/IntegrationTests/ContainerApplications/CustomBaseContainerBuild/Dockerfile.AmazonBaseImage renamed to tests/Agent/IntegrationTests/ContainerApplications/CustomBaseContainerBuild/Dockerfile.CustomBaseImage

@@ -1,4 +1,5 @@
-FROM amazonlinux:latest AS base
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE} AS base
 
 # configure dnf and upgrade all packages
 RUN echo "max_parallel_downloads=10" >> /etc/dnf/dnf.conf

tests/Agent/IntegrationTests/ContainerApplications/CustomBaseContainerBuild/Dockerfile.FedoraBaseImage

@@ -30,11 +30,11 @@ docker buildx create --use
 3. Build the base images. The images will be pushed to the container registry with tags per .NET version
 
 ```
-docker buildx  build --build-arg DOTNET_VERSION="8.0" --build-arg DOTNET_QUALITY="GA" -f Dockerfile.AmazonBaseImage --tag $acrName/amazonlinux-aspnet:8.0 --platform linux/amd64,linux/arm64/v8 --push .
-docker buildx  build --build-arg DOTNET_VERSION="10.0" --build-arg DOTNET_QUALITY="GA" -f Dockerfile.AmazonBaseImage --tag $acrName/amazonlinux-aspnet:10.0 --platform linux/amd64,linux/arm64/v8 --push .
+docker buildx  build --build-arg BASE_IMAGE="amazonlinux:latest" --build-arg DOTNET_VERSION="8.0" --build-arg DOTNET_QUALITY="GA" -f Dockerfile.CustomBaseImage --tag $acrName/amazonlinux-aspnet:8.0 --platform linux/amd64,linux/arm64/v8 --push .
+docker buildx  build --build-arg BASE_IMAGE="amazonlinux:latest" --build-arg DOTNET_VERSION="10.0" --build-arg DOTNET_QUALITY="GA" -f Dockerfile.CustomBaseImage --tag $acrName/amazonlinux-aspnet:10.0 --platform linux/amd64,linux/arm64/v8 --push .
 
-docker buildx  build --build-arg DOTNET_VERSION="8.0" --build-arg DOTNET_QUALITY="GA" -f Dockerfile.FedoraBaseImage --tag $acrName/fedora-aspnet:8.0 --platform linux/amd64,linux/arm64/v8 --push .
-docker buildx  build --build-arg DOTNET_VERSION="10.0" --build-arg DOTNET_QUALITY="GA" -f Dockerfile.FedoraBaseImage --tag $acrName/fedora-aspnet:10.0 --platform linux/amd64,linux/arm64/v8 --push .
+docker buildx  build --build-arg BASE_IMAGE="fedora:latest" --build-arg DOTNET_VERSION="8.0" --build-arg DOTNET_QUALITY="GA" -f Dockerfile.CustomBaseImage --tag $acrName/fedora-aspnet:8.0 --platform linux/amd64,linux/arm64/v8 --push .
+docker buildx  build --build-arg BASE_IMAGE="fedora:latest" --build-arg DOTNET_VERSION="10.0" --build-arg DOTNET_QUALITY="GA" -f Dockerfile.CustomBaseImage --tag $acrName/fedora-aspnet:10.0 --platform linux/amd64,linux/arm64/v8 --push .
 ```
 
 4. Disable buildx in Docker Desktop

tests/Agent/IntegrationTests/ContainerApplications/CustomBaseContainerBuild/README.md

@@ -1,81 +1,56 @@
 # The following must be set either in environment variables or via a .env file in the same folder as this file:
 #
-# AGENT_PATH      host path to the Agent linux home folder - will map to /usr/local/newrelic-dotnet-agent in the container
-# LOG_PATH        host path for Agent logfile output - will map to /app/logs in the container
-# DISTRO_TAG      distro tag for build, not including the architecture suffix
-# TARGET_ARCH     the target architecture for the run -- either amd64 or arm64
-# BUILD_ARCH      the build architecture-- either amd64 or arm64
-# PORT            external port for the smoketest API
-# CONTAINER_NAME  The name for the container
-# PLATFORM        The platform that the service runs on -- linux/amd64 or linux/arm64/v8
-# DOTNET_VERSION  The dotnet version number to use (8.0, etc)
-# NETWORK_NAME    The network name to use for containers in this app. Should be unique among all running instances.
-# TEST_DOCKERFILE The path and dockerfile to use for the service.
-# 
-# and the usual suspects:
-# NEW_RELIC_LICENSE_KEY
-# NEW_RELIC_HOST
-# NEW_RELIC_APP_NAME
+# See docker-compose-base.yml for common required variables.
+# CONTAINER_NAME       The name for the container
+# AWSSDK_INITCOLLECTIONS
+# LOCALSTACK_VOLUME_DIR
 #
-#
-# To build and run, execute `docker compose -f <path to docker-compose.yml> up` 
+# To build and run, execute `docker compose -f <path to docker-compose.yml> up`
 # Alternatively, set COMPOSE_FILE environment variable to the path and omit the -f parameter
 
 services:
-  localstack:
-    image: localstack/localstack:4.14.0@sha256:3ebc37595918b8accb852f8048fef2aff047d465167edd655528065b07bc364a
-    expose: # ports are only available internal to the service, not external so there's no chance for conflicts
-      - "4566"       # LocalStack Gateway
-      - "4559"  # external services port range
-    environment:
-      # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
-      - DEBUG=${DEBUG:-0}
-      # firehose depends on s3 and sts
-      - SERVICES=sqs,sns,ses,kinesis,firehose,s3,sts # http://localhost:4566/_localstack/health to see the entire list of possible services and their status
-    volumes:
-      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
-      - "/var/run/docker.sock:/var/run/docker.sock"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-      start_period: 10s
+    localstack:
+        image: localstack/localstack:4.14.0@sha256:3ebc37595918b8accb852f8048fef2aff047d465167edd655528065b07bc364a
+        expose: # ports are only available internal to the service, not external so there's no chance for conflicts
+            - "4566"      # LocalStack Gateway
+            - "4559" # external services port range
+        environment:
+            # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
+            - DEBUG=${DEBUG:-0}
+            # firehose depends on s3 and sts
+            - SERVICES=sqs,sns,ses,kinesis,firehose,s3,sts # http://localhost:4566/_localstack/health to see the entire list of possible services and their status
+        volumes:
+            - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+            - "/var/run/docker.sock:/var/run/docker.sock"
+        healthcheck:
+            test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
+            interval: 5s
+            timeout: 5s
+            retries: 10
+            start_period: 10s
 
-  dynamodb:
-    command: "-jar DynamoDBLocal.jar -inMemory"
-    image: "amazon/dynamodb-local:latest"
-    expose: # ports are only available internal to the service, not external so there's no chance for conflicts
-      - "8000"
-    working_dir: /home/dynamodblocal
+    dynamodb:
+        command: "-jar DynamoDBLocal.jar -inMemory"
+        image: "amazon/dynamodb-local:latest"
+        expose: # ports are only available internal to the service, not external so there's no chance for conflicts
+            - "8000"
+        working_dir: /home/dynamodblocal
 
-  awssdktestapp:
-    depends_on:
-      localstack:
-        condition: service_healthy
-      dynamodb:
-        condition: service_started
-    container_name: ${CONTAINER_NAME}
-    image: ${CONTAINER_NAME}
-    platform: ${PLATFORM}
-    build:
-      context: .
-      dockerfile: ${TEST_DOCKERFILE}
-      args:
-        DISTRO_TAG: ${DISTRO_TAG}
-        TARGET_ARCH: ${TARGET_ARCH}
-        BUILD_ARCH: ${BUILD_ARCH}
-        NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY}
-        NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME}
-        NEW_RELIC_HOST: ${NEW_RELIC_HOST}
-        DOTNET_VERSION: ${DOTNET_VERSION}
-        APP_DOTNET_VERSION: ${APP_DOTNET_VERSION}
-        AWSSDK_INITCOLLECTIONS: ${AWSSDK_INITCOLLECTIONS}
-    ports:
-      - "${PORT}:80"
-    volumes:
-      - ${AGENT_PATH}:/usr/local/newrelic-dotnet-agent # AGENT_PATH from .env, points to newrelichome_linux_x64
-      - ${LOG_PATH}:/app/logs # LOG_PATH from .env, should be a folder unique to this run of the smoketest app
+    awssdktestapp:
+        extends:
+            file: docker-compose-base.yml
+            service: base-app
+        depends_on:
+            localstack:
+                condition: service_healthy
+            dynamodb:
+                condition: service_started
+        container_name: ${CONTAINER_NAME}
+        image: ${CONTAINER_NAME}
+        platform: ${PLATFORM}
+        build:
+            args:
+                AWSSDK_INITCOLLECTIONS: ${AWSSDK_INITCOLLECTIONS}
 
 networks:
     default:

tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml

@@ -0,0 +1,41 @@
+# Base service definition for New Relic .NET Agent container integration tests.
+# Individual docker-compose files extend this service via the 'extends' keyword.
+# See https://docs.docker.com/compose/how-tos/multiple-compose-files/extends/
+#
+# Common required variables (set in environment or .env file):
+#
+# AGENT_PATH      host path to the Agent linux home folder - will map to /usr/local/newrelic-dotnet-agent in the container
+# LOG_PATH        host path for Agent logfile output - will map to /app/logs in the container
+# DISTRO_TAG      distro tag for build, not including the architecture suffix
+# TARGET_ARCH     the target architecture for the run -- either amd64 or arm64
+# BUILD_ARCH      the build architecture -- either amd64 or arm64
+# PORT            external port for the test API
+# PLATFORM        The platform that the service runs on -- linux/amd64 or linux/arm64/v8
+# DOTNET_VERSION  The dotnet version number to use (8.0, etc)
+# APP_DOTNET_VERSION  The dotnet version number to use for the app (8.0, etc)
+# TEST_DOCKERFILE The path and dockerfile to use for the service
+# CONTAINER_TEST_ACR_NAME  The name of the Azure Container Registry to use for custom base images
+# NEW_RELIC_LICENSE_KEY
+# NEW_RELIC_HOST
+# NEW_RELIC_APP_NAME
+
+services:
+    base-app:
+        build:
+            context: .
+            dockerfile: ${TEST_DOCKERFILE}
+            args:
+                DISTRO_TAG: ${DISTRO_TAG}
+                TARGET_ARCH: ${TARGET_ARCH}
+                BUILD_ARCH: ${BUILD_ARCH}
+                NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY}
+                NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME}
+                NEW_RELIC_HOST: ${NEW_RELIC_HOST}
+                DOTNET_VERSION: ${DOTNET_VERSION}
+                APP_DOTNET_VERSION: ${APP_DOTNET_VERSION}
+                CONTAINER_TEST_ACR_NAME: ${CONTAINER_TEST_ACR_NAME}
+        ports:
+          - "${PORT}:80"
+        volumes:
+          - ${AGENT_PATH}:/usr/local/newrelic-dotnet-agent # AGENT_PATH from .env, points to newrelichome_linux_x64
+          - ${LOG_PATH}:/app/logs # LOG_PATH from .env, should be a folder unique to this run of the test app

tests/Agent/IntegrationTests/ContainerApplications/docker-compose-base.yml

@@ -1,49 +1,20 @@
 # The following must be set either in environment variables or via a .env file in the same folder as this file:
 #
-# AGENT_PATH      host path to the Agent linux home folder - will map to /usr/local/newrelic-dotnet-agent in the container
-# LOG_PATH        host path for Agent logfile output - will map to /app/logs in the container
-# DISTRO_TAG      distro tag for build, not including the architecture suffix
-# TARGET_ARCH     the target architecture for the run -- either amd64 or arm64
-# BUILD_ARCH      the build architecture -- either amd64 or arm64
-# PORT            external port for the smoketest API
+# See docker-compose-base.yml for common required variables.
 # CONTAINER_NAME  The name for the container
-# PLATFORM        The platform that the service runs on -- linux/amd64 or linux/arm64/v8
-# DOTNET_VERSION  The dotnet version number to use (8.0, etc)
-# TEST_DOCKERFILE The path and dockerfile to use for the service.
-#
 # NEW_RELIC_KAFKA_TOPIC
-# 
-# and the usual suspects:
-# NEW_RELIC_LICENSE_KEY
-# NEW_RELIC_HOST
-# NEW_RELIC_APP_NAME
-#
 #
-# To build and run, execute `docker compose -f <path to docker-compose.yml> up` 
+# To build and run, execute `docker compose -f <path to docker-compose.yml> up`
 # Alternatively, set COMPOSE_FILE environment variable to the path and omit the -f parameter
 
 services:
     LinuxSmokeTestApp:
+        extends:
+            file: docker-compose-base.yml
+            service: base-app
         container_name: ${CONTAINER_NAME}
         image: ${CONTAINER_NAME}
         platform: ${PLATFORM}
-        build:
-            context: .
-            dockerfile: ${TEST_DOCKERFILE}
-            args:
-                DISTRO_TAG: ${DISTRO_TAG}
-                TARGET_ARCH: ${TARGET_ARCH}
-                BUILD_ARCH: ${BUILD_ARCH}
-                NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY}
-                NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME}
-                NEW_RELIC_HOST: ${NEW_RELIC_HOST}
-                DOTNET_VERSION: ${DOTNET_VERSION}
-                APP_DOTNET_VERSION: ${APP_DOTNET_VERSION}
-        ports:
-          - "${PORT}:80"
-        volumes:
-          - ${AGENT_PATH}:/usr/local/newrelic-dotnet-agent # AGENT_PATH from .env, points to newrelichome_linux_x64
-          - ${LOG_PATH}:/app/logs # LOG_PATH from .env, should be a folder unique to this run of the smoketest app           
         depends_on:
             - kafka-broker
         environment:
@@ -68,8 +39,7 @@ services:
           KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
           KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
           CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
-        
-            
+
 networks:
     default:
         driver: bridge

tests/Agent/IntegrationTests/ContainerApplications/docker-compose-kafka.yml

@@ -1,56 +1,25 @@
 # The following must be set either in environment variables or via a .env file in the same folder as this file:
 #
-# AGENT_PATH      host path to the Agent linux home folder - will map to /usr/local/newrelic-dotnet-agent in the container
-# LOG_PATH        host path for Agent logfile output - will map to /app/logs in the container
-# DISTRO_TAG      distro tag for build, not including the architecture suffix
-# TARGET_ARCH     the target architecture for the run -- either amd64 or arm64
-# BUILD_ARCH      the build architecture -- either amd64 or arm64
-# PORT            external port for the smoketest API
+# See docker-compose-base.yml for common required variables.
 # CONTAINER_NAME  The name for the container
-# PLATFORM        The platform that the service runs on -- linux/amd64 or linux/arm64/v8
-# DOTNET_VERSION  The dotnet version number to use (8.0, etc)
-# TEST_DOCKERFILE The path and dockerfile to use for the service.
 #
-# NEW_RELIC_KAFKA_TOPIC
-# 
-# and the usual suspects:
-# NEW_RELIC_LICENSE_KEY
-# NEW_RELIC_HOST
-# NEW_RELIC_APP_NAME
-#
-#
-# To build and run, execute `docker compose -f <path to docker-compose.yml> up` 
+# To build and run, execute `docker compose -f <path to docker-compose.yml> up`
 # Alternatively, set COMPOSE_FILE environment variable to the path and omit the -f parameter
 
 services:
     MemcachedTestApp:
+        extends:
+            file: docker-compose-base.yml
+            service: base-app
         container_name: ${CONTAINER_NAME}
         image: ${CONTAINER_NAME}
         platform: ${PLATFORM}
-        build:
-            context: .
-            dockerfile: ${TEST_DOCKERFILE}
-            args:
-                DISTRO_TAG: ${DISTRO_TAG}
-                TARGET_ARCH: ${TARGET_ARCH}
-                BUILD_ARCH: ${BUILD_ARCH}
-                NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY}
-                NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME}
-                NEW_RELIC_HOST: ${NEW_RELIC_HOST}
-                DOTNET_VERSION: ${DOTNET_VERSION}
-                APP_DOTNET_VERSION: ${APP_DOTNET_VERSION}
-        ports:
-          - "${PORT}:80"
-        volumes:
-          - ${AGENT_PATH}:/usr/local/newrelic-dotnet-agent # AGENT_PATH from .env, points to newrelichome_linux_x64
-          - ${LOG_PATH}:/app/logs # LOG_PATH from .env, should be a folder unique to this run of the smoketest app           
         depends_on:
             - memcached-server
 
     memcached-server:
         image: memcached:1.6.31
 
-
 networks:
     default:
         driver: bridge

tests/Agent/IntegrationTests/ContainerApplications/docker-compose-memcached.yml

@@ -1,45 +1,16 @@
-version: "3"
-
 # The following must be set either in environment variables or via a .env file in the same folder as this file:
 #
-# AGENT_PATH      host path to the Agent linux home folder - will map to /usr/local/newrelic-dotnet-agent in the container
-# LOG_PATH        host path for Agent logfile output - will map to /app/logs in the container
-# DISTRO_TAG      distro tag for build, not including the architecture suffix - possible values 8.0-bookworm-slim, 8.0-alpine, 8.0-jammy
-# TARGET_ARCH     the target architecture for the build and run -- either amd64 or arm64
-# PORT            external port for the smoketest API
+# See docker-compose-base.yml for common required variables.
 # CONTAINER_NAME  The name for the container
-# PLATFORM        The platform that the service runs on -- linux/amd64 or linux/arm64/v8
-# DOTNET_VERSION  The dotnet version number to use (8.0, etc)
-# NETWORK_NAME    The network name to use for containers in this app. Should be unique among all running instances.
-# TEST_DOCKERFILE The path and dockerfile to use for the service.
-# 
-# and the usual suspects:
-# NEW_RELIC_LICENSE_KEY
-# NEW_RELIC_HOST
-# NEW_RELIC_APP_NAME
-#
 #
-# To build and run, execute `docker compose -f <path to docker-compose.yml> up` 
+# To build and run, execute `docker compose -f <path to docker-compose.yml> up`
 # Alternatively, set COMPOSE_FILE environment variable to the path and omit the -f parameter
 
 services:
     smoketestapp:
+        extends:
+            file: docker-compose-base.yml
+            service: base-app
         container_name: ${CONTAINER_NAME}
         image: ${CONTAINER_NAME}
         platform: ${PLATFORM}
-        build:
-            context: .
-            dockerfile: ${TEST_DOCKERFILE}
-            args:
-                DISTRO_TAG: ${DISTRO_TAG}
-                TARGET_ARCH: ${TARGET_ARCH}
-                NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY}
-                NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME}
-                NEW_RELIC_HOST: ${NEW_RELIC_HOST}
-                DOTNET_VERSION: ${DOTNET_VERSION}
-                APP_DOTNET_VERSION: ${APP_DOTNET_VERSION}
-        ports:
-          - "${PORT}:80"
-        volumes:
-          - ${AGENT_PATH}:/usr/local/newrelic-dotnet-agent # AGENT_PATH from .env, points to newrelichome_linux_x64
-          - ${LOG_PATH}:/app/logs # LOG_PATH from .env, should be a folder unique to this run of the smoketest app