Changes for v1.1.0 (#5)

Author: JimTharioAmazon

.github/ISSUE_TEMPLATE/bug_report.md

@@ -25,7 +25,7 @@ To get the version of the solution, you can look at the description of the creat
     "Environment": {
       "Variables": {
         "SOLUTION_ID": "SO0248",
-        "SOLUTION_VERSION": "v1.0.2"
+        "SOLUTION_VERSION": "v1.1.0"
       }
     },
 ```
@@ -34,7 +34,7 @@ This information is also provided in `source/infrastructure/cdk.json`:
 
 ```json
     "SOLUTION_ID": "SO0248",
-    "SOLUTION_VERSION": "v1.0.2",
+    "SOLUTION_VERSION": "v1.1.0",
 ```
 
 

.gitignore

@@ -73,3 +73,7 @@ source/aws_lambda/shared/util/build/*
 *.tmp
 *.sav
 *.dtmp
+
+# generated files
+source/infrastructure/custom_resources/docker_configs_bucket_lambda/current-config/
+source/infrastructure/custom_resources/docker_configs_bucket_lambda/default-config/

CHANGELOG.md

@@ -16,8 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Remove python `setuptools` and `pip` from prebid server docker image
 - Include missing copyright header for `source/infrastructure/prebid_server/stack_constants.py`
 
-
 ## [1.0.2] - 2024-09-23
 
 - Upgrade Python `requests` package to version 2.32.3 in requirements.txt
 - Bug fix for launch failure of EfsCleanupContainerStop Lambda function
+
+## [1.1.0] - 2024-10-31
+- Upgrade to Prebid Server v3.13 and underlying Docker base image
+- ECS runtime logs in AWS CloudWatch instead of S3
+- Option to opt-out of installing CloudFront and WAF
+- Customize Prebid Server configuration through files in S3
+- Option to specify a custom container image 

NOTICE.txt

@@ -13,11 +13,7 @@ THIRD PARTY COMPONENTS
 This software includes third party software subject to the following copyrights:
 
  Name                                             Version    License
- prebid-server-java                               2.12.0     Apache-2.0
- Jinja2                                           3.1.2      BSD License
- MarkupSafe                                       2.1.1      BSD License
- PyYAML                                           6.0        MIT License
- Werkzeug                                         2.1.2      BSD License
+ prebid-server-java                               3.13.0     Apache-2.0
  attrs                                            22.1.0     MIT License
  avro                                             1.11.1     Apache Software License
  aws-cdk-lib                                      2.140.0    Apache-2.0
@@ -33,7 +29,6 @@ This software includes third party software subject to the following copyrights:
  aws-solutions-constructs.core                    2.25.0     Apache-2.0
  aws-solutions-python                             2.0.0      Apache Software License
  aws-xray-sdk                                     2.11.0     Apache Software License
- black                                            22.10.0    MIT License
  boto3                                            1.26.12    Apache Software License
  botocore                                         1.29.12    Apache Software License
  cattrs                                           22.2.0     MIT License
@@ -53,8 +48,10 @@ This software includes third party software subject to the following copyrights:
  idna                                             3.4        BSD License
  infrastructure                                   1.0.0      Apache Software License
  iniconfig                                        1.1.1      MIT License
+ Jinja2                                           3.1.2      BSD License
  jmespath                                         1.0.1      MIT License
  jsii                                             1.71.0     Apache Software License
+ MarkupSafe                                       2.1.1      BSD License
  moto                                             4.0.3      Apache Software License
  mypy-extensions                                  0.4.3      MIT License
  packaging                                        21.3       Apache Software License; BSD License
@@ -71,6 +68,7 @@ This software includes third party software subject to the following copyrights:
  pytest-mock                                      3.10.0     MIT License
  python-dateutil                                  2.8.2      Apache Software License; BSD License
  pytz                                             2022.6     MIT License
+ PyYAML                                           6.0.2      MIT License
  requests                                         2.32.3     Apache Software License
  responses                                        0.17.0     Apache 2.0
  s3transfer                                       0.6.0      Apache Software License
@@ -85,4 +83,5 @@ This software includes third party software subject to the following copyrights:
  wrapt                                            1.14.1     BSD License
  xmltodict                                        0.13.0     MIT License
  wheel                                            0.38.4     MIT License
+ Werkzeug                                         2.1.2      BSD License
 

README.md

@@ -1,6 +1,7 @@
+
 # Prebid Server Deployment on AWS
 
-This solution deploys v2.12.0 of [Prebid Server Java](https://github.com/prebid/prebid-server-java.git) with infrastructure in a single region of the AWS Cloud to handle a wide range of request traffic, and recording of auction and bid transaction data.
+This solution deploys v3.13.0 of [Prebid Server Java](https://github.com/prebid/prebid-server-java.git) with infrastructure in a single region of the AWS Cloud to handle a wide range of request traffic, and recording of auction and bid transaction data.
 
 ## Architecture 
 
@@ -60,6 +61,13 @@ sh ../deployment/run-unit-tests.sh --in-venv 1
 
 ### 3. Build the solution for deployment
 
+#### Prebid Server Container Image
+By default, the Prebid Server container image will be built locally using Docker ([README](deployment/ecr/prebid-server/README.md)). If you prefer to use a remote image (from ECR or Docker Hub), set the following environment variable with your fully qualified image name before building the template:
+
+```bash
+export OVERRIDE_ECR_REGISTRY=your-fully-qualified-image-name
+```
+
 #### Using AWS CDK (recommended) 
 Packaging and deploying the solution with the AWS CDK allows for the most flexibility in development
 ```bash 
@@ -90,6 +98,7 @@ export DIST_BUCKET_PREFIX=my-bucket-name
 export SOLUTION_NAME=my-solution-name  
 export VERSION=my-version  
 export REGION_NAME=my-region
+export OVERRIDE_ECR_REGISTRY=my-ecr-registry
 
 build-s3-cdk-dist deploy \
   --source-bucket-name $DIST_BUCKET_PREFIX \
@@ -108,6 +117,7 @@ build-s3-cdk-dist deploy \
 - `$SOLUTION_NAME` - The name of This solution (example: solution-customization)
 - `$VERSION` - The version number to use (example: v0.0.1)
 - `$REGION_NAME` - The region name to use (example: us-east-1)
+- `$OVERRIDE_ECR_REGISTRY` - The ecr-registry to use (example: public.ecr.aws/abc12345/prebid-server:latest)
 
 This will result in all global assets being pushed to the `DIST_BUCKET_PREFIX`, and all regional assets being pushed to 
 `DIST_BUCKET_PREFIX-<REGION_NAME>`. If your `REGION_NAME` is us-east-1, and the `DIST_BUCKET_PREFIX` is
@@ -122,19 +132,24 @@ After running the command, you can deploy the template:
 
 > **Note:** You can drop `--sync` from the command to only perform the build and synthesis of the template without uploading to a remote location. This is helpful when testing new changes to the code.
 
-#### Build prebid-server docker locally
-* [Docker README.md](deployment/ecr/prebid-server/README.md)
-
 ## Prebid Server Java Container Customization
 
 You may choose to customize the container configuration, or create your own container to use with this solution. The infrastructure for this solution has only been tested on Prebid Server Java.
 
+#### Deploy with Customized Prebid Server Configurations
+* After deploying the CloudFormation template stack, find the S3 bucket in the CloudFormation stack outputs named `ContainerImagePrebidSolutionConfigBucket`.
+1. Review the `/prebid-server/default/README.md` and `/prebid-server/current/README.md` files in the bucket.
+2. Upload your changes to the `/prebid-server/current/` prefix in that bucket.
+3. To update the ECS service manually, navigate to the Amazon ECS cluster associated with the deployed CloudFormation stack using the AWS Management Console. Then, update the ECS service by selecting the 'Force New Deployment' option with the new task definition version, as described in the official AWS documentation for updating an ECS service via the console. [ref](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/update-service-console-v2.html).[ref](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/update-service-console-v2.html).
+
 ### Runtime and Metric Logging for ETL
 
+The Prebid Server container shipped with this solution is configured for two types of logging.
+
+Runtime logs from the Prebid Server are sent to CloudWatch logs under the `PrebidContainerLogGroup` log group. This log group collects runtime logs for all containers and includes the container ID from ECS. Container logs are available to use with CloudWatch Log Insights and Live Tailing.
+
 Review the file named `deployment/ecr/prebid-server/prebid-logging.xml` for the required locations of log file output. Resources outside of the containers instances, including Lambda Functions and AWS DataSync jobs, expect to find log files at the following locations.
 
-* `/mnt/efs/logs/CONTAINER_ID/prebid-server.log` is where the current stdout/stderr output log is written.
-* `/mnt/efs/logs/CONTAINER_ID/archived/prebid-server.TIMESTAMP.log.gz` is where logs are rotated on a schedule. This location is scanned by AWS DataSync periodically to migrate logs from EFS to S3 for the long-term storage. Rotated logs are removed from EFS after migration to S3.
 * `/mnt/efs/metrics/CONTAINER_ID/prebid-metrics.log` is where the current metrics output log is written. The default interval for outputting metrics to this file is 30 seconds.
 * `/mnt/efs/metrics/CONTAINER_ID/archived/prebid-metrics.TIMESTAMP.log.gz`is where logs are rotated on a schedule. This location is scanned by AWS DataSync periodically to migrate logs from EFS to S3 for the ETL process to AWS Glue Catalog. Rotated logs are removed from EFS after migration to S3.
 
@@ -153,7 +168,7 @@ If you'd like to use a different container:
 * Build and host the solution assets for installation in your account using the process described above
 * Change to the `deployment/global-s3-assets` folder on the build workstation
 * Open the file `prebid-server-deployment-on-aws.template` in an editor
-* Find the line in the template under the Task Definition resource that is `"Image": "public.ecr.aws/aws-solutions/prebid-server:v1.0.2",`
+* Find the line in the template under the Task Definition resource that is `"Image": "public.ecr.aws/aws-solutions/prebid-server:v1.1.0",`
 * Update the Image property value with your container image URI
 * Create the stack by uploading the changed template to the CloudFormation console
 

deployment/build-s3-dist.sh

@@ -9,7 +9,7 @@
 # Paramenters:
 #  - source-bucket-base-name: Name for the S3 bucket location where the template will source the Lambda
 #    code from. The template will append '-[region_name]' to this bucket name.
-#    For example: ./build-s3-dist.sh solutions v1.0.2
+#    For example: ./build-s3-dist.sh solutions v1.1.0
 #    The template will then expect the source code to be located in the solutions-[region_name] bucket
 #
 #  - solution-name: name of the solution for consistency
@@ -21,7 +21,7 @@
 # Check to see if input has been provided:
 if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
     echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside."
-    echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.0.2"
+    echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.1.0"
     exit 1
 fi
 

deployment/ecr/prebid-server/.gitignore

@@ -0,0 +1,4 @@
+/logs/*
+/metrics/*
+bootstrap.log
+sample/configs/prebid-config.yaml

deployment/ecr/prebid-server/Dockerfile

@@ -1,54 +1,50 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-FROM public.ecr.aws/amazonlinux/amazonlinux:2023
 
-# Setup environment for building prebid-server-java
-RUN yum -y install maven-amazon-corretto17
+# ------------------------------------------------------------------------------------------------------------------------------------------------------
+# PURPOSE:
+#  This Dockerfile is designed to build a Docker image for the Prebid Server Java application by cloning the repository,
+#  building the application using Maven, and setting up the necessary configuration and scripts to run the application inside the container on aws.
+# -------------------------------------------------------------------------------------------------------------------------------------------------------
+# USAGE:
+#  For instructions on how to build this Dockerfile locally, please refer to the README.md file in the current directory.
+# -------------------------------------------------------------------------------------------------------------------------------------------------------
 
-# Remove python setuptools and package installer
-RUN yum -y remove python-setuptools
-RUN yum -y remove python-pip 
 
-RUN yum -y install git
-RUN yum -y install jq
-RUN git clone https://github.com/prebid/prebid-server-java.git
-COPY config.json prebid-server-java/
-COPY prebid-logging.xml prebid-server-java/
-COPY prebid-server-java-patches.diff prebid-server-java/
+FROM public.ecr.aws/docker/library/maven:3-amazoncorretto-21-al2023
+
+# Install required packages
+RUN yum -y install git jq aws-cli
+
+# Update Python packages to address security vulnerabilities
+RUN dnf update python-setuptools --releasever 2023.1.20230719
+RUN dnf update python-pip --releasever 2023.3.20231211
+
+# Clone the Prebid Server Java repository
+RUN git clone --single-branch --branch master https://github.com/prebid/prebid-server-java.git
+
+# Copy source build configuration file
+COPY docker-build-config.json docker-build-config.json
+
+# Set the working directory to the cloned repository
 WORKDIR /prebid-server-java
 
-# Download the user-specified version of prebid-server-java
+# Fetch tags and checkout the specified version
 RUN git fetch origin --tags
-RUN git checkout $(jq -r .GIT_TAG_VERSION config.json)
-RUN git apply prebid-server-java-patches.diff
+RUN git checkout $(jq -r .GIT_TAG_VERSION ../docker-build-config.json)
 
-# Build prebid-server-java using Maven
-RUN mvn clean package $(jq -r .MVN_CLI_OPTIONS config.json)
+# Build the Prebid Server Java application using Maven
+RUN mvn clean package $(jq -r .MVN_CLI_OPTIONS ../docker-build-config.json)
 
-# Remove the local Maven repository
+# Remove the local Maven repository to reduce image size
 RUN rm -rf /root/.m2
 
-EXPOSE 8080
-EXPOSE 8060
-
-# An environment variable named ECS_CONTAINER_METADATA_URI_V4
-# is injected by AWS Fargate into each container in a task.
-# The entrypoint defined below parses the container's unique
-# ID from that environment variable and uses it to ensure the
-# prebid-server.log is written to a unique directory under
-# /mnt/efs/.
-#
-# Metrics are sent to /mnt/efs/metrics folder also using the
-# container ID in the path. Files have the name prebid-metrics.log.
-#
-# The space that is prepended to ${ECS_CONTAINER_METADATA_URI_V4})
-# is needed so that the container can be built locally where that
-# environment variable is not defined.
-ENTRYPOINT /usr/bin/java \
-    -DcontainerId=$(basename " "${ECS_CONTAINER_METADATA_URI_V4}) \
-    -Dlogging.config=./prebid-logging.xml \
-    -XX:+UseParallelGC \
-    -jar target/prebid-server.jar \
-    --spring.config.additional-location=sample/prebid-config.yaml
-    
+# Expose ports for the Prebid Server Java application
+EXPOSE 8080 8060
+
+# Copy the bootstrap script
+COPY bootstrap.sh ../bootstrap.sh
+
+# Set the entrypoint to execute the bootstrap script
+ENTRYPOINT ["sh", "../bootstrap.sh"]

deployment/ecr/prebid-server/README.md

@@ -8,9 +8,11 @@
 * Build prebid-server image.
   `docker build -t prebid-server .`
 * Run prebid-server image: 
- `docker run -v ./:/mnt/efs -p 8080:8080 prebid-server`
+ `docker run -v ./:/mnt/efs -v ~/.aws:/root/.aws -p 8080:8080 prebid-server`
 * View prebid server log:  
   `tail -f ./logs/prebid-server.log`
+* Check server to verify status `200 OK`, `{"application":{"status":"ok"}}`:
+  `curl -i http://localhost:8080/status`
 
 
 ### Docker Config

deployment/ecr/prebid-server/bootstrap.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# -----------------------------------------------------------------------------------------------------------------------------------------------------------------
+# PURPOSE:
+#  * Download Prebid Server configuration files and scripts from an S3 bucket.
+#  * The S3 bucket name is obtained from the environment variable DOCKER_CONFIGS_S3_BUCKET_NAME.
+#  * The configuration files are downloaded into a local /prebid-configs directory.
+#  * The default and current configuration files are fetched from two specific prefixes in the S3 bucket.
+#  * After download, the script verifies that essential configuration files exist.
+#  * The entrypoint script is then executed to start the Docker containers.
+# -----------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+set -euo pipefail
+
+# Set variables
+PREBID_CONFIGS_DIR="prebid-configs"
+ENTRYPOINT_SCRIPT="entrypoint.sh"
+REQUIRED_CONFIG_FILES="${ENTRYPOINT_SCRIPT} prebid-config.yaml prebid-logging.xml"
+ENTRYPOINT_DIR="../${PREBID_CONFIGS_DIR}"
+
+# Check if the S3 bucket environment variable is set
+if [ -z "${DOCKER_CONFIGS_S3_BUCKET_NAME:-}" ]; then
+    echo "Error: DOCKER_CONFIGS_S3_BUCKET_NAME environment variable is not set"
+    exit 1
+else
+    # Define S3 paths
+    DEFAULT_S3_PATH="s3://${DOCKER_CONFIGS_S3_BUCKET_NAME}/prebid-server/default/"
+    CURRENT_S3_PATH="s3://${DOCKER_CONFIGS_S3_BUCKET_NAME}/prebid-server/current/"
+    
+    echo "Cleaning up and recreating ${ENTRYPOINT_DIR}"
+    rm -rvf "${ENTRYPOINT_DIR}" || { echo "Failed to remove ${ENTRYPOINT_DIR}"; exit 1; }
+    mkdir -pv "${ENTRYPOINT_DIR}" || { echo "Failed to create ${ENTRYPOINT_DIR}"; exit 1; }
+
+    # Download default Prebid configuration files from S3
+    echo "Downloading default configuration files from S3 bucket: ${DEFAULT_S3_PATH}"
+    if aws s3 cp "$DEFAULT_S3_PATH" "$ENTRYPOINT_DIR" --recursive --exclude "README.md"; then
+        echo "Successfully downloaded default configuration files"
+    else
+        echo "Failed to download default configuration files"
+        exit 1
+    fi
+
+    # Download current Prebid configuration files from S3 (ignore if missing)
+    echo "Downloading current configuration files from S3 bucket: ${CURRENT_S3_PATH}"
+    if aws s3 cp "$CURRENT_S3_PATH" "$ENTRYPOINT_DIR" --recursive --exclude "README.md"; then
+        echo "Successfully downloaded current configuration files"
+    else
+        echo "Warning: Failed to download current configuration files, proceeding without them"
+    fi
+fi
+
+# Check if all required configuration files exist
+for required_config_file in $REQUIRED_CONFIG_FILES; do
+    echo "Checking if ${required_config_file} exists"
+    if [ ! -f "${ENTRYPOINT_DIR}/${required_config_file}" ]; then
+       echo "Error: Required configuration file ${required_config_file} is missing"
+       exit 1
+    fi
+done
+
+# Execute the entrypoint script to start Docker containers
+echo "Executing ${ENTRYPOINT_SCRIPT}"
+sh "${ENTRYPOINT_DIR}/${ENTRYPOINT_SCRIPT}" || { echo "Failed to execute ${ENTRYPOINT_SCRIPT}"; exit 1; }