Merge branch 'main' into marrobi/azuread-autogen

Author: Tim Allen

.devcontainer/Dockerfile

@@ -8,15 +8,13 @@ FROM --platform="${TARGETPLATFORM}" mcr.microsoft.com/vscode/devcontainers/pytho
 # This will be set to true when running in VSCode
 ARG INTERACTIVE="false"
 
-ARG USERNAME=vscode
 ARG USER_UID=1000
-ARG USER_GID=$USER_UID
+ARG USERNAME=vscode
 
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+# make user ID match user ID on host machine
+RUN usermod --uid $USER_UID $USERNAME
 
-# Set up non-root user
-COPY .devcontainer/scripts/non-root-user.sh /tmp/
-RUN bash /tmp/non-root-user.sh "${USERNAME}" "${USER_UID}" "${USER_GID}"
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Set env for tracking that we're running in a devcontainer
 ENV DEVCONTAINER=true
@@ -26,7 +24,7 @@ ARG NODE_VERSION="lts/*"
 RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 
 # Install terraform
-ARG TERRAFORM_VERSION="1.3.6"
+ARG TERRAFORM_VERSION="1.4.5"
 COPY .devcontainer/scripts/terraform.sh /tmp/
 RUN bash /tmp/terraform.sh "${TERRAFORM_VERSION}" /usr/bin
 
@@ -39,7 +37,7 @@ RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release
     && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \
     && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \
     | tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    && apt-get update && apt-get install -y docker-ce="5:20.10.21~3-0~debian-bullseye" docker-ce-cli="5:20.10.21~3-0~debian-bullseye" containerd.io="1.6.12-1" --no-install-recommends \
+    && apt-get update && apt-get install -y docker-ce="5:24.0.0-1~debian.11~bullseye" docker-ce-cli="5:24.0.0-1~debian.11~bullseye" docker-compose-plugin="2.21.0-1~debian.11~bullseye" containerd.io="1.6.24-1" docker-buildx-plugin --no-install-recommends \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
 # Install Certbot
@@ -50,13 +48,11 @@ RUN if [ "${INTERACTIVE}" = "true" ]; then \
     && /opt/certbot/bin/pip install --no-cache-dir certbot \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/* ; fi
 
-# can't be in a non default path
-# ARG PORTER_HOME_V1=/home/$USERNAME/.porter-v1/
 ARG PORTER_HOME_V1=/home/$USERNAME/.porter/
-ARG PORTER_VERSION=v1.0.8
-ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.1
-ARG PORTER_AZ_MIXIN_VERSION=v1.0.0
-ARG PORTER_AZURE_PLUGIN_VERSION=v1.1.0
+ARG PORTER_VERSION=v1.0.15
+ARG PORTER_TERRAFORM_MIXIN_VERSION=v1.0.2
+ARG PORTER_AZ_MIXIN_VERSION=v1.0.1
+ARG PORTER_AZURE_PLUGIN_VERSION=v1.2.0
 COPY .devcontainer/scripts/porter-v1.sh /tmp/
 RUN export PORTER_VERSION=${PORTER_VERSION} \
     PORTER_TERRAFORM_MIXIN_VERSION=${PORTER_TERRAFORM_MIXIN_VERSION} \
@@ -68,6 +64,8 @@ RUN export PORTER_VERSION=${PORTER_VERSION} \
 ENV PATH ${PORTER_HOME_V1}:$PATH
 
 # Install requirements
+ARG PIP_VERSION=23.3.1
+RUN  pip3 --no-cache-dir install pip==${PIP_VERSION} && pip3 config set global.disable-pip-version-check true
 COPY ["requirements.txt", "/tmp/pip-tmp/" ]
 COPY ["api_app/requirements.txt", "api_app/requirements-dev.txt", "/tmp/pip-tmp/api_app/" ]
 COPY ["resource_processor/vmss_porter/requirements.txt", "/tmp/pip-tmp/resource_processor/vmss_porter/" ]
@@ -77,18 +75,23 @@ COPY ["airlock_processor/requirements.txt", "/tmp/pip-tmp/airlock_processor/"]
 RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt
 
 # Install azure-cli
-ARG AZURE_CLI_VERSION=2.37.0-1~bullseye
+ARG AZURE_CLI_VERSION=2.57.0-1~bullseye
 COPY .devcontainer/scripts/azure-cli.sh /tmp/
 RUN export AZURE_CLI_VERSION=${AZURE_CLI_VERSION} \
     && /tmp/azure-cli.sh
 
-ARG YQ_VERSION="v4.30.6"
+ARG YQ_VERSION="v4.33.3"
 RUN curl -L --fail -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
     && chmod +x /usr/local/bin/yq
 
 ARG PAJV_VERSION="1.2.0"
 RUN npm install -g pajv@${PAJV_VERSION}
 
+# Install git - required for terraform's git modules
+RUN if [ "${INTERACTIVE}" = "false" ]; then \
+apt-get update && apt-get install --no-install-recommends -y git \
+&& apt-get clean -y && rm -rf /var/lib/apt/lists/* ; fi
+
 USER $USERNAME
 
 # Save command line history

.devcontainer/devcontainer.json

@@ -1,5 +1,11 @@
 {
   "name": "AzureTRE",
+  // Uncomment when debugging using Jetbrains
+  // "features": {
+  //   "ghcr.io/devcontainers/features/sshd:1": {
+  //       "version": "latest"
+  //   }
+  // },
   "build": {
     "context": "..",
     "dockerfile": "Dockerfile",
@@ -55,8 +61,10 @@
         "python.testing.unittestEnabled": false,
         "python.testing.pytestEnabled": true,
         "python.testing.pytestArgs": [
-          "api_app"
+          "--ignore=e2e_tests",
+          "-W ignore::DeprecationWarning"
         ],
+        "python.testing.cwd": "${workspaceFolder}",
         "files.associations": {
           "*.terraform": "terraform"
         },
@@ -79,7 +87,11 @@
               "justMyCode": false,
               "console": "integratedTerminal",
               "preLaunchTask": "Copy_env_file_for_api_debug",
-              "cwd": "${workspaceFolder}/api_app"
+              "cwd": "${workspaceFolder}/api_app",
+              "envFile": "${workspaceFolder}/api_app/.env",
+              "env": {
+                "OTEL_RESOURCE_ATTRIBUTES": "service.name=api,service.instance.id=local_debug,service.version=dev"
+              }
             },
             {
               "name": "E2E Extended",
@@ -181,8 +193,10 @@
               "cwd": "${workspaceFolder}/resource_processor",
               "envFile": "${workspaceFolder}/core/private.env",
               "env": {
-                "PYTHONPATH": "."
-              }
+                "PYTHONPATH": ".",
+                "OTEL_RESOURCE_ATTRIBUTES": "service.name=resource_processor,service.instance.id=local_debug,service.version=dev"
+              },
+              "justMyCode": false
             },
             {
               "name": "Debug Python file",

.devcontainer/scripts/non-root-user.sh

@@ -17,3 +17,7 @@ A clear and concise description of what the bug is.
 1.
 1.
 1.
+
+**Azure TRE release version (e.g. v0.14.0 or main):**
+
+**Deployed Azure TRE components - click the (i) in the UI:**

.github/ISSUE_TEMPLATE/bug_report.md

@@ -11,6 +11,10 @@ inputs:
   AZURE_CREDENTIALS:
     description: "Credentials to access Azure."
     required: true
+  AZURE_ENVIRONMENT:
+    description: "Azure Cloud Environment"
+    default: "AzureCloud"
+    required: false
   API_CLIENT_ID:
     description: "The API Client Id."
     required: false
@@ -123,22 +127,56 @@ inputs:
     description: "JSON string containing key/value pairs to injet into the Resource Processor as ENV vars"
     required: false
     default: ""
+  KV_PURGE_PROTECTION_ENABLED:
+    description: "A boolean indicating if the purge protection will be enabled on the core keyvault."
+    required: false
+    default: "true"
+  FIREWALL_SKU:
+    description: "Firewall SKU"
+    required: false
+    default: ""
 
 runs:
   using: composite
   steps:
+    - name: Construct AZURE_ENVIRONMENT
+      shell: bash
+      run: |
+        azure_env="${{ (inputs.AZURE_ENVIRONMENT != '' && inputs.AZURE_ENVIRONMENT) || 'AzureCloud' }}"
+        echo "AZURE_ENVIRONMENT=$azure_env" >> $GITHUB_ENV
+
     - name: Azure Login
-      uses: azure/login@v1
+      uses: azure/login@v2
       if: contains(inputs.COMMAND, 'bootstrap') != true
       with:
         creds: ${{ inputs.AZURE_CREDENTIALS }}
+        environment: "${{ env.AZURE_ENVIRONMENT }}"
 
     - name: ACR Login
       shell: bash
       if: contains(inputs.COMMAND, 'bootstrap') != true
       run: |
         az acr login --name "${{ inputs.CI_CACHE_ACR_NAME }}"
 
+    - name: Construct TRE_URL
+      shell: bash
+      run: |
+        source devops/scripts/construct_tre_url.sh
+        tre_url=$(construct_tre_url "${{ inputs.TRE_ID }}" "${{ inputs.LOCATION }}" "${{ env.AZURE_ENVIRONMENT }}")
+        echo "TRE_URL=$tre_url" >> $GITHUB_ENV
+
+    - name: Construct ARM_ENVIRONMENT
+      shell: bash
+      run: |
+        source devops/scripts/convert_azure_env_to_arm_env.sh
+        arm_environment=$(convert_azure_env_to_arm_env "${{ env.AZURE_ENVIRONMENT }}")
+        echo "ARM_ENVIRONMENT=$arm_environment" >> $GITHUB_ENV
+
+    - name: Construct ACR Domain Suffix
+      shell: bash
+      run: |
+        echo "ACR_DOMAIN_SUFFIX=$(az cloud show --query suffixes.acrLoginServerEndpoint --output tsv)" >> $GITHUB_ENV
+
     - name: Run command in DevContainer
       shell: bash
       run: |
@@ -152,6 +190,9 @@ runs:
           -e USE_ENV_VARS_NOT_FILES="true" \
           -e BUNDLE_TYPE="${{ inputs.BUNDLE_TYPE }}" \
           -e WORKSPACE_SERVICE_NAME="${{ inputs.WORKSPACE_SERVICE_NAME }}" \
+          -e ARM_ENVIRONMENT="${{ env.ARM_ENVIRONMENT }}" \
+          -e TF_VAR_arm_environment="${{ env.ARM_ENVIRONMENT }}" \
+          -e AZURE_ENVIRONMENT="${{ env.AZURE_ENVIRONMENT }}" \
           -e LOCATION="${{ inputs.LOCATION }}" \
           -e TF_VAR_location="${{ inputs.LOCATION }}" \
           -e RESOURCE_LOCATION="${{ inputs.LOCATION }}" \
@@ -177,13 +218,15 @@ runs:
             && inputs.ENABLE_SWAGGER) || 'false' }}" \
           -e SWAGGER_UI_CLIENT_ID="${{ inputs.SWAGGER_UI_CLIENT_ID }}" \
           -e TF_VAR_swagger_ui_client_id="${{ inputs.SWAGGER_UI_CLIENT_ID }}" \
-          -e TF_VAR_core_address_space="${{ inputs.core_address_space }}" \
-          -e TF_VAR_tre_address_space="${{ inputs.tre_address_space }}" \
+          -e TF_VAR_core_address_space="${{ (inputs.CORE_ADDRESS_SPACE != ''
+            && inputs.CORE_ADDRESS_SPACE) || '10.0.0.0/22' }}" \
+          -e TF_VAR_tre_address_space="${{ (inputs.TRE_ADDRESS_SPACE != ''
+            && inputs.TRE_ADDRESS_SPACE) || '10.0.0.0/16' }}" \
           -e API_CLIENT_ID="${{ inputs.API_CLIENT_ID }}" \
           -e AAD_TENANT_ID="${{ inputs.AAD_TENANT_ID }}" \
           -e TRE_ID="${{ inputs.TRE_ID }}" \
           -e TF_VAR_tre_id="${{ inputs.TRE_ID }}" \
-          -e TRE_URL="https://${{inputs.TRE_ID}}.${{inputs.LOCATION}}.cloudapp.azure.com" \
+          -e TRE_URL="${{ env.TRE_URL }}" \
           -e TEST_WORKSPACE_APP_ID="${{ inputs.TEST_WORKSPACE_APP_ID }}" \
           -e TEST_WORKSPACE_APP_SECRET="${{ inputs.TEST_WORKSPACE_APP_SECRET }}" \
           -e TEST_APP_ID="${{ inputs.TEST_APP_ID }}" \
@@ -192,15 +235,18 @@ runs:
           -e IS_API_SECURED=${{ inputs.IS_API_SECURED }} \
           -e DOCKER_BUILDKIT=1 \
           -e TF_VAR_stateful_resources_locked=${{ inputs.STATEFUL_RESOURCES_LOCKED }} \
+          -e TF_VAR_kv_purge_protection_enabled="${{ inputs.KV_PURGE_PROTECTION_ENABLED }}" \
           -e TF_VAR_enable_airlock_malware_scanning=${{ inputs.ENABLE_AIRLOCK_MALWARE_SCANNING }} \
           -e CI_CACHE_ACR_NAME="${{ inputs.CI_CACHE_ACR_NAME }}" \
           -e TF_VAR_core_app_service_plan_sku="${{ (inputs.CORE_APP_SERVICE_PLAN_SKU != ''
             && inputs.CORE_APP_SERVICE_PLAN_SKU) || 'P1v2' }}" \
           -e WORKSPACE_APP_SERVICE_PLAN_SKU="${{ (inputs.WORKSPACE_APP_SERVICE_PLAN_SKU != ''
             && inputs.WORKSPACE_APP_SERVICE_PLAN_SKU) || 'P1v2' }}" \
-          -e TF_VAR_rp_bundle_values='${{ inputs.RP_BUNDLE_VALUES }}' \
+          -e TF_VAR_rp_bundle_values='${{ (toJson(inputs.RP_BUNDLE_VALUES) != '""'
+            && inputs.RP_BUNDLE_VALUES) || '{}' }}' \
           -e TF_VAR_resource_processor_number_processes_per_instance="${{ (inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE != ''
-            && inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE) || 5  }}" \
+            && inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE) || 5 }}" \
+          -e TF_VAR_firewall_sku=${{ inputs.FIREWALL_SKU }} \
           -e E2E_TESTS_NUMBER_PROCESSES="${{ inputs.E2E_TESTS_NUMBER_PROCESSES }}" \
-          '${{ inputs.CI_CACHE_ACR_NAME }}.azurecr.io/tredev:${{ inputs.DEVCONTAINER_TAG }}' \
+          '${{ inputs.CI_CACHE_ACR_NAME }}${{ env.ACR_DOMAIN_SUFFIX }}/tredev:${{ inputs.DEVCONTAINER_TAG }}' \
         bash -c "${{ inputs.COMMAND }}"

.github/actions/devcontainer_run_command/action.yml

@@ -9,6 +9,7 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
+    open-pull-requests-limit: 0
 
   - package-ecosystem: "npm"
     directory: "/ui/app"
@@ -17,6 +18,7 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
+    open-pull-requests-limit: 0
 
   - package-ecosystem: "pip"
     directory: "/"
@@ -25,6 +27,7 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
+    open-pull-requests-limit: 0
 
   - package-ecosystem: "maven"
     directory: "/templates/workspace_services/guacamole/guacamole-server/guacamole-auth-azure"
@@ -33,6 +36,7 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
+    open-pull-requests-limit: 0
 
   - package-ecosystem: "terraform"
     directory: "/devops/terraform"
@@ -41,6 +45,7 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
+    open-pull-requests-limit: 0
 
   - package-ecosystem: "terraform"
     directory: "/core/terraform"
@@ -49,3 +54,13 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
+    open-pull-requests-limit: 0
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]
+    open-pull-requests-limit: 0