diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e6a83a377..c4c30945c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ FEATURES: ENHANCEMENTS: BUG FIXES: -* Update to Resource Processor Image, now using Ubuntu 22.04 (jammy). Part of ([#3523](https://github.com/microsoft/AzureTRE/issues/3523)) +* Update Guacomole Linux VM Images to 22.04 and 20.04 Data Science VM. Part of ([#3523](https://github.com/microsoft/AzureTRE/issues/3523)) +* Update to Resource Processor Image, now using Ubuntu 22.04 (jammy). Part of ([#3523](https://github.com/microsoft/AzureTRE/issues/3523)) COMPONENTS: diff --git a/e2e_tests/test_performance.py b/e2e_tests/test_performance.py index 9f284fc235..6c6d836d9d 100644 --- a/e2e_tests/test_performance.py +++ b/e2e_tests/test_performance.py @@ -106,7 +106,7 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify) -> "properties": { "display_name": "Perf test VM", "description": "", - "os_image": "Ubuntu 18.04" + "os_image": "Ubuntu 22.04 LTS" } } diff --git a/templates/shared_services/sonatype-nexus-vm/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml index 307bee19bc..2e73d5f318 100644 --- a/templates/shared_services/sonatype-nexus-vm/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-sonatype-nexus -version: 2.8.13 +version: 2.8.15 description: "A Sonatype Nexus shared service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/microsoft_download_conf.json b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/microsoft_download_conf.json new file mode 100644 index 0000000000..a153c626a9 --- /dev/null +++ b/templates/shared_services/sonatype-nexus-vm/scripts/nexus_repos_config/microsoft_download_conf.json @@ -0,0 +1,32 @@ +{ + "name": "microsoft-download", + "online": true, + "storage": { + "blobStoreName": "default", + "strictContentTypeValidation": true, + "write_policy": "ALLOW" + }, + "proxy": { + "remoteUrl": "https://download.microsoft.com/download", + "contentMaxAge": 1440, + "metadataMaxAge": 1440 + }, + "negativeCache": { + "enabled": true, + "timeToLive": 1440 + }, + "httpClient": { + "blocked": false, + "autoBlock": false, + "connection": { + "retries": 0, + "userAgentSuffix": "string", + "timeout": 60, + "enableCircularRedirects": false, + "enableCookies": false, + "useTrustStore": false + } + }, + "baseType": "raw", + "repoType": "proxy" + } \ No newline at end of file diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf index 27a6a3d04f..79dfa04472 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf @@ -83,6 +83,7 @@ resource "azurerm_user_assigned_identity" "nexus_msi" { location = data.azurerm_resource_group.rg.location resource_group_name = local.core_resource_group_name tags = local.tre_shared_service_tags + lifecycle { ignore_changes = [tags] } } diff --git a/templates/workspace_services/guacamole/user_resources/README.md b/templates/workspace_services/guacamole/user_resources/README.md index c553e1673a..d0db21cb86 100644 --- a/templates/workspace_services/guacamole/user_resources/README.md +++ b/templates/workspace_services/guacamole/user_resources/README.md @@ -2,10 +2,9 @@ This folder contains user resources that can be deployed with the Guacamole workspace service: -- linuxvm - a Linux-based virtual machine (expects an Ubuntu 18.04-based VM) +- linuxvm - a Linux-based virtual machine - windowsvm - A Windows-based virtual machine - ## Customising the user resources The `guacamole-azure-linuxvm` and `guacamole-azure-windowsvm` folders follow a consistent layout. @@ -29,21 +28,32 @@ custom: "8 CPU | 32GB RAM": Standard_D8s_v5 "16 CPU | 64GB RAM": Standard_D16s_v5 image_options: - "Ubuntu 18.04": + "Ubuntu 22.04 LTS": source_image_reference: publisher: canonical - offer: ubuntuserver - sku: 18_04-lts-gen2 + offer: 0001-com-ubuntu-server-jammy + sku: 22_04-lts-gen2 version: latest + apt_sku: 22.04 install_ui: true conda_config: false - "Ubuntu 18.04 Data Science VM": + "Ubuntu 20.04 LTS": + source_image_reference: + publisher: canonical + offer: 0001-com-ubuntu-server-focal + sku: 20_04-lts-gen2 + version: latest + apt_sku: 20.04 + install_ui: true + conda_config: false + "Ubuntu 20.04 LTS Data Science VM": source_image_reference: publisher: microsoft-dsvm - offer: ubuntu-1804 - sku: 1804-gen2 + offer: ubuntu-2004 + sku: 2004-gen2 version: latest - install_ui: false + apt_sku: 20.04 + install_ui: true conda_config: true # "Custom Image From Gallery": # source_image_name: your-image @@ -68,8 +78,6 @@ When specifying images using `source_image_name`, the image must be stored in an To enable re-using built user resource templates across environments where the image may vary, the image gallery is configured via the `RP_BUNDLE_VALUES` environment variable when deploying the TRE. The `RP_BUNDLE_VALUES` variable is a JSON object, and the `image_gallery_id` property within it identifies the image gallery that contains the images specified by `source_image_name`: - ```bash RP_BUNDLE_VALUES='{"image_gallery_id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Compute/galleries/"} ``` - diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index f36a668258..ef8971cbc6 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-linuxvm -version: 0.6.9 +version: 0.6.35 description: "An Azure TRE User Resource Template for Guacamole (Linux)" dockerfile: Dockerfile.tmpl registry: azuretre @@ -14,20 +14,31 @@ custom: "8 CPU | 32GB RAM": Standard_D8s_v5 "16 CPU | 64GB RAM": Standard_D16s_v5 image_options: - "Ubuntu 18.04": + "Ubuntu 22.04 LTS": source_image_reference: publisher: canonical - offer: ubuntuserver - sku: 18_04-lts-gen2 + offer: 0001-com-ubuntu-server-jammy + sku: 22_04-lts-gen2 version: latest + apt_sku: 22.04 install_ui: true conda_config: false - "Ubuntu 18.04 Data Science VM": + "Ubuntu 20.04 LTS": + source_image_reference: + publisher: canonical + offer: 0001-com-ubuntu-server-focal + sku: 20_04-lts-gen2 + version: latest + apt_sku: 20.04 + install_ui: false + conda_config: false + "Ubuntu 20.04 LTS Data Science VM": source_image_reference: publisher: microsoft-dsvm - offer: ubuntu-1804 - sku: 1804-gen2 + offer: ubuntu-2004 + sku: 2004-gen2 version: latest + apt_sku: 20.04 install_ui: false conda_config: true # For information on using custom images, see README.me in the guacamole/user-resources folder @@ -36,7 +47,6 @@ custom: # install_ui: true # conda_config: true - credentials: - name: azure_tenant_id env: ARM_TENANT_ID @@ -91,7 +101,7 @@ parameters: default: "public" - name: os_image type: string - default: "Ubuntu 18.04 Data Science VM" + default: "Ubuntu 22.04 LTS Data Science VM" - name: vm_size type: string default: "2 CPU | 8GB RAM" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json index 9c7aec15a3..2e13d8b6fc 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json @@ -16,8 +16,8 @@ "title": "Linux image", "description": "Select Linux image to use for VM", "enum": [ - "Ubuntu 18.04", - "Ubuntu 18.04 Data Science VM" + "Ubuntu 22.04 LTS", + "Ubuntu 20.04 LTS" ] }, "vm_size": { diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/apt_sources_config.yml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/apt_sources_config.yml index 22b3418d5b..faeaa61cd3 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/apt_sources_config.yml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/apt_sources_config.yml @@ -14,6 +14,7 @@ apt: deb [trusted=yes] $PRIMARY $RELEASE main restricted universe multiverse deb [trusted=yes] $PRIMARY $RELEASE-updates main restricted universe multiverse deb [trusted=yes] $SECURITY $RELEASE main restricted universe multiverse - deb [signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] ${nexus_proxy_url}/repository/microsoft-apt/ubuntu/18.04/prod $RELEASE main + deb [signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] ${nexus_proxy_url}/repository/microsoft-apt/ubuntu/${apt_sku}/prod $RELEASE main deb [signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] ${nexus_proxy_url}/repository/microsoft-apt/repos/edge stable main + deb [signed-by=/etc/apt/trusted.gpg.d/microsoft.gpg] ${nexus_proxy_url}/repository/microsoft-apt/repos/vscode stable main deb [signed-by=/etc/apt/trusted.gpg.d/docker-archive-keyring.gpg] ${nexus_proxy_url}/repository/docker/ $RELEASE stable diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/get_apt_keys.sh b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/get_apt_keys.sh index 5849eaede3..6e69009525 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/get_apt_keys.sh +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/get_apt_keys.sh @@ -6,6 +6,10 @@ set -o nounset # Uncomment this line to see each command for debugging (careful: this will show secrets!) # set -o xtrace +#remove key if they already exist +sudo rm -f /etc/apt/trusted.gpg.d/docker-archive-keyring.gpg || true +sudo rm -f /etc/apt/trusted.gpg.d/microsoft.gpg || true + # Get Docker Public key from Nexus curl -fsSL "${NEXUS_PROXY_URL}"/repository/docker-public-key/gpg | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/docker-archive-keyring.gpg diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf index 247c4f77e0..dfc32c1997 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf @@ -110,6 +110,7 @@ data "template_file" "vm_config" { FILESHARE_NAME = var.shared_storage_access ? data.azurerm_storage_share.shared_storage[0].name : "" NEXUS_PROXY_URL = local.nexus_proxy_url CONDA_CONFIG = local.selected_image.conda_config ? 1 : 0 + VM_USER = random_string.username.result } } @@ -131,6 +132,7 @@ data "template_file" "apt_sources_config" { template = file("${path.module}/apt_sources_config.yml") vars = { nexus_proxy_url = local.nexus_proxy_url + apt_sku = local.apt_sku } } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf index 43a6d5982b..e0281269fd 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf @@ -15,7 +15,6 @@ locals { tre_user_resource_id = var.tre_resource_id } nexus_proxy_url = "https://nexus-${data.azurerm_public_ip.app_gateway_ip.fqdn}" - # Load VM SKU/image details from porter.yaml porter_yaml = yamldecode(file("${path.module}/../porter.yaml")) vm_sizes = local.porter_yaml["custom"]["vm_sizes"] @@ -26,4 +25,5 @@ locals { # selected_image_source_refs is an array to enable easy use of a dynamic block selected_image_source_refs = lookup(local.selected_image, "source_image_reference", null) == null ? [] : [local.selected_image.source_image_reference] selected_image_source_id = lookup(local.selected_image, "source_image_name", null) == null ? null : "${var.image_gallery_id}/images/${local.selected_image.source_image_name}" + apt_sku = local.selected_image_source_refs[0]["apt_sku"] } diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh index 6d70862655..9380cbc9e1 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/pypi_sources_config.sh @@ -1,5 +1,5 @@ #!/bin/bash -sudo tee /etc/pip.conf > dev/null <<'EOF' +sudo tee /etc/pip.conf > /dev/null <<'EOF' [global] index = ${nexus_proxy_url}/repository/pypi/pypi index-url = ${nexus_proxy_url}/repository/pypi/simple diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/vm_config.sh b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/vm_config.sh index 7db5f89b82..f0d0c8a63b 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/vm_config.sh +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/vm_config.sh @@ -2,31 +2,80 @@ set -o errexit set -o pipefail -# set -o nounset +set -o nounset # Uncomment this line to see each command for debugging (careful: this will show secrets!) -# set -o xtrace +set -o xtrace # Remove apt sources not included in sources.list file sudo rm -f /etc/apt/sources.list.d/* # Update apt packages from configured Nexus sources -sudo apt-get update - -# Install xrdp so Guacamole can connect via RDP -sudo apt-get install xrdp -y +echo "init_vm.sh: START" +sudo apt update || continue +sudo apt install -y gnupg2 software-properties-common apt-transport-https wget dirmngr gdebi-core +sudo apt-get update || continue + +## Desktop +echo "init_vm.sh: Desktop" +echo /usr/sbin/gdm3 > /etc/X11/default-display-manager +DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true dpkg-reconfigure gdm3 +echo set shared/default-x-display-manager gdm3 | debconf-communicate &> /dev/null + +sudo DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true +sudo apt install -y xfce4 xfce4-goodies xorg dbus-x11 x11-xserver-utils + +## Install xrdp so Guacamole can connect via RDP +echo "init_vm.sh: xrdp" +sudo apt install -y xrdp xorgxrdp xfce4-session sudo adduser xrdp ssl-cert - -# Install desktop environment if image doesn't have one already -if [ "${INSTALL_UI}" -eq 1 ]; then - sudo apt-get install xorg xfce4 xfce4-goodies dbus-x11 x11-xserver-utils -y - echo xfce4-session > ~/.xsession -fi +sudo -u "${VM_USER}" -i bash -c 'echo xfce4-session > ~/.xsession' + +## Python 3.8 and Jupyter +sudo apt install -y jupyter-notebook + +## VS Code +echo "init_vm.sh: VS Code" +sudo apt install -y code +sudo apt install -y gvfs-bin || contine + +# echo "init_vm.sh: azure-cli" +# sudo apt install azure-cli -y + +# TODO: need to look at proxy extentions +# echo "init_vm.sh: Folders" +# sudo mkdir /opt/vscode/user-data +# sudo mkdir /opt/vscode/extensions +## VSCode Extensions +# echo "init_vm.sh: VSCode extensions" +# code --extensions-dir="/opt/vscode/extensions" --user-data-dir="/opt/vscode/user-data" --install-extension ms-python.python +# code --extensions-dir="/opt/vscode/extensions" --user-data-dir="/opt/vscode/user-data" --install-extension REditorSupport.r +# code --extensions-dir="/opt/vscode/extensions" --user-data-dir="/opt/vscode/user-data" --install-extension RDebugger.r-debugger + +## R +echo "init_vm.sh: R Setup" +# wget -q https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc -O- | sudo apt-key add - +# sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" +# sudo apt update +sudo apt install -y r-base + +## RStudio Desktop +# echo "init_vm.sh: RStudio" +# wget https://download1.rstudio.org/desktop/bionic/amd64/rstudio-2022.07.2-576-amd64.deb -P /tmp +# sudo gdebi --non-interactive /tmp/rstudio-2022.07.2-576-amd64.deb + +## Azure Storage Explorer +sudo apt install gnome-keyring -y +wget -q ${NEXUS_PROXY_URL}/microsoft-download/A/E/3/AE32C485-B62B-4437-92F7-8B6B2C48CB40/StorageExplorer-linux-x64.tar.gz -P /tmp +sudo mkdir /opt/storage-explorer +tar -xf /tmp/StorageExplorer-linux-x64.tar.gz -C /opt/storage-explorer +sudo chmod +x /opt/storage-explorer/*.sh # Fix for blank screen on DSVM (/sh -> /bash due to conflict with profile.d scripts) sudo sed -i 's|!/bin/sh|!/bin/bash|g' /etc/xrdp/startwm.sh # Make sure xrdp service starts up with the system sudo systemctl enable xrdp +sudo service xrdp restart if [ "${SHARED_STORAGE_ACCESS}" -eq 1 ]; then # Install required packages @@ -75,6 +124,7 @@ fi ### Anaconda Config if [ "${CONDA_CONFIG}" -eq 1 ]; then + echo "init_vm.sh: Anaconda" export PATH="/anaconda/condabin":$PATH export PATH="/anaconda/bin":$PATH export PATH="/anaconda/envs/py38_default/bin":$PATH @@ -85,11 +135,24 @@ if [ "${CONDA_CONFIG}" -eq 1 ]; then fi # Docker install and config +sudo apt-get remove -y moby-tini || true +sudo apt-get install -y r-base-core sudo apt-get install -y ca-certificates curl gnupg lsb-release -sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin jq +sudo apt-get install -y docker-compose-plugin docker-ce-cli containerd.io jq +sudo apt-get install -y docker-ce jq -n --arg proxy "${NEXUS_PROXY_URL}:8083" '{"registry-mirrors": [$proxy]}' > /etc/docker/daemon.json sudo systemctl daemon-reload sudo systemctl restart docker # R config sudo echo -e "local({\n r <- getOption(\"repos\")\n r[\"Nexus\"] <- \"""${NEXUS_PROXY_URL}\"/repository/r-proxy/\"\n options(repos = r)\n})" | sudo tee /etc/R/Rprofile.site + +## Add ouh_researcher group for directory permissions +echo "init_vm.sh: directory permissions" +sudo chgrp -R ouh_researcher /opt/anaconda +sudo chmod -R g+w /opt/anaconda + +# ## Cleanup +echo "init_vm.sh: Cleanup" +sudo apt -y autoremove +sudo apt install unattended-upgrades \ No newline at end of file