diff --git a/.github/workflows/virtual_workstation_checks.yml b/.github/workflows/virtual_workstation_checks.yml new file mode 100644 index 00000000..1afcaee7 --- /dev/null +++ b/.github/workflows/virtual_workstation_checks.yml @@ -0,0 +1,159 @@ +name: Virtual Workstation Checks + +# Runs the virtual_workstation sample end to end on both platforms, so a change +# to it is proven to still produce a working workstation rather than only to +# still parse. Separate from static_checks.yml, which is offline and fast. +# +# Not for the required status checks: path-filtered runs report no status, and +# every download comes from a third-party endpoint. The weekly run catches a new +# submitter or monitor release breaking the sample, since both resolve "latest". +on: + pull_request: + branches: ["mainline"] + paths: + - "utility_scripts/virtual_workstation/**" + - ".github/workflows/virtual_workstation_checks.yml" + push: + branches: ["mainline"] + paths: + - "utility_scripts/virtual_workstation/**" + - ".github/workflows/virtual_workstation_checks.yml" + schedule: + - cron: "23 9 * * 2" + workflow_dispatch: + +permissions: {} + +concurrency: + group: virtual-workstation-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + # Shape-valid but nonexistent: create-profile does not contact the URL. + MONITOR_URL: "https://ci-example.us-west-2.deadlinecloud.amazonaws.com/" + PROFILE_NAME: "ci-example-us-west-2" + +jobs: + linux: + name: Ubuntu 22.04 + runs-on: ubuntu-latest + # The sample pins Ubuntu 22.04, the last release with libwebkit2gtk-4.0-37. + # A container, not the ubuntu-22.04 runner label, which retires in 2027. + container: ubuntu@sha256:0e0a0fc6d18feda9db1590da249ac93e8d5abfea8f4c3c0c849ce512b5ef8982 + timeout-minutes: 30 + permissions: + contents: read + # A bare container resolves "sh -e {0}" for every step, which has no pipefail. + defaults: + run: + shell: bash + steps: + # Stands in for the desktop environment the sample requires. shell: sh + # because this step is what installs bash. + - name: Install prerequisites + shell: sh + run: | + set -eu + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq --no-install-recommends \ + bash git sudo ca-certificates \ + libx11-6 libxi6 libxxf86vm1 libxfixes3 libxrender1 libxext6 \ + libxkbcommon0 libsm6 libice6 libgl1 libegl1 libglu1-mesa libdbus-1-3 \ + libglib2.0-0 libfontconfig1 libfreetype6 \ + libxkbcommon-x11-0 libxcb-cursor0 libxcb-icccm4 libxcb-image0 \ + libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-shape0 \ + libxcb-xkb1 + + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Create the artist account + run: useradd --create-home artist + + - name: Run the setup script + working-directory: utility_scripts/virtual_workstation + run: ./setup_workstation_linux.sh "$MONITOR_URL" artist + + # Assert against the machine, since the script self-verifies. + - name: Verify the workstation + run: | + set -euo pipefail + /opt/blender/blender --version + sudo -u artist -H /opt/blender/blender --background --python-expr \ + 'import bpy, sys; sys.exit(0 if "deadline_cloud_blender_submitter" in bpy.context.preferences.addons.keys() else 1)' + # A login shell: the submitter sets PATH via /etc/profile.d. + sudo -u artist -H bash -lc 'deadline --version' + grep -qF "[profile ${PROFILE_NAME}]" /home/artist/.aws/config + # Empty monitor_id makes the monitor drop the profile from its picker. + grep -qE '^monitor_id[[:space:]]*=[[:space:]]*.+$' /home/artist/.aws/config + + windows: + name: Windows Server 2022 (${{ matrix.label }}) + runs-on: windows-2022 + timeout-minutes: 45 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + # 5.1 is the default on a Windows workstation and differs from 7 in how + # native stderr and Invoke-WebRequest behave. Invoked from run: rather + # than shell:, which takes no expression. + include: + - exe: powershell + label: Windows PowerShell 5.1 + - exe: pwsh + label: PowerShell 7 + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run the setup script + shell: pwsh + working-directory: utility_scripts/virtual_workstation + env: + PS_EXE: ${{ matrix.exe }} + run: | + & $env:PS_EXE -NoProfile -File .\setup_workstation_windows.ps1 $env:MONITOR_URL + if ($LASTEXITCODE -ne 0) { throw "the setup script exited $LASTEXITCODE" } + + - name: Verify the workstation + shell: pwsh + run: | + $blender = "C:\Program Files\Blender\blender.exe" + & $blender --version + if ($LASTEXITCODE -ne 0) { throw "Blender did not run" } + + # A file, not --python-expr: PowerShell drops the inner quotes. + Set-Content -Path check_addon.py -Encoding ASCII -Value @' + import bpy + import sys + + sys.exit(0 if "deadline_cloud_blender_submitter" in bpy.context.preferences.addons.keys() else 1) + '@ + & $blender --background --python check_addon.py | Out-Null + if ($LASTEXITCODE -ne 0) { throw "the Blender add-on is not registered" } + + $config = Join-Path $env:USERPROFILE ".aws\config" + if (-not (Select-String -Path $config -SimpleMatch -Quiet -Pattern "[profile $env:PROFILE_NAME]")) { + throw "the profile stanza is missing from $config" + } + # Empty monitor_id makes the monitor drop the profile from its picker. + if (-not (Select-String -Path $config -Quiet -Pattern '^monitor_id\s*=\s*.+$')) { + throw "monitor_id is empty" + } + + # The script keeps its downloads on failure so installer logs survive. + - name: Collect installer logs + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: installer-logs-${{ matrix.exe }} + path: ${{ runner.temp }}\deadline-workstation-setup + if-no-files-found: ignore + retention-days: 7 diff --git a/tests/README.md b/tests/README.md index e6050e28..c16c32cf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -29,6 +29,7 @@ suite. To run locally, install the tools listed in |------|------|-------| | Open Job Description job & environment templates | `test_openjd_templates.py` | Every standalone template with an OpenJD `specificationVersion` passes `openjd check`. | | Host configuration scripts | `test_host_configuration_scripts.py` | Byte length is within the Deadline Cloud service limit (`HostConfiguration.scriptBody` max **15000**). Linux (`*.sh`) scripts pass `bash -n`, and Windows (`*.ps1`) scripts parse with the PowerShell parser. | +| Utility scripts | `test_utility_scripts.py` | Linux (`*.sh`) scripts pass `bash -n`, and Windows (`*.ps1`) scripts parse with the PowerShell parser. Unlike a host configuration script, a utility script runs on a workstation rather than being uploaded to the service, so no `scriptBody` length limit applies. Some also run end to end in their own workflow. See [Beyond parsing](#beyond-parsing). | | Queue environments | `test_openjd_templates.py` | Serialized `environment-2023-09` templates are within the service limit for `EnvironmentTemplate` (max **15000**). | | CloudFormation templates | `test_cloudformation.py` | Templates parse as CloudFormation YAML (intrinsic tags such as `!Sub`/`!Ref` supported) and pass `cfn-lint` (errors only). | | CDK apps | `test_cdk.py` | A queue environment copied into a CDK app is byte-identical to its original under `queue_environments/`. Everything else about a CDK app is proven by building it. See [Why so little here for CDK?](#why-so-little-here-for-cdk) | @@ -42,6 +43,21 @@ before rendering. The full recipe is still validated, and only the intentionally-blank checksum field is normalized. Genuinely invalid recipes (unknown fields, bad structure) still fail. +### Beyond parsing + +Parsing is the most this offline suite can prove about a script it must not run: +these install system packages and download roughly 1 GB, so executing one here +would defeat the "fast and offline" property the whole suite depends on. + +Where a script is worth proving further, that belongs in its own workflow. The +[virtual workstation](../utility_scripts/virtual_workstation/) sample is run end +to end by +[`virtual_workstation_checks.yml`](../.github/workflows/virtual_workstation_checks.yml) +on Ubuntu 22.04 and Windows Server 2022, which asserts against the resulting +machine rather than against the script's own output. It is path-filtered to that +sample and also runs weekly, because the submitter and monitor it installs are +resolved as "latest" and can change with no commit here. + ### Why so little here for CDK? The CDK samples are TypeScript, and they are validated by building them: the diff --git a/tests/conftest.py b/tests/conftest.py index 7301fcf9..768a3830 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -125,6 +125,25 @@ def find_host_configuration_scripts() -> list[Path]: return sorted(set(scripts)) +def find_utility_scripts() -> list[Path]: + """Shell / PowerShell scripts under ``utility_scripts/``. + + These are run by an administrator on a workstation rather than uploaded to the + service, so the host configuration length limit does not apply to them. They + still get the same syntax checks, since a sample that does not parse is broken + for everyone who copies it. + """ + base = REPO_ROOT / "utility_scripts" + if not base.is_dir(): + return [] + scripts = [] + for pattern in ("*.sh", "*.ps1"): + for path in base.rglob(pattern): + if not _is_excluded(path.relative_to(REPO_ROOT)): + scripts.append(path) + return sorted(set(scripts)) + + def find_cloudformation_templates() -> list[Path]: """CloudFormation templates (YAML files under ``cloudformation/``). diff --git a/tests/test_utility_scripts.py b/tests/test_utility_scripts.py new file mode 100644 index 00000000..3845521e --- /dev/null +++ b/tests/test_utility_scripts.py @@ -0,0 +1,75 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""Syntax checks for the standalone scripts under ``utility_scripts/``. + +Unlike a host configuration script, these are not uploaded to the service, so the +``scriptBody`` length limit does not apply. What does apply is that they parse: a +sample that does not is broken for everyone who copies it, and these run as root or +an administrator, where a syntax error can surface halfway through an install. + +The same reasoning as ``test_host_configuration_scripts.py`` applies to the tools -- +``bash`` and ``pwsh`` are required, and a missing one fails rather than skips, +because a skipped check is indistinguishable from a passing one. +""" +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +from conftest import find_utility_scripts, rel, require_tool + +_SCRIPTS = find_utility_scripts() +_SHELL_SCRIPTS = [s for s in _SCRIPTS if s.suffix == ".sh"] +_POWERSHELL_SCRIPTS = [s for s in _SCRIPTS if s.suffix == ".ps1"] + + +def test_utility_scripts_discovered(): + assert _SCRIPTS, "no utility scripts were discovered" + + +@pytest.mark.parametrize("script", _SCRIPTS, ids=rel) +def test_script_is_not_empty(script: Path): + assert script.read_text(encoding="utf-8", errors="replace").strip(), f"{rel(script)} is empty" + + +@pytest.mark.parametrize("script", _SHELL_SCRIPTS, ids=rel) +def test_shell_script_syntax(script: Path): + """`bash -n` catches syntax errors without executing anything.""" + bash = require_tool("bash", "install bash (present by default on Linux/macOS)") + result = subprocess.run( + [bash, "-n", str(script)], capture_output=True, text=True, timeout=30 + ) + assert result.returncode == 0, ( + f"bash syntax check failed for {rel(script)}:\n{result.stderr}" + ) + + +@pytest.mark.parametrize("script", _POWERSHELL_SCRIPTS, ids=rel) +def test_powershell_script_syntax(script: Path): + """Parse each PowerShell script with the PowerShell parser (no execution).""" + pwsh = require_tool( + "pwsh", + "install PowerShell (https://learn.microsoft.com/powershell/); " + "pre-installed on GitHub-hosted runners", + ) + # The script path goes through an environment variable rather than being + # interpolated into the command, so it cannot be interpreted as PowerShell. + ps_command = ( + "$p = $env:PWSH_TARGET_SCRIPT; $errors = $null; " + "[System.Management.Automation.Language.Parser]::ParseFile(" + "$p, [ref]$null, [ref]$errors) | Out-Null; " + "if ($errors) { $errors | ForEach-Object { Write-Output $_.ToString() }; exit 1 } " + "else { exit 0 }" + ) + result = subprocess.run( + [pwsh, "-NoProfile", "-NonInteractive", "-Command", ps_command], + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "PWSH_TARGET_SCRIPT": str(script)}, + ) + assert result.returncode == 0, ( + f"PowerShell parse failed for {rel(script)}:\n{result.stdout}\n{result.stderr}" + ) diff --git a/utility_scripts/README.md b/utility_scripts/README.md index 03826157..b72ab00e 100644 --- a/utility_scripts/README.md +++ b/utility_scripts/README.md @@ -9,6 +9,7 @@ This table covers every immediate user-selectable sample directory in `utility_s | Sample | What it demonstrates | Start here when | |---|---|---| | [Upload to job attachments](upload_to_job_attachments/) | Uploading files into content-addressable job attachment storage with deduplication | Large or reused datasets should be staged before job submission | +| [Virtual workstation](virtual_workstation/) | Provisioning a Linux or Windows workstation with a DCC, the Deadline Cloud submitter, and a pre-configured monitor profile | Artists should find a submission-ready machine and only need to sign in | ## Upload to job attachments @@ -38,6 +39,21 @@ python upload_to_job_attachments/upload_to_job_attachments.py \ See the [sample README](upload_to_job_attachments/) for installation, permissions, options, and manifest details. +## Virtual workstation + +Example scripts for Linux and Windows prepare a workstation for Deadline Cloud submission. Each one installs Blender, then installs the Deadline Cloud submitter and monitor through their silent installers. It finishes by creating a monitor profile non-interactively, so an artist only has to sign in. + +```console +# Linux, as root. Add the artist's account when there is no SUDO_USER to infer, +# as under EC2 user data. +sudo virtual_workstation/setup_workstation_linux.sh https://mystudio.us-west-2.deadlinecloud.amazonaws.com/ + +# Windows, in an elevated PowerShell session as the artist's own account +.\virtual_workstation\setup_workstation_windows.ps1 https://mystudio.us-west-2.deadlinecloud.amazonaws.com/ +``` + +Blender stands in for whichever DCC you run. See the [sample README](virtual_workstation/) for prerequisites, adapting the scripts to another DCC, and cleanup. + ## Additional resources * [AWS Deadline Cloud user guide](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/index.html) diff --git a/utility_scripts/virtual_workstation/README.md b/utility_scripts/virtual_workstation/README.md new file mode 100644 index 00000000..d426da90 --- /dev/null +++ b/utility_scripts/virtual_workstation/README.md @@ -0,0 +1,250 @@ +# Virtual workstation setup + +Example scripts that turn a fresh Linux or Windows workstation into an AWS Deadline Cloud submission machine. An artist who logs in finds Blender and the Deadline Cloud submitter installed, alongside Deadline Cloud monitor with a profile already configured. The only remaining step is signing in. + +Treat each script as a worked example to copy and adapt. Each takes one argument and keeps its settings as constants at the top, so the whole flow reads top to bottom. + +## What this sample demonstrates + +How to complete the workstation setup that normally requires a person clicking through installers and a monitor sign-in dialog: + +* Installing Blender from an official release archive. +* Installing the Deadline Cloud submitter with its silent installer. +* Enabling the submitter's Blender add-on, which the silent installer alone does not do. +* Installing Deadline Cloud monitor. +* Creating a monitor profile non-interactively with `deadline-cloud-monitor create-profile`, so the profile exists before anyone signs in. + +Run either script during provisioning, from EC2 user data, during an AMI or image bake, or by hand on a workstation VM. + +Blender stands in for whichever DCC you run. It is used here because it installs unattended from a public archive with no license server, which keeps the example runnable as-is. See [Adapting to another DCC](#adapting-to-another-dcc). + +## Prerequisites + +* Ubuntu 22.04 or a Windows image, with a desktop environment already present because the scripts do not install one. Blender, the submitter GUI, and the monitor are all desktop applications. An AWS Deadline Cloud base image, a NICE DCV workstation, or a Windows Server image with the Desktop Experience all work. + + Deadline Cloud monitor's `.deb` depends on `libwebkit2gtk-4.0-37`, which Ubuntu 24.04 no longer publishes; it carries `libwebkit2gtk-4.1-0` instead, and no official repository offers the 4.0 build for it. That is why this example pins Ubuntu 22.04. The Linux script checks for the package up front and stops with an explanation rather than failing partway through. + + On a newer release, install the submitter without the monitor and authenticate a different way. `deadline auth login` is not an alternative, because it drives the monitor and only accepts profiles the monitor created. Use an ordinary AWS credential source instead, such as an IAM Identity Center profile created with `aws configure sso` or an instance profile, and delete the monitor and profile steps from the script. The artist then signs in through that mechanism rather than the monitor, so what this sample pre-configures no longer applies. +* An x86-64 host. Blender's archive, Deadline Cloud monitor, and the `libssl1.1` package the Linux script fetches are all pinned to x86-64, so an arm64 instance such as Graviton needs those three substituted. +* Administrator access, and on Windows it has to be **the artist's own account**. Windows cannot write another user's per-user state without that user's password, so the script has to run as an administrator and as the account that signs in, both at once. It refuses to run as `SYSTEM` for the same reason. That means the artist's account needs to be in the local `Administrators` group. Linux only needs `root`, since it writes the per-user state with `runuser`. + + Where artists are standard users, split the Windows script in two. Run the Blender, submitter, and monitor installers under any administrator account. Then run only the add-on step and `create-profile` as the artist, without administrator rights. The monitor installs per user into `%LOCALAPPDATA%` and `create-profile` writes to `%USERPROFILE%`, so those two steps do not need them. +* Outbound HTTPS to `downloads.deadlinecloud.amazonaws.com` and to the Blender mirror. +* A working default web browser. Deadline Cloud monitor hands off to it to complete sign-in, so without one the artist sees "Failed to execute default Web Browser". Windows Server images normally include Microsoft Edge, so nothing extra is needed there. On Ubuntu 22.04 and later, `apt install firefox` gets a transitional package that installs the Firefox snap, and snaps do not work in every remote-desktop session. Install Firefox from the [Mozilla apt repository](https://support.mozilla.org/kb/install-firefox-linux) instead, and add an apt pin so the `.deb` wins over Ubuntu's snap transitional package. Verified on Ubuntu 22.04: the Mozilla `.deb` completes sign-in in a VNC session. +* Your monitor URL, from the **Monitors** page of the Deadline Cloud console. It must include the Region segment, as in `https://mystudio.us-west-2.deadlinecloud.amazonaws.com/`. +* No AWS credentials. The scripts call no AWS APIs. + +The Linux script was written and tested against Ubuntu 22.04 on x86-64 only. Other Debian-family releases are likely to work, since the script uses nothing Ubuntu-specific beyond `apt-get` and the `libssl1.1` package it fetches. On a non-Debian distribution, replace the `apt-get` calls, install the monitor from its `.rpm` rather than the `.deb`, and satisfy OpenSSL 1.1 the way that distribution expects. + +Both scripts are run end to end in CI by [`virtual_workstation_checks.yml`](../../.github/workflows/virtual_workstation_checks.yml), on Ubuntu 22.04 and on Windows Server 2022 under both Windows PowerShell 5.1 and PowerShell 7, whenever this sample changes and once a week. The weekly run catches a new submitter or monitor release breaking the sample, since both are resolved as "latest" rather than pinned. + +## Run + +Linux, as root. Under `sudo` the artist's account is inferred from `SUDO_USER`: + +```console +sudo ./setup_workstation_linux.sh https://mystudio.us-west-2.deadlinecloud.amazonaws.com/ +``` + +Name the account explicitly when provisioning runs as `root` with nothing to infer from, which includes EC2 user data and an AMI bake. **Pass it there**, because the profile and Blender's add-on preferences are per user: without it the script configures `root` and the artist finds nothing set up. + +```console +./setup_workstation_linux.sh https://mystudio.us-west-2.deadlinecloud.amazonaws.com/ artist +``` + +Windows, in an elevated PowerShell session **as the artist's own account**. Start PowerShell with **Run as administrator** first: the script declares `#Requires -RunAsAdministrator`, so launching it from an unelevated shell fails with `ScriptRequiresElevation` rather than prompting. + +```console +.\setup_workstation_windows.ps1 https://mystudio.us-west-2.deadlinecloud.amazonaws.com/ +``` + +The monitor, its profile, and Blender's add-on preferences are all per user. Linux writes them for another account with `runuser`, but Windows cannot do so without that account's password, so the Windows script has no equivalent of the second argument. + +On Windows, running the script through a mechanism that executes as `SYSTEM` rather than as a user, such as Systems Manager Run Command or an EC2 user data script, writes the profile and Blender preferences into a service profile the artist never logs in to. The artist then sees no pre-configured monitor. Run it as the artist's own account: interactively, or as a scheduled task created with `/RU /RL HIGHEST`. + +After either script finishes, the artist signs in through a desktop session on the machine. The scripts install no desktop or remote-access server, so provide one separately. + +## How it works + +Both scripts run the same five steps, in the same order, under section headers that name each one. The Linux script has one extra section, `Prerequisites`, covering the packages and OpenSSL 1.1 described below. + +1. **Validate the monitor URL** and derive the Region, the subdomain, and the profile name (`-`). +2. **Install Blender** from the official archive, verified against its published checksum, into a fixed prefix (`/opt/blender` or `C:\Program Files\Blender`). +3. **Install the submitter** from its `latest` URL, verify its checksum, and run it with `--mode unattended`. +4. **Enable the Blender add-on.** The silent install stages the add-on but cannot enable it, because add-ons live in Blender's per-user preferences while the install runs at system scope. The scripts run the installer's own `add_submitter_to_pref.py` through Blender in background mode, then read the preferences back to confirm. +5. **Install the monitor and create the profile** with `create-profile`, a non-GUI subcommand that writes the profile and exits without needing a display. + +Every download is verified against a published SHA-256 checksum, and the scripts fail if a checksum cannot be fetched. An internal Blender mirror must also serve Blender's `blender-.sha256` manifest. + +### Download links + +Both the submitter and the monitor publish a `latest` path per platform that always serves the current release, each with a `.sha256` beside it. The scripts use the two that apply to them. The rest are here for adapting to another platform. + +| Component | Platform | URL, under `https://downloads.deadlinecloud.amazonaws.com/` | +|---|---|---| +| Submitter | Linux | `submitters/latest/linux/DeadlineCloudSubmitter-linux-x64-installer.run` | +| Submitter | Windows | `submitters/latest/windows/DeadlineCloudSubmitter-windows-x64-installer.exe` | +| Submitter | macOS | `submitters/latest/macos/DeadlineCloudSubmitter-osx-installer.app.zip` | +| Monitor | Debian family | `dcm/latest/deadline-cloud-monitor_amd64.deb` | +| Monitor | RPM family | `dcm/latest/deadline-cloud-monitor.x86_64.rpm` | +| Monitor | Linux, generic | `dcm/latest/deadline-cloud-monitor_amd64.AppImage` | +| Monitor | Windows | `dcm/latest/DeadlineCloudMonitor_x64-setup.exe` | +| Monitor | macOS, Intel | `dcm/latest/Deadline Cloud Monitor x64.dmg` | +| Monitor | macOS, Apple silicon | `dcm/latest/Deadline Cloud Monitor aarch64.dmg` | + +Append `.sha256` to any of these for its checksum. + +The Linux script also installs `libssl1.1`, because Deadline Cloud monitor links against OpenSSL 1.1 while no current Ubuntu release provides it. Ubuntu 20.04 is the last release to carry the package, so the script takes it from the Ubuntu archive. That one artifact is published without a `.sha256` beside it, so its expected hash is a constant at the top of the script alongside the version, with a comment naming the index to read a newer hash from. + +On another distribution, prefer whatever OpenSSL 1.1 package your own repositories provide and delete that step, rather than installing an Ubuntu-built `.deb` elsewhere. + +### The profile + +`create-profile` writes an AWS profile that resolves credentials through the monitor rather than through IAM Identity Center stanzas: + +```ini +[profile mystudio-us-west-2] +region=us-west-2 +credential_process=cat "/home/artist/.cache/com.amazonaws.deadline.monitor/credentials_mystudio-us-west-2.json" +user_id= +identity_store_id= +monitor_id=pending-first-login +``` + +The scripts pass two further flags, both optional: + +* `--set-as-deadline-default` points the Deadline Cloud CLI at this profile, by writing `aws_profile_name` under `[defaults]` in `~/.deadline/config`. Without it, `deadline` commands need `--profile` or `AWS_PROFILE`. Drop it on a workstation that submits to more than one monitor. +* `--enable-auto-login` starts sign-in as soon as the monitor opens, rather than making the artist pick the profile first. Keep it unless you want the picker, since skipping the picker is most of what pre-configuring the profile buys. + +On Windows the same profile instead delegates to the monitor executable: + +```ini +credential_process="C:\Users\artist\AppData\Local\DeadlineCloudMonitor\DeadlineCloudMonitor.exe" get-credentials --profile mystudio-us-west-2 +``` + +The placeholder and empty fields are expected. `create-profile` requires a `--monitor-id`, but the real ID cannot be discovered without AWS credentials, so the scripts pass `pending-first-login`. The monitor replaces it, along with `user_id` and `identity_store_id`, with authoritative values from the portal at the artist's first sign-in. Either form of `credential_process` yields no credentials until that sign-in happens, so it is the intended remaining step. + +The placeholder must be non-empty. An empty `--monitor-id` makes the monitor drop the profile from its picker and fall back to asking for the monitor URL, which defeats the point of pre-configuring it. The value is shown verbatim in the monitor's profile list until first sign-in, so it reads as a status rather than looking like a real ID. + +Because the cache path is written into the profile at creation time and lives under the invoking user's home directory, the profile only works for the account it was created for. + +### Adapting to another DCC + +Everything Deadline Cloud does is identical for every DCC, so switching to Maya, Nuke, Houdini, 3ds Max, Cinema 4D, After Effects, or VRED means changing three things, called out in comments in both scripts: + +1. **The component names** (`SUBMITTER_COMPONENT` and `BLENDER_COMPONENT`, or `$SubmitterComponent` and `$BlenderComponent`), such as `deadline_cloud_for_houdini` plus `houdini_20_5`. Run ` --help` for the current `--enable-components` values. The `---path` flag is derived from the version component, so it follows automatically. +2. **The Blender install step.** Commercial DCCs need a vendor installer and, in most cases, a license server, so replace that block entirely. Also update the install prefix constant. +3. **The add-on enable step.** It is Blender-specific, including the `Submitters/Blender/` paths and the `deadline_cloud_blender_submitter` name it verifies. Other DCCs are wired up by the installer itself or by an environment variable such as `MAYA_MODULE_PATH` or `NUKE_PATH`, so you can often delete it. + +Note that the submitter installer's `---path` flag takes the DCC executable on Windows but the install directory on Linux. + +To install more than one DCC, pass a comma-separated `--enable-components` list with every DCC and version component you need, one `---path` flag each, and repeat step 2 per DCC. + +## What ends up on the machine + +| Item | Linux | Windows | +|---|---|---| +| Blender | `/opt/blender`, symlinked to `/usr/local/bin/blender` | `C:\Program Files\Blender` | +| Submitter and Deadline Cloud CLI | `/opt/DeadlineCloudSubmitter` | `C:\Program Files\DeadlineCloudSubmitter` | +| Monitor | `/usr/bin/deadline-cloud-monitor` (system-wide) | `%LOCALAPPDATA%\DeadlineCloudMonitor` (per user) | +| AWS profile | `~/.aws/config` | `%USERPROFILE%\.aws\config` | +| Deadline Cloud CLI config | `~/.deadline/config` | `%USERPROFILE%\.deadline\config` | + +The submitter installer puts the `deadline` CLI on `PATH` itself. On Linux it writes `/etc/profile.d/deadline.sh`, which appends `/opt/DeadlineCloudSubmitter/DeadlineClient`, so the CLI appears in new login shells rather than the one that ran the script. + +## Security, cost, and cleanup + +* **No credentials are stored, and none are needed.** The scripts write no secrets and call no AWS APIs. The profile delegates to the monitor, which acquires credentials only when the artist signs in interactively. +* **Every installer is checksum-verified**, and verification cannot be skipped. If you mirror Blender internally, serve its checksum manifest too and point the mirror constant at it. +* **Licensing.** Blender is distributed under the GNU GPL. Review its terms for your use. +* **Cost.** The scripts create no AWS resources. Running the workstation is billable, and jobs submitted from it are billed normally. +* **Cleanup.** Uninstall the submitter, then the monitor, then delete the Blender prefix: + + ```console + # Linux + sudo /opt/DeadlineCloudSubmitter/uninstall --mode unattended + sudo rm -rf /opt/DeadlineCloudSubmitter # the uninstaller leaves THIRD_PARTY_LICENSES behind + sudo apt-get remove -y deadline-cloud-monitor + sudo rm -rf /opt/blender /usr/local/bin/blender + + # Windows + & "C:\Program Files\DeadlineCloudSubmitter\uninstall.exe" --mode unattended + Remove-Item -Recurse -Force "C:\Program Files\Blender" + ``` + + The Linux uninstaller removes `/etc/profile.d/deadline.sh`, so the `deadline` CLI leaves new login shells. Remove the monitor on Windows through **Settings > Apps > Installed apps**. Then remove the profile stanza from `~/.aws/config` and the `[defaults]` entry from `~/.deadline/config`, and delete the monitor's credential cache (`~/.cache/com.amazonaws.deadline.monitor` on Linux). + +## Troubleshooting + +**Blender downloads fail with HTTP 403.** `download.blender.org` rejects some automated clients, so the scripts default to a mirror. Pick another from [mirror.blender.org](https://mirror.blender.org/), or host the archive and its checksum manifest internally. + +**The script stops with "exists but holds no blender executable."** A previous run was interrupted partway through installing Blender, leaving the prefix incomplete. The guard refuses to delete a prefix it cannot recognize as one of its own, because that constant is meant to be edited and deleting it unconditionally as root would destroy whatever it names. Confirm the path is the one you intended, then remove it and re-run: + +```console +# Linux +sudo rm -rf /opt/blender + +# Windows +Remove-Item -Recurse -Force "C:\Program Files\Blender" +``` + +Interrupted runs do not cause this any more: Blender is unpacked to a staging directory beside the prefix and moved into place, so the prefix only ever exists complete. + +**The add-on step fails with `qtpy.QtBindingsNotFoundError: No Qt bindings could be found`.** The bindings are present: the submitter bundles PySide6. That message is `qtpy` reporting an `ImportError` it could not attribute, and the real cause is a system library that PySide6 links against and this image does not have. On a minimal Ubuntu 22.04 image, the missing packages are: + +```console +sudo apt-get install -y libglib2.0-0 libfontconfig1 libfreetype6 \ + libxkbcommon-x11-0 libxcb-cursor0 libxcb-icccm4 libxcb-image0 \ + libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-xkb1 +``` + +To see the actual cause rather than the `qtpy` summary, run `ldd` over the bundled Qt and look for `not found`: + +```console +ldd /opt/DeadlineCloudSubmitter/Submitters/Blender/python/modules/PySide6/QtCore.abi3.so | grep "not found" +``` + +Ignore the `libQt6*.so.6` entries there: those resolve within the bundle at load time. A full desktop environment provides every one of these packages, so the failure only appears on an image that has no desktop. It is the same class of failure as Blender's own missing X11 and GL libraries, which the script reports directly. + +**The Deadline Cloud menu is missing in Blender.** Add-ons register per user, so confirm the script ran for the account that is signing in. On Linux that is the second argument. On Windows it is the account that ran the script. To check, as that same user: + +```console +blender --background --python-expr 'import bpy; print("deadline_cloud_blender_submitter" in bpy.context.preferences.addons.keys())' +``` + +On Windows, write the same two lines to a file and pass `--python ` instead. Windows PowerShell does not preserve the inner quotes of an expression passed on the command line, so `--python-expr` raises `NameError` there. The script itself uses a file for this reason. Blender is not on `PATH`, so call it by path: + +```console +& 'C:\Program Files\Blender\blender.exe' --background --python C:\Temp\check_addon.py +``` + +**Deadline Cloud monitor does not appear in the applications menu.** Its desktop entry declares no menu category, so some desktop environments file it nowhere. Launch it by path instead, or add a launcher of your own: + +```console +# Linux +deadline-cloud-monitor + +# Windows +& "$env:LOCALAPPDATA\DeadlineCloudMonitor\DeadlineCloudMonitor.exe" +``` + +**The monitor asks for a monitor URL instead of using the profile.** The monitor found no usable profile, most often for one of these reasons: + +* The profile went to a different account than the one signing in. On Windows, running the script as `SYSTEM` produces exactly that. Check that the stanza is in the signing-in user's own `~/.aws/config` or `%USERPROFILE%\.aws\config`. +* The profile's `monitor_id` is empty, which makes the monitor drop it from the picker. The scripts always write a non-empty placeholder, so this points to a profile created by hand. Recreate it with a non-empty `--monitor-id` as described under [The profile](#the-profile). + +**Submission fails with a credentials error.** Expected until the artist signs in to the monitor once. Check with `deadline auth status`, which reports `NEEDS_LOGIN` before sign-in and `AUTHENTICATED` after. + +**On Windows, the script cannot find the monitor after installing it.** The installer honors WOW64 redirection, so under a 32-bit host process it installs into `C:\Windows\SysWOW64\config\systemprofile\AppData\Local\DeadlineCloudMonitor\` even though `%LOCALAPPDATA%` points elsewhere, and the `InstallLocation` it records still names `System32`. The script tries the recorded path, its `SysWOW64` equivalent, and `%LOCALAPPDATA%`, and reports every candidate when none exists. To find it by hand: + +```console +Get-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*" | + Where-Object { $_.DisplayName -eq "DeadlineCloudMonitor" } | + Select-Object InstallLocation +``` + +## Related resources + +* [Set up your workstation](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/submitter.html) +* [Deadline Cloud CLI](https://github.com/aws-deadline/deadline-cloud) +* [Blender download mirror](https://mirror.blender.org/) diff --git a/utility_scripts/virtual_workstation/setup_workstation_linux.sh b/utility_scripts/virtual_workstation/setup_workstation_linux.sh new file mode 100755 index 00000000..75eadb97 --- /dev/null +++ b/utility_scripts/virtual_workstation/setup_workstation_linux.sh @@ -0,0 +1,337 @@ +#!/usr/bin/env bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 +# +# Example: pre-configure a Linux workstation for AWS Deadline Cloud submission. +# +# Installs Blender, the Deadline Cloud submitter, and Deadline Cloud monitor, +# then creates a monitor profile so an artist only has to sign in. +# +# This is a worked example rather than a general-purpose tool. It targets Ubuntu +# 22.04 on x86-64, which is the last release carrying the libwebkit2gtk-4.0-37 +# that Deadline Cloud monitor needs. Edit the constants below for your +# environment. Run as root during provisioning (EC2 user data, an AMI bake, or by +# hand). +# +# Usage: setup_workstation_linux.sh MONITOR_URL [WORKSTATION_USER] +# +# MONITOR_URL https://..deadlinecloud.amazonaws.com/ +# WORKSTATION_USER Account that signs in to the monitor. The profile is written +# to this user's home directory. Defaults to SUDO_USER when +# run under sudo. Required otherwise, including under EC2 user +# data and in an AMI bake, where there is no account to infer. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Edit these for your environment +# --------------------------------------------------------------------------- + +BLENDER_VERSION="4.5.0" + +# The submitter's installer components for this DCC: the submitter plug-in itself, +# and the specific DCC version it integrates with. Both change together when you +# switch DCC; see "Adapting to another DCC". +SUBMITTER_COMPONENT="deadline_cloud_for_blender" +BLENDER_COMPONENT="blender_45" + +# download.blender.org rejects some automated clients, so this points at +# Blender's official mirror redirector, which forwards to a nearby mirror. +# Point it at an internal mirror if you host the archives yourself. +BLENDER_MIRROR="https://mirror.blender.org/release" + +BLENDER_PREFIX="/opt/blender" +SUBMITTER_PREFIX="/opt/DeadlineCloudSubmitter" + +DOWNLOADS_BASE="https://downloads.deadlinecloud.amazonaws.com" + +# Deadline Cloud monitor links against OpenSSL 1.1, which no current Ubuntu +# release provides. Ubuntu 20.04 is the last release to carry libssl1.1, so +# install that package here. Pinned to a specific build and checksum: it is not +# published with a .sha256 alongside it, so the expected hash lives here. Take a +# newer hash from the "SHA256:" field for libssl1.1 in +# https://archive.ubuntu.com/ubuntu/dists/focal-updates/main/binary-amd64/Packages.gz +LIBSSL_DEB="libssl1.1_1.1.1f-1ubuntu2.24_amd64.deb" +LIBSSL_DEB_SHA256="7cf39d70a639017d1dd7c8d36daa2258063608688e449fddf40ffdd46f992a78" + +# --------------------------------------------------------------------------- +# Adapting to another DCC +# --------------------------------------------------------------------------- +# +# Blender stands in for whichever DCC you run. It is used here because it +# installs unattended from a public archive with no license server, which keeps +# this example runnable as-is. Everything Deadline Cloud does is identical for +# every DCC, so switching to Maya, Nuke, Houdini, 3ds Max, Cinema 4D, After +# Effects, or VRED means changing three things: +# +# 1. SUBMITTER_COMPONENT and BLENDER_COMPONENT above, for example +# deadline_cloud_for_houdini plus houdini_20_5. Run " --help" for +# the current --enable-components values. The ---path flag is derived +# from BLENDER_COMPONENT, so it follows automatically. +# 2. The "Install Blender" step. Commercial DCCs need a vendor installer and +# usually a license server, so replace that block entirely. +# 3. The "Enable the add-on in Blender" step. It is Blender-specific. Other +# DCCs are wired up by the installer itself or by an environment variable +# such as MAYA_MODULE_PATH or NUKE_PATH, so you can often delete it. + +log() { printf '[setup-workstation] %s\n' "$*"; } +die() { printf '[setup-workstation] ERROR: %s\n' "$*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Arguments +# --------------------------------------------------------------------------- + +MONITOR_URL="${1:-}" +[[ -n "$MONITOR_URL" ]] || die "usage: $0 MONITOR_URL [WORKSTATION_USER]" + +[[ $EUID -eq 0 ]] || die "run as root: this installs system packages" + +# The profile and Blender's add-on preferences are per user, so pass the artist's +# account when there is no SUDO_USER to infer, as under EC2 user data. Configuring +# root instead produces a workstation where the artist finds nothing set up. +WORKSTATION_USER="${2:-${SUDO_USER:-root}}" + +# The URL must carry its Region segment: the monitor accepts one without it and +# then writes a profile with the wrong region. +[[ "$MONITOR_URL" =~ ^https://([a-z0-9-]+)\.([a-z0-9-]+)\.deadlinecloud\.amazonaws\.com/?$ ]] \ + || die "monitor URL must be https://..deadlinecloud.amazonaws.com/ (got: $MONITOR_URL)" +MONITOR_REGION="${BASH_REMATCH[2]}" +PROFILE_NAME="${BASH_REMATCH[1]}-${MONITOR_REGION}" + +USER_HOME="$(getent passwd "$WORKSTATION_USER" | cut -d: -f6)" +[[ -n "$USER_HOME" ]] || die "user does not exist: $WORKSTATION_USER" + +log "workstation user: $WORKSTATION_USER ($USER_HOME)" +log "monitor profile: $PROFILE_NAME" + +# --------------------------------------------------------------------------- +# Prerequisites +# --------------------------------------------------------------------------- + +DEBIAN_FRONTEND=noninteractive apt-get update -qq + +# Deadline Cloud monitor's .deb needs libwebkit2gtk-4.0-37, which Ubuntu dropped +# after 22.04 in favor of the 4.1 build. Check before installing anything, since +# otherwise this fails at apt dependency resolution after a 1 GB download. +# apt-cache policy, not show, which also succeeds for a virtual package. Capture +# first: piping into grep -q kills apt-cache with SIGPIPE, and under pipefail a +# package that is present looks missing. +webkit_policy="$(apt-cache policy libwebkit2gtk-4.0-37 2>/dev/null)" +if ! grep -q 'Candidate: [0-9]' <<<"$webkit_policy"; then + die "Deadline Cloud monitor needs libwebkit2gtk-4.0-37, which this image does not provide. Use Ubuntu 22.04." +fi + +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ca-certificates curl xz-utils + +WORK_DIR="$(mktemp -d)" + +# Remove the ~1 GB of downloads on success, keep them on failure so the installer +# logs survive for diagnosis. +cleanup() { + local status=$? + if [[ $status -eq 0 ]]; then + rm -rf "$WORK_DIR" + else + printf '[setup-workstation] downloads left in %s\n' "$WORK_DIR" >&2 + fi +} +trap cleanup EXIT + +# Download a file and verify it against a published sha256. Verification is not +# optional: an unreachable checksum is an error, not a reason to skip the check. +# Pass a filename to select one line from a multi-file checksum manifest. +download_verified() { + local url="$1" dest="$2" checksum_url="$3" match_name="${4:-}" body expected actual + + log "downloading ${url##*/}" + curl -fsSL --retry 3 --retry-delay 2 -o "$dest" "$url" \ + || die "cannot download ${dest##*/} from $url" + + body="$(curl -fsSL --retry 3 --retry-delay 2 "$checksum_url")" \ + || die "cannot fetch the checksum for ${dest##*/} from $checksum_url" + + if [[ -n "$match_name" ]]; then + expected="$(awk -v w="$match_name" '$2 == w || $2 == "./" w {print $1; exit}' <<<"$body")" + else + expected="$(awk 'NR==1 {print $1}' <<<"$body")" + fi + [[ "$expected" =~ ^[0-9a-fA-F]{64}$ ]] || die "no usable sha256 for ${dest##*/} in $checksum_url" + + actual="$(sha256sum "$dest" | awk '{print $1}')" + [[ "${actual,,}" == "${expected,,}" ]] \ + || die "checksum mismatch for ${dest##*/} (expected $expected, got $actual)" + log "verified ${dest##*/}" +} + +# Verify a file against a checksum given directly, for artifacts published +# without a .sha256 of their own. +verify_sha256() { + local path="$1" expected="$2" actual + actual="$(sha256sum "$path" | awk '{print $1}')" + [[ "${actual,,}" == "${expected,,}" ]] \ + || die "checksum mismatch for ${path##*/} (expected $expected, got $actual)" + log "verified ${path##*/}" +} + +# Install OpenSSL 1.1 for Deadline Cloud monitor. The monitor's .deb declares no +# SSL dependency, so a missing libssl.so.1.1 installs fine and then cannot start. +# Capture first rather than piping into grep -q: under pipefail, grep -q exits on +# the first match and ldconfig dies with SIGPIPE, so a library that is present +# looks missing. +ldconfig_libs="$(ldconfig -p)" +if ! grep -qF 'libssl.so.1.1' <<<"$ldconfig_libs"; then + log "installing libssl1.1 for Deadline Cloud monitor" + curl -fsSL --retry 3 --retry-delay 2 -o "$WORK_DIR/$LIBSSL_DEB" \ + "https://archive.ubuntu.com/ubuntu/pool/main/o/openssl/$LIBSSL_DEB" + verify_sha256 "$WORK_DIR/$LIBSSL_DEB" "$LIBSSL_DEB_SHA256" + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "$WORK_DIR/$LIBSSL_DEB" +fi + +# --------------------------------------------------------------------------- +# Install Blender +# --------------------------------------------------------------------------- + +blender_series="${BLENDER_VERSION%.*}" +blender_archive="blender-${BLENDER_VERSION}-linux-x64.tar.xz" + +download_verified \ + "${BLENDER_MIRROR}/Blender${blender_series}/${blender_archive}" \ + "$WORK_DIR/$blender_archive" \ + "${BLENDER_MIRROR}/Blender${blender_series}/blender-${BLENDER_VERSION}.sha256" \ + "$blender_archive" + +# Unpack into a staging directory beside the prefix and move it into place, so an +# interrupted run cannot leave a half-extracted prefix behind. +# +# Only delete a prefix that looks like one of ours: BLENDER_PREFIX is a constant +# you are meant to edit, and rm -rf as root on whatever it names is unforgiving. +if [[ -e "$BLENDER_PREFIX" && ! -x "$BLENDER_PREFIX/blender" ]]; then + die "$BLENDER_PREFIX exists but holds no blender executable. Refusing to delete it; check BLENDER_PREFIX." +fi +rm -rf "$BLENDER_PREFIX" "${BLENDER_PREFIX}.staging" +mkdir -p "${BLENDER_PREFIX}.staging" +tar -xJf "$WORK_DIR/$blender_archive" -C "${BLENDER_PREFIX}.staging" --strip-components=1 +chmod 755 "${BLENDER_PREFIX}.staging" +mv "${BLENDER_PREFIX}.staging" "$BLENDER_PREFIX" +ln -sf "$BLENDER_PREFIX/blender" /usr/local/bin/blender + +# Run Blender rather than only testing for the file, so one that unpacked but +# cannot start fails here. Capture the output before taking a line: piping into +# head closes the pipe early and Blender's SIGPIPE would make a working Blender +# look broken. A real failure usually means the image lacks Blender's X11 and GL +# libraries, so report those. +if ! blender_output="$("$BLENDER_PREFIX/blender" --version 2>&1)"; then + # "|| true" on the ldd: it exits non-zero for a binary it cannot recognize as + # dynamic, and set -e would then exit before either message is printed. + missing="$( { ldd "$BLENDER_PREFIX/blender" || true; } 2>/dev/null \ + | awk '/not found/ {print $1}' | paste -sd' ' - )" + [[ -n "$missing" ]] \ + && die "Blender cannot start, missing shared libraries: $missing. This image needs a desktop environment or Blender's dependencies." + die "Blender installed to $BLENDER_PREFIX but will not run: $(head -1 <<<"$blender_output")" +fi +log "Blender installed: $(head -1 <<<"$blender_output")" + +# --------------------------------------------------------------------------- +# Install the Deadline Cloud submitter +# --------------------------------------------------------------------------- + +# The "latest" path always serves the current release, and its .sha256 alongside. +SUBMITTER_URL="${DOWNLOADS_BASE}/submitters/latest/linux/DeadlineCloudSubmitter-linux-x64-installer.run" + +installer="$WORK_DIR/submitter-installer.run" +download_verified "$SUBMITTER_URL" "$installer" "${SUBMITTER_URL}.sha256" +chmod +x "$installer" + +# --mode unattended runs without a GUI. deadline_client (the Deadline Cloud CLI +# and libraries) is always installed; enable only the DCC components needed here. +log "installing the submitter (unattended)" +"$installer" \ + --mode unattended \ + --unattendedmodeui none \ + --installscope system \ + --prefix "$SUBMITTER_PREFIX" \ + --enable-components "${SUBMITTER_COMPONENT},${BLENDER_COMPONENT}" \ + --"${BLENDER_COMPONENT//_/-}-path" "$BLENDER_PREFIX" +log "submitter installed at $SUBMITTER_PREFIX" + +# --------------------------------------------------------------------------- +# Enable the add-on in Blender +# --------------------------------------------------------------------------- + +# The unattended install stages the add-on but cannot enable it, because add-ons +# live in Blender's per-user preferences while the install runs at system scope. +# Run the installer's own script as the workstation user to register it. +addon_script="$SUBMITTER_PREFIX/Submitters/Blender/add_submitter_to_pref.py" +addon_path="$SUBMITTER_PREFIX/Submitters/Blender/python" + +# Report Blender's own output on failure, which is where the cause actually is. +log "enabling the Blender add-on for $WORKSTATION_USER" +addon_output="$( + runuser -u "$WORKSTATION_USER" -- env HOME="$USER_HOME" \ + "$BLENDER_PREFIX/blender" --background --python "$addon_script" \ + -- --deadline_cloud_install_path "$addon_path" 2>&1 +)" || die "failed to enable the Blender add-on: $addon_output" + +# Confirm from Blender's preferences rather than trusting the exit code. +verify_output="$( + runuser -u "$WORKSTATION_USER" -- env HOME="$USER_HOME" \ + "$BLENDER_PREFIX/blender" --background --python-expr \ + 'import bpy, sys; sys.exit(0 if "deadline_cloud_blender_submitter" in bpy.context.preferences.addons.keys() else 1)' 2>&1 +)" || die "the Blender add-on did not register in $WORKSTATION_USER's preferences: $verify_output" +log "Blender add-on enabled" + +# --------------------------------------------------------------------------- +# Install Deadline Cloud monitor and create the profile +# --------------------------------------------------------------------------- + +MONITOR_BIN="/usr/bin/deadline-cloud-monitor" +MONITOR_BASE="${DOWNLOADS_BASE}/dcm/latest" + +download_verified "${MONITOR_BASE}/deadline-cloud-monitor_amd64.deb" \ + "$WORK_DIR/dcm.deb" "${MONITOR_BASE}/deadline-cloud-monitor_amd64.deb.sha256" +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "$WORK_DIR/dcm.deb" + +# Run it, so a monitor that installs but cannot start fails here. +monitor_version="$("$MONITOR_BIN" --version)" || die "the monitor at $MONITOR_BIN will not run" +log "monitor installed: $monitor_version" + +# create-profile needs no display. Run it as the workstation user, so the profile +# and the credential cache path baked into it land in that user's home directory. +# +# --monitor-id is required but need not be correct: the real ID cannot be found +# without AWS credentials, and the monitor overwrites it, along with the user and +# identity store IDs, at first sign-in. It must be non-empty though -- an empty +# value makes the monitor drop the profile from its picker and ask for the URL +# instead. It shows verbatim until first sign-in, so make it self-explanatory. +MONITOR_ID_PLACEHOLDER="pending-first-login" + +log "creating monitor profile '$PROFILE_NAME'" +profile_output="$( + runuser -u "$WORKSTATION_USER" -- env HOME="$USER_HOME" "$MONITOR_BIN" create-profile \ + --profile "$PROFILE_NAME" \ + --monitor-id "$MONITOR_ID_PLACEHOLDER" \ + --monitor-url "$MONITOR_URL" \ + --enable-auto-login \ + --set-as-deadline-default 2>&1 +)" || true + +# create-profile exits 0 even when it fails, so check its output and the file. +grep -qF "Created profile ${PROFILE_NAME}" <<<"$profile_output" \ + || die "failed to create the monitor profile: $profile_output" +grep -qF "[profile ${PROFILE_NAME}]" "$USER_HOME/.aws/config" \ + || die "profile $PROFILE_NAME is missing from $USER_HOME/.aws/config" +log "profile created and verified in $USER_HOME/.aws/config" + +cat <..deadlinecloud.amazonaws.com/ + +.EXAMPLE + .\setup_workstation_windows.ps1 https://mystudio.us-west-2.deadlinecloud.amazonaws.com/ +#> + +#Requires -RunAsAdministrator + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, Position = 0)] + [string]$MonitorUrl +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" # Much faster Invoke-WebRequest downloads +$InformationPreference = "Continue" # Show progress messages during provisioning + +# --------------------------------------------------------------------------- +# Edit these for your environment +# --------------------------------------------------------------------------- + +$BlenderVersion = "4.5.0" + +# The submitter's installer components for this DCC: the submitter plug-in itself, +# and the specific DCC version it integrates with. Both change together when you +# switch DCC; see "Adapting to another DCC". +$SubmitterComponent = "deadline_cloud_for_blender" +$BlenderComponent = "blender_45" + +# download.blender.org rejects some automated clients, so this points at +# Blender's official mirror redirector, which forwards to a nearby mirror. +# Point it at an internal mirror if you host the archives yourself. +$BlenderMirror = "https://mirror.blender.org/release" + +$BlenderPrefix = "C:\Program Files\Blender" +$SubmitterPrefix = "C:\Program Files\DeadlineCloudSubmitter" + +$DownloadsBase = "https://downloads.deadlinecloud.amazonaws.com" + +# --------------------------------------------------------------------------- +# Adapting to another DCC +# --------------------------------------------------------------------------- +# +# Blender stands in for whichever DCC you run. It is used here because it +# installs unattended from a public archive with no license server, which keeps +# this example runnable as-is. Everything Deadline Cloud does is identical for +# every DCC, so switching to Maya, Nuke, Houdini, 3ds Max, Cinema 4D, After +# Effects, or VRED means changing three things: +# +# 1. $SubmitterComponent and $BlenderComponent above, for example +# deadline_cloud_for_houdini plus houdini_20_5. Run " --help" for +# the current --enable-components values. The ---path flag is derived +# from $BlenderComponent, so it follows automatically. +# 2. The "Install Blender" step. Commercial DCCs need a vendor installer and +# usually a license server, so replace that block entirely. +# 3. The "Enable the add-on in Blender" step. It is Blender-specific. Other +# DCCs are wired up by the installer itself or by an environment variable +# such as MAYA_MODULE_PATH or NUKE_PATH, so you can often delete it. + +function Write-Step { param([string]$Message) Write-Information "[setup-workstation] $Message" } +function Write-Fatal { param([string]$Message) throw "[setup-workstation] ERROR: $Message" } + +# Run a native command, returning its merged output and leaving $LASTEXITCODE for +# the caller. Windows PowerShell 5.1 turns a native command's stderr into error +# records, which $ErrorActionPreference = "Stop" escalates to a terminating +# NativeCommandError -- so a Blender that prints a driver warning would fail the +# script instead of reaching the exit-code check. Relax it for the call only. +function Get-NativeOutput { + param([scriptblock]$Command) + $previous = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { & $Command 2>&1 } finally { $ErrorActionPreference = $previous } +} + +# --------------------------------------------------------------------------- +# Arguments +# --------------------------------------------------------------------------- + +# The profile and Blender's add-on preferences are per user, so this must run as +# the account that signs in. #Requires -RunAsAdministrator is satisfied by SYSTEM, +# which Run Command and EC2 user data both use, and everything would then land in +# a service profile no artist logs in to. +if ([System.Security.Principal.WindowsIdentity]::GetCurrent().IsSystem) { + Write-Fatal "running as SYSTEM. The profile and Blender preferences are per user, so they would be written to a service profile the artist never logs in to. Run this as the artist's own account in an elevated session." +} + +# The URL must carry its Region segment: the monitor accepts one without it and +# then writes a profile with the wrong region. Check the scheme separately, since +# [System.Uri] parses a host out of any scheme, http:// included. +$monitorUri = [System.Uri]$MonitorUrl +if ($monitorUri.Scheme -ne "https") { + Write-Fatal "monitor URL must use https (got: $MonitorUrl)" +} +$monitorHost = $monitorUri.Host +if ($monitorHost -notmatch '^([a-z0-9-]+)\.([a-z0-9-]+)\.deadlinecloud\.amazonaws\.com$') { + Write-Fatal "monitor URL must be https://..deadlinecloud.amazonaws.com/ (got: $MonitorUrl)" +} +$MonitorSubdomain = $Matches[1] +$MonitorRegion = $Matches[2] +$ProfileName = "$MonitorSubdomain-$MonitorRegion" + +Write-Step "workstation user: $env:USERNAME" +Write-Step "monitor: $MonitorSubdomain in $MonitorRegion, profile '$ProfileName'" + +$WorkDir = Join-Path $env:TEMP "deadline-workstation-setup" +New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null + +# Fetch a URL as text. Windows PowerShell 5.1 returns Content as a Byte[] for +# -UseBasicParsing while PowerShell 7 returns a String, so decode when needed. +# Treating the byte array as text yields the first byte value, not the body. +function Get-RemoteText { + param([string]$Uri) + $content = (Invoke-WebRequest -Uri $Uri -UseBasicParsing).Content + if ($content -is [byte[]]) { + $content = [System.Text.Encoding]::UTF8.GetString($content) + } + return $content +} + +# Download a file and verify it against a published sha256. Verification is not +# optional: an unreachable checksum is an error, not a reason to skip the check. +# Pass -MatchName to select one line from a multi-file checksum manifest. +function Get-VerifiedFile { + param([string]$Uri, [string]$OutFile, [string]$ChecksumUri, [string]$MatchName) + + Write-Step "downloading $(Split-Path -Leaf $Uri)" + Invoke-WebRequest -Uri $Uri -OutFile $OutFile -UseBasicParsing + + try { + $body = Get-RemoteText -Uri $ChecksumUri + } + catch { + Write-Fatal "cannot fetch the checksum for $(Split-Path -Leaf $OutFile) from ${ChecksumUri}: $($_.Exception.Message)" + } + + $expected = $null + if ($MatchName) { + foreach ($line in ($body -split "`n")) { + $fields = $line.Trim() -split '\s+' + if ($fields.Count -ge 2 -and ($fields[1] -eq $MatchName -or $fields[1] -eq "./$MatchName")) { + $expected = $fields[0] + break + } + } + } + else { + $expected = ($body.Trim() -split '\s+')[0] + } + if ($expected -notmatch '^[0-9a-fA-F]{64}$') { + Write-Fatal "no usable sha256 for $(Split-Path -Leaf $OutFile) in $ChecksumUri" + } + + $actual = (Get-FileHash -Path $OutFile -Algorithm SHA256).Hash + if ($actual.ToLower() -ne $expected.ToLower()) { + Write-Fatal "checksum mismatch for $OutFile (expected $expected, got $actual)" + } + Write-Step "verified $(Split-Path -Leaf $OutFile)" +} + +# --------------------------------------------------------------------------- +# Install Blender +# --------------------------------------------------------------------------- + +$blenderSeries = $BlenderVersion.Substring(0, $BlenderVersion.LastIndexOf(".")) +$blenderArchive = "blender-$BlenderVersion-windows-x64.zip" +$blenderZip = Join-Path $WorkDir $blenderArchive + +Get-VerifiedFile -Uri "$BlenderMirror/Blender$blenderSeries/$blenderArchive" -OutFile $blenderZip ` + -ChecksumUri "$BlenderMirror/Blender$blenderSeries/blender-$BlenderVersion.sha256" ` + -MatchName $blenderArchive + +# Expand into a staging directory beside the prefix and move it into place, so an +# interrupted run cannot leave a half-extracted prefix behind. Beside the prefix, +# not in $WorkDir: a cross-volume Move-Item copies rather than renames. +$extractDir = "$BlenderPrefix.staging" +if (Test-Path $extractDir) { Remove-Item -Recurse -Force $extractDir } +Expand-Archive -Path $blenderZip -DestinationPath $extractDir -Force + +# The archive holds a single blender--windows-x64\ directory. Capture it +# before the move: .FullName on $null makes Move-Item throw a parameter-binding +# error before any message here could explain why. +$extracted = Get-ChildItem -Path $extractDir -Directory | Select-Object -First 1 +if (-not $extracted) { + Write-Fatal "the Blender archive did not expand to a top-level directory in $extractDir" +} +if (-not (Test-Path (Join-Path $extracted.FullName "blender.exe"))) { + Write-Fatal "the Blender archive did not contain blender.exe" +} + +# Only delete a prefix that looks like one of ours: $BlenderPrefix is a constant +# you are meant to edit, and a blind recursive delete is unforgiving. +if (Test-Path $BlenderPrefix) { + if (-not (Test-Path (Join-Path $BlenderPrefix "blender.exe"))) { + Write-Fatal "$BlenderPrefix exists but holds no blender.exe. Refusing to delete it; check `$BlenderPrefix, and see Troubleshooting in the README if a previous run was interrupted." + } + Remove-Item -Recurse -Force $BlenderPrefix +} +Move-Item -Path $extracted.FullName -Destination $BlenderPrefix +Remove-Item -Recurse -Force $extractDir -ErrorAction SilentlyContinue +$blenderExe = Join-Path $BlenderPrefix "blender.exe" + +# Run Blender, so one that unpacked but cannot start fails here. Capture the +# output before narrowing it: Select-Object -First 1 halts the upstream pipeline, +# which can terminate the still-running native command and leave $LASTEXITCODE +# reflecting that rather than Blender's own exit. +$blenderOutput = Get-NativeOutput { & $blenderExe --version } +if ($LASTEXITCODE -ne 0) { + Write-Fatal "Blender installed to $BlenderPrefix but will not run: $($blenderOutput | Select-Object -First 1)" +} +Write-Step "Blender installed: $($blenderOutput | Select-Object -First 1)" + +# --------------------------------------------------------------------------- +# Install the Deadline Cloud submitter +# --------------------------------------------------------------------------- + +# The "latest" path always serves the current release, and its .sha256 alongside. +$submitterUrl = "$DownloadsBase/submitters/latest/windows/DeadlineCloudSubmitter-windows-x64-installer.exe" + +$installer = Join-Path $WorkDir "submitter-installer.exe" +Get-VerifiedFile -Uri $submitterUrl -OutFile $installer -ChecksumUri "$submitterUrl.sha256" + +# --mode unattended runs without a GUI. deadline_client (the Deadline Cloud CLI +# and libraries) is always installed; enable only the DCC components needed here. +# +# On Windows the ---path flag takes the executable, not the install +# directory as on Linux. Values with spaces must be quoted: Start-Process joins +# -ArgumentList without quoting, so "C:\Program Files\..." would split in two. +Write-Step "installing the submitter (unattended)" +$installerArgs = @( + "--mode", "unattended" + "--unattendedmodeui", "none" + "--installscope", "system" + "--prefix", "`"$SubmitterPrefix`"" + "--enable-components", "$SubmitterComponent,$BlenderComponent" + ("--" + $BlenderComponent.Replace("_", "-") + "-path"), "`"$blenderExe`"" +) +$process = Start-Process -FilePath $installer -ArgumentList $installerArgs -Wait -PassThru -NoNewWindow +if ($process.ExitCode -ne 0) { + Write-Fatal "the submitter installer exited with code $($process.ExitCode)" +} +Write-Step "submitter installed at $SubmitterPrefix" + +# --------------------------------------------------------------------------- +# Enable the add-on in Blender +# --------------------------------------------------------------------------- + +# The unattended install stages the add-on but cannot enable it, because add-ons +# live in Blender's per-user preferences while the install runs at system scope. +# Run the installer's own script to register it for this account. +$addonScript = Join-Path $SubmitterPrefix "Submitters\Blender\add_submitter_to_pref.py" +$addonPath = Join-Path $SubmitterPrefix "Submitters\Blender\python" + +Write-Step "enabling the Blender add-on" +$addonOutput = Get-NativeOutput { + & $blenderExe --background --python $addonScript -- --deadline_cloud_install_path $addonPath +} +if ($LASTEXITCODE -ne 0) { + Write-Fatal "failed to enable the Blender add-on (exit code $LASTEXITCODE): $($addonOutput | Out-String)" +} + +# Confirm from Blender's preferences rather than trusting the exit code. Use a +# script file, not --python-expr: PowerShell does not preserve the inner quotes +# of an expression passed on the command line, so Blender raises NameError. +$checkScript = Join-Path $WorkDir "check_addon.py" +Set-Content -Path $checkScript -Encoding ASCII -Value @' +import bpy +import sys + +sys.exit(0 if "deadline_cloud_blender_submitter" in bpy.context.preferences.addons.keys() else 1) +'@ +Get-NativeOutput { & $blenderExe --background --python $checkScript } | Out-Null +if ($LASTEXITCODE -ne 0) { + Write-Fatal "the Blender add-on did not register in Blender preferences" +} +Write-Step "Blender add-on enabled" + +# --------------------------------------------------------------------------- +# Install Deadline Cloud monitor and create the profile +# --------------------------------------------------------------------------- + +$monitorSetupUrl = "$DownloadsBase/dcm/latest/DeadlineCloudMonitor_x64-setup.exe" +$monitorSetup = Join-Path $WorkDir "DeadlineCloudMonitor_x64-setup.exe" +Get-VerifiedFile -Uri $monitorSetupUrl -OutFile $monitorSetup -ChecksumUri "$monitorSetupUrl.sha256" + +# /S is the monitor installer's silent switch. +$process = Start-Process -FilePath $monitorSetup -ArgumentList "/S" -Wait -PassThru -NoNewWindow +if ($process.ExitCode -ne 0) { + Write-Fatal "the monitor installer exited with code $($process.ExitCode)" +} + +# Find the installed executable. The installer is 32-bit, so its writes can be +# redirected into the SysWOW64 view of the profile while the InstallLocation it +# records still names System32. Neither is reliable alone, so try both. +$monitorCandidates = [System.Collections.Generic.List[string]]::new() + +foreach ($key in @( + "HKCU:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*", + "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*", + "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*")) { + Get-ItemProperty $key -ErrorAction SilentlyContinue | + Where-Object { $_.DisplayName -eq "DeadlineCloudMonitor" -and $_.InstallLocation } | + ForEach-Object { + $location = $_.InstallLocation.Trim('"') + $monitorCandidates.Add((Join-Path $location "DeadlineCloudMonitor.exe")) + # The same path under the other WOW64 view of the profile. + $monitorCandidates.Add((Join-Path ($location -replace '\\[Ss]ystem32\\', '\SysWOW64\') "DeadlineCloudMonitor.exe")) + } +} +$monitorCandidates.Add((Join-Path $env:LOCALAPPDATA "DeadlineCloudMonitor\DeadlineCloudMonitor.exe")) +$monitorCandidates.Add("C:\Program Files\DeadlineCloudMonitor\DeadlineCloudMonitor.exe") + +$monitorBin = $monitorCandidates | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $monitorBin) { + Write-Fatal "cannot find DeadlineCloudMonitor.exe after install. Looked in:`n $($monitorCandidates -join "`n ")" +} +Write-Step "monitor installed: $monitorBin" + +# create-profile needs no display. --monitor-id is required but need not be +# correct: the real ID cannot be found without AWS credentials, and the monitor +# overwrites it, along with the user and identity store IDs, at first sign-in. It +# must be non-empty though -- an empty value makes the monitor drop the profile +# from its picker and ask for the URL instead. It shows verbatim until first +# sign-in, so make it self-explanatory. +$monitorIdPlaceholder = "pending-first-login" + +Write-Step "creating monitor profile '$ProfileName'" + +# A direct pipeline into Out-String, not Get-NativeOutput like the calls above: +# DeadlineCloudMonitor.exe is a GUI-subsystem binary and PowerShell does not wait +# for one, so it is this pipe that forces the wait and captures the output. In a +# scriptblock the call returns instantly with nothing. The 5.1 stderr concern still +# applies, so relax $ErrorActionPreference around just this call. +$previousEap = $ErrorActionPreference +try { + $ErrorActionPreference = "Continue" + $profileOutput = & $monitorBin create-profile ` + --profile $ProfileName ` + --monitor-id $monitorIdPlaceholder ` + --monitor-url $MonitorUrl ` + --enable-auto-login ` + --set-as-deadline-default 2>&1 | Out-String +} +finally { + $ErrorActionPreference = $previousEap +} + +# create-profile exits 0 even when it fails, so check its output and the file. +# Report whether the file appeared: it says whether the command ran at all. +if ($profileOutput -notmatch [regex]::Escape("Created profile $ProfileName")) { + $configPath = Join-Path $env:USERPROFILE ".aws\config" + $configState = if (Test-Path $configPath) { "$configPath exists" } else { "$configPath does not exist" } + Write-Fatal "failed to create the monitor profile ($configState). Output was: '$($profileOutput.Trim())'" +} + +# Test for the file first: Select-String on a missing path throws +# ItemNotFoundException, so the message below would never be reached. +$awsConfig = Join-Path $env:USERPROFILE ".aws\config" +if (-not (Test-Path $awsConfig)) { + Write-Fatal "profile $ProfileName is missing from $awsConfig (the file does not exist)" +} +if (-not (Select-String -Path $awsConfig -SimpleMatch -Pattern "[profile $ProfileName]" -Quiet)) { + Write-Fatal "profile $ProfileName is missing from $awsConfig" +} +Write-Step "profile created and verified in $awsConfig" + +# Remove the ~1 GB of downloads on success. A failed run keeps them on purpose, +# so the installer logs survive. +Remove-Item -Recurse -Force $WorkDir -ErrorAction SilentlyContinue +Write-Step "removed temporary downloads from $WorkDir" + +Write-Information @" + +[setup-workstation] Done. + + Blender: $BlenderPrefix ($BlenderVersion) + Submitter: $SubmitterPrefix + Monitor: $monitorBin + Profile: $ProfileName ($MonitorUrl) + +$env:USERNAME can now open Deadline Cloud monitor, sign in to the +'$ProfileName' profile, and submit from Blender. + +"@