diff --git a/.github/external-link-ignore.txt b/.github/external-link-ignore.txt new file mode 100644 index 00000000..75c87563 --- /dev/null +++ b/.github/external-link-ignore.txt @@ -0,0 +1,20 @@ +# Exact original-link domains that consistently reject this repository's live checker. +# Entries may cover true subdomains by DNS-label boundary; wildcards and substrings are forbidden. +# Each domain must be immediately preceded by: # YYYY-MM-DD: empirically observed bot rejection. + +# 2026-07-14: HEAD and minimal Range GET both returned HTTP 403 for public API documentation. +docs.blender.org +# 2026-07-14: HEAD and minimal Range GET both returned HTTP 403 for public release downloads. +download.blender.org +# 2026-07-14: HEAD and minimal Range GET both returned HTTP 429 after bounded retries. +docs.vllm.ai +# 2026-07-14: HEAD and minimal Range GET both returned HTTP 429 after bounded retries. +pip.pypa.io +# 2026-07-14: HEAD and minimal Range GET both returned HTTP 429 after bounded retries. +rez.readthedocs.io +# 2026-07-14: HEAD and minimal Range GET both returned HTTP 403 for public product pages. +www.autodesk.com +# 2026-07-14: HEAD and minimal Range GET both returned a self-referential HTTP 302 without a browser session. +manage.autodesk.com +# 2026-07-15: HEAD and minimal Range GET both returned HTTP 403 for public product pages that load in a browser. +www.keyshot.com diff --git a/.github/workflows/static_validation.yml b/.github/workflows/static_validation.yml new file mode 100644 index 00000000..a53770d9 --- /dev/null +++ b/.github/workflows/static_validation.yml @@ -0,0 +1,115 @@ +name: Documentation validation + +on: + pull_request: + push: + branches: [mainline] + schedule: + - cron: "17 8 * * 2" + workflow_dispatch: + +permissions: {} + +concurrency: + group: documentation-validation-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + static: + name: Unit and static checks + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Run repository validation + run: python3 scripts/validate_repository.py + + # On pull requests, only the Markdown files the PR changed have their live external + # links checked. This keeps the required PR signal fast and resilient to unrelated + # third-party outages. The whole repository is swept on the weekly schedule below. + changed-external-links: + name: Live external links (changed files) + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Check changed Markdown external links + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + # NUL-delimited to stay correct for renamed or unusually named files. + changed=() + while IFS= read -r -d '' file; do + changed+=("$file") + done < <(git diff -z --name-only --diff-filter=ACMRT "$BASE_SHA" "$HEAD_SHA" -- '*.md') + if [ ${#changed[@]} -eq 0 ]; then + echo "No Markdown files changed; nothing to check." + exit 0 + fi + echo "Checking changed Markdown files:" + printf ' %s\n' "${changed[@]}" + python3 scripts/check_external_links.py "${changed[@]}" + + # On the weekly schedule and manual runs, every tracked Markdown file is checked. + # A failure opens (or updates) a tracking issue so link rot gets triaged instead of + # silently failing a scheduled run nobody is watching. + all-external-links: + name: Live external links (full sweep) + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + issues: write + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Check all external Markdown links + id: check + run: | + set +e + python3 scripts/check_external_links.py 2>&1 | tee link-report.txt + echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + - name: Open or update link-rot issue on failure + if: steps.check.outputs.exit_code != '0' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + label="link-rot" + title="Broken external documentation links detected" + # The label is required for de-duplication; ignore the error if it exists. + gh label create "$label" --color B60205 --description "Automated documentation link-rot tracking" 2>/dev/null || true + # Cap the embedded report so the issue body stays well under GitHub's limit. + report="$(head -c 50000 link-report.txt)" + body="$(printf 'The scheduled documentation link check failed.\n\nWorkflow run: %s\n\n```\n%s\n```\n' "$RUN_URL" "$report")" + existing="$(gh issue list --state open --label "$label" --json number --jq '.[0].number')" + if [ -n "$existing" ]; then + echo "Updating existing issue #$existing" + gh issue comment "$existing" --body "$body" + else + echo "Opening new tracking issue" + gh issue create --title "$title" --label "$label" --body "$body" + fi + - name: Fail if links are broken + if: steps.check.outputs.exit_code != '0' + run: | + echo "External link check failed; see the tracking issue." >&2 + exit 1 diff --git a/AGENTS.md b/AGENTS.md index 48bdf3d0..7d8e7b0f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,105 +1,91 @@ # AGENTS.md — deadline-cloud-samples -This file gives AI coding assistants (Codex CLI, Aider, Cline, Continue, -Cursor, Copilot, Gemini, ChatGPT, Claude Code, Kiro, etc.) the context they -need to work effectively in this repository. +This file gives AI coding assistants the context they need to work effectively in this repository. ## What this repo is `deadline-cloud-samples` is a public collection of samples for -[AWS Deadline Cloud](https://aws.amazon.com/deadline-cloud/). It is **not** a -single buildable package — there is no top-level build, test, or lint command. -Each sample is self-contained. +[AWS Deadline Cloud](https://aws.amazon.com/deadline-cloud/). It is not a single buildable package; +each sample is self-contained. The repository does have one top-level, standard-library-only static +validation command: -## Where things live +```console +python3 scripts/validate_repository.py +``` +Run it after every repository change, in addition to tests or validation owned by the sample you edit. +External Markdown links use a separate network-dependent command: + +```console +python3 scripts/check_external_links.py ``` + +Use `--no-ignore` to audit the narrowly documented domain ignore list before changing it; genuine +broken links must be fixed rather than ignored. + +## Find samples + +Start with the task paths and repository map in [`README.md`](README.md), then use the nearest area or +collection README. Its index table is complete for the scope it declares. Search tracked paths or +contents directly when you need implementation or support files that are intentionally excluded from +user-selectable sample tables. + +## Where things live + +```text deadline-cloud-samples/ -├── job_bundles/ OpenJD job bundle samples (template.yaml + assets) -├── conda_recipes/ Conda recipes for DCC packages used by Deadline Cloud -├── queue_environments/ Queue environment YAMLs (Conda/Rez software providers) -├── host_configuration_scripts/ Per-OS scripts for service-managed fleet workers -├── submission_hooks/ Pre-submission Python hooks for the Deadline Cloud CLI -├── containers/ Dockerfiles (e.g. AL2023 worker-equivalent for local builds) -├── cloudformation/ CloudFormation farm + infra templates -├── terraform/ Terraform farm + infra templates -├── utility_scripts/ Standalone CLI helpers -└── skills/ LLM-agnostic, task-specific guides (see below) +├── cloudformation/ CloudFormation farm and infrastructure templates +├── terraform/ Terraform farm and infrastructure templates +├── job_bundles/ OpenJD job bundles (template.yaml plus assets) +├── conda_recipes/ DCC and application Conda recipes +├── containers/ Worker-compatible and application containers +├── queue_environments/ Session software environments (Conda, Rez, and pip) +├── host_configuration_scripts/ Privileged service-managed fleet setup scripts +├── submission_hooks/ Pre-submission Deadline Cloud CLI hooks +├── utility_scripts/ Standalone workflow helpers +├── skills/ Task-specific guides for coding agents +├── docs/ Contributor guidance and documentation starting points +└── scripts/ Standard-library repository validation ``` -**Most samples have their own `README.md`** with prerequisites, parameters, -and run/submit instructions. Read the relevant `README.md` before modifying -or adding to a sample directory. - -## Skills — task-specific instructions - -The [`skills/`](./skills/) directory contains self-contained, LLM-agnostic -guides for common tasks. Each skill is a Markdown file with YAML frontmatter -(`name`, `description`, `tags`) followed by step-by-step instructions, -references, and examples. - -**Before starting work, check `skills/` for a matching guide and read it.** -The `description` field tells you when to use each skill. - -| Skill | Use when | -|-------|----------| -| [`skills/deadline-cloud-job/`](./skills/deadline-cloud-job/SKILL.md) | Creating or updating a Deadline Cloud job (OpenJD job bundle) under `job_bundles/` | -| [`skills/conda-builder/`](./skills/conda-builder/SKILL.md) | Creating or updating a DCC conda recipe under `conda_recipes/` | -| [`skills/3dsmax-host-config/`](./skills/3dsmax-host-config/SKILL.md) | Creating or updating a 3ds Max host configuration script | -| [`skills/host-config-from-installer/`](./skills/host-config-from-installer/SKILL.md) | Creating a host configuration script from a vendor installer | - -Skills are auto-discovered via `.claude/skills` and `.kiro/skills` symlinks. -For other tools, point your assistant at the relevant `SKILL.md` directly -(paste, `@`-mention, or include in context). - -## Repo conventions - -- **Inclusive language** — avoid `master`/`slave`, `whitelist`/`blacklist`. - Use `primary`/`replica`, `allowlist`/`denylist`. -- **Python install commands** — use `pip install ...` (works on Windows, - macOS, and Linux). Avoid `pip3` unless the sample is Linux/macOS-only. -- **Job bundles** live under `job_bundles//` with a `template.yaml`, - optional `parameter_values.yaml`, and a `README.md`. -- **Conda recipes** live under `conda_recipes/-/` with a - `recipe/` subdirectory and a `deadline-cloud.yaml`. -- **Iterate locally before submitting** — for OpenJD templates, run - `openjd check` and `openjd run --tasks ` to verify a single task end- - to-end before submitting the full parameter range to a Deadline Cloud farm. +Read the relevant sample `README.md` before modifying its files. Use +[`docs/SAMPLE_README_TEMPLATE.md`](docs/SAMPLE_README_TEMPLATE.md) as an adaptable starting point when +adding a nontrivial sample. + +Before implementing a sample, inspect [`skills/`](skills/) for a matching `SKILL.md` guide. + +## Repository conventions + +* Use inclusive language: prefer `primary`/`replica` and `allowlist`/`denylist`. +* Use `pip install ...` in cross-platform Python instructions; avoid `pip3` unless the sample is + explicitly Linux/macOS-only. +* Job bundles live under `job_bundles//` with a `template.yaml`, optional + `parameter_values.yaml`, and a `README.md` for nontrivial samples. +* Conda recipes live under `conda_recipes/-/` with a `recipe/` directory and + `deadline-cloud.yaml`. +* For OpenJD templates, run `openjd check` and `openjd run --tasks ` to verify a representative + task locally before submitting the full parameter range when possible. +* Add, rename, move, or delete a sample in the nearest category table. Change root navigation only + when a recommended path changes. +* Keep indexes in Markdown; do not add catalog or metadata-generation machinery. +* Do not add third-party runtime dependencies to repository validation. ## Pre-PR checklist -Before opening a pull request, make sure every commit on the branch satisfies the following: - -- [ ] **Conventional commit title** — every commit title MUST use - [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) syntax - (see the type table below). PRs without it will be blocked by CI. -- [ ] **Signed-off commits** — every commit MUST carry a `Signed-off-by` trailer - ([Developer Certificate of Origin](https://developercertificate.org/)). Create - commits with `git commit -s`, or add the trailer to an existing commit with - `git commit --amend -s`. -- [ ] **Sample README updated** — if you changed a sample's behavior, prerequisites, - or parameters, update its `README.md`. -- [ ] **Inclusive language** — no `master`/`slave`, `whitelist`/`blacklist`. - -### Conventional commit types - -| Type | Use for | -|------------|-----------------------------------------------------------| -| `feat` | New sample, new feature in an existing sample | -| `fix` | Bug fix | -| `docs` | Documentation only | -| `test` | Test additions or changes only | -| `refactor` | Code refactor with no behavior change | -| `ci` | CI infrastructure changes | -| `chore` | Generic maintenance | -| `feat!` / `fix!` | Breaking change (also add `BREAKING CHANGE:` footer) | - -See [`CONTRIBUTING.md`](./CONTRIBUTING.md) for the full contribution workflow. +* [ ] Run `python3 scripts/validate_repository.py` successfully (unit tests and static local-link checks). +* [ ] Run `python3 scripts/check_external_links.py` successfully when Markdown links change. +* [ ] Run the affected sample's own relevant tests or static checks. +* [ ] Update the sample README when behavior, prerequisites, parameters, outputs, or risks change. +* [ ] Update the nearest category table when a sample is added, renamed, moved, or deleted. +* [ ] Update root task paths only when a recommended starting point changes. +* [ ] Use a [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) title. +* [ ] Sign off every commit under the [Developer Certificate of Origin](https://developercertificate.org/). +* [ ] Check changed content for inclusive language. + +See [`CONTRIBUTING.md`](CONTRIBUTING.md) for the full contribution and licensing workflow. ## External references -- [AWS Deadline Cloud user guide](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/index.html) -- [AWS Deadline Cloud developer guide](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/index.html) -- [Open Job Description spec](https://github.com/OpenJobDescription/openjd-specifications/wiki) -- [`README.md`](./README.md) — directory overview and high-level usage -- [`CONTRIBUTING.md`](./CONTRIBUTING.md) — full contribution workflow, MIT-0 licensing, security reporting +* [AWS Deadline Cloud developer guide](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/index.html) +* [AWS Deadline Cloud user guide](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/index.html) +* [Open Job Description specification](https://github.com/OpenJobDescription/openjd-specifications/wiki) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8fd2c09f..227c7605 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,7 @@ Table of contents: * [Finding contributions to work on](#finding-contributions-to-work-on) * [Talk with us first](#talk-with-us-first) * [Contributing via Pull Requests](#contributing-via-pull-requests) + * [Adding or updating a sample](#adding-or-updating-a-sample) * [Conventional Commits](#conventional-commits) * [Licensing](#licensing) @@ -34,8 +35,7 @@ informative; this is the model that we follow. ### Finding contributions to work on If you are not sure what you would like to contribute, then looking at the existing issues is a great way to find -something to contribute on. Looking at -[issues that have the "help wanted" or "good first issue" labels](https://github.com/aws-deadline/deadline-cloud-samples/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22%2C%22help+wanted%22) +something to contribute on. [Issues that have the "help wanted" or "good first issue" labels](https://github.com/aws-deadline/deadline-cloud-samples/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22%2C%22help+wanted%22) are a good place to start, but please dive into any issue that interests you whether it has those labels or not. ### Talk with us first @@ -68,10 +68,60 @@ To send us a pull request, please: GitHub provides additional documentation on [forking a repository](https://help.github.com/articles/fork-a-repo/) and [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). +### Adding or updating a sample + +Each sample area and nested collection declares its tracked scope in a category README. Its table is +the complete index of user-selectable samples in that scope; implementation files and support +infrastructure can be documented in nearby prose instead. Do not add a generated catalog, metadata +schema, or other parallel inventory. When you add, rename, move, or delete a sample: + +1. Put it in the appropriate top-level area and give a nontrivial sample its own README. +2. Update the nearest category table so its relative link and task-oriented description remain + accurate. If a move crosses category boundaries, update both affected tables. +3. Change the root README navigation only when a recommended path or starting point changes; do not + duplicate the category's exhaustive index at the root. +4. Use [`docs/SAMPLE_README_TEMPLATE.md`](docs/SAMPLE_README_TEMPLATE.md) as a suggested starting point + for a nontrivial sample. Adapt it freely: remove irrelevant prompts, rename/reorder/combine + sections, and add material that helps users choose, run, and clean up the sample. +5. Run the complete local unit and static validation from the repository root: + + ```console + python3 scripts/validate_repository.py + ``` + +The top-level command uses only the Python standard library. It runs all checker unit tests and checks +local links, including same-document and cross-document anchors, in every tracked Markdown file. Also +run tests specific to the sample you changed; for OpenJD templates, validate and run a representative +task locally when possible. + +Live external links are intentionally a separate network-dependent validation mode: + +```console +python3 scripts/check_external_links.py +``` + +The live checker aggregates source locations, validates every URL, redirect, and DNS answer against +public-network-only rules, tries `HEAD` before a minimal one-byte `GET`, and does not honor environment +proxy settings. It checks all external Markdown links on pull requests, mainline pushes, the weekly +schedule, and manual workflow runs. + +The narrow [external-link ignore file](.github/external-link-ignore.txt) is only for domains that +empirically reject both checker requests as bot traffic. Before adding an exact domain, run the audit: + +```console +python3 scripts/check_external_links.py --no-ignore +``` + +Confirm the failure is a repeatable bot rejection rather than a missing page, then add the exact domain +with an immediately preceding dated comment that records the observed status or error. Entries match +the domain and true subdomains by DNS-label boundary only; wildcards and substring matching are not +supported. Ignores apply only to original link hosts, never redirect destinations. Fix genuine broken +links, including HTTP 404 responses, instead of ignoring them. + ### Conventional commits The commits in this repository are all required to use [conventional commit syntax](https://www.conventionalcommits.org/en/v1.0.0/) -in their title to help us identify the kind of change that is being made, automatically generate the changelog, and +in their title to help us identify the kind of change that is being made, automatically generate the changelog, and automatically identify next release version number. Only the first commit that deviates from mainline in your pull request must adhere to this requirement. @@ -82,16 +132,16 @@ We ask that you use these commit types in your commit titles: * `test` - When the pull request is only implementing an addition or change to tests or the testing infrastructure; * `docs` - When the pull request is primarily implementing an addition or change to the package's documentation; * `refactor` - When the pull request is implementing only a refactor of existing code; -* `ci` - When the pull request is implementing a change to the CI infrastructure of the packge; +* `ci` - When the pull request is implementing a change to the CI infrastructure of the package; * `chore` - When the pull request is a generic maintenance task. -We also require that the type in your conventional commit title end in an exclaimation point (e.g. `feat!` or `fix!`) +We also require that the type in your conventional commit title end in an exclamation point (e.g. `feat!` or `fix!`) if the pull request should be considered to be a breaking change in some way. Please also include a "BREAKING CHANGE" footer in the description of your commit in this case ([example](https://www.conventionalcommits.org/en/v1.0.0/#commit-message-with-both--and-breaking-change-footer)). -Examples of breaking changes include any that implements a backwards-imcompatible change to a public Python interface, -the command-line interface, or the like. +Examples of breaking changes include any change that implements a backwards-incompatible change to a public Python interface, +the command-line interface, or the like. -If you need change a commit message, then please see the +If you need to change a commit message, then please see the [GitHub documentation on the topic](https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/changing-a-commit-message) to guide you. diff --git a/README.md b/README.md index 6df2d763..b58ff3c2 100644 --- a/README.md +++ b/README.md @@ -1,86 +1,77 @@ -## Deadline Cloud samples +# AWS Deadline Cloud samples + +Build, submit, and operate real workloads on [AWS Deadline Cloud](https://aws.amazon.com/deadline-cloud/). +Start with the task you want to complete; each sample stays self-contained in its existing directory. + +## What do you want to do? + +| Goal | Start here | +|---|---| +| Deploy a farm | [CloudFormation starter farm](cloudformation/farm_templates/starter_farm/) or [Terraform starter farm](terraform/farm_templates/starter_farm/) | +| Learn how a job is structured | [Job development progression](job_bundles/job_dev_progression/) or the [minimal job](job_bundles/simple_job/) | +| Render with a DCC | [Blender render](job_bundles/blender_render/), [Maya CLI render](job_bundles/maya_cli_render/), or browse the [job bundles](job_bundles/) | +| Run a new DCC or application | Read about [custom software delivery](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/deploy-custom-software.html), then browse [Conda recipes](conda_recipes/), [host configuration scripts](host_configuration_scripts/), [containers](containers/), and [job bundles](job_bundles/) | +| Deliver custom plugins | Read about [Plugin Sync](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/plugin-sync.html), then compare [plugin packages](conda_recipes/), [host installations](host_configuration_scripts/), and [queue environments](queue_environments/) | +| Connect studio systems | Browse [submission hooks](submission_hooks/), [custom submitters](job_bundles/custom_submitters/), [queue environments](queue_environments/), and [event notifications](cloudformation/notification_templates/) | +| Find a specific example | Use the [repository map](#repository-map), then browse that area's complete category table | +| Create a sample with an AI agent | Inspect [skills](skills/) for a matching task guide | + +## Quick start + +1. Configure a Deadline Cloud farm and install the + [Deadline Cloud CLI](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/submit-jobs-how.html). + If you need a farm, deploy one of the starter templates above. +2. Clone this repository and open its root directory. +3. Preview a job's submission interface: + + ```console + deadline bundle gui-submit job_bundles/gui_control_showcase + ``` + +4. Submit the minimal job to your configured queue: + + ```console + deadline bundle submit job_bundles/simple_job + ``` + +Read each sample's README before deployment or submission. Samples can create billable AWS resources +or run licensed software; review parameters, IAM permissions, licensing, and cleanup instructions first. + +## Repository map + +| Area | Use it for | +|---|---| +| [CloudFormation](cloudformation/) | Deploy starter farms, fleet support, storage, capacity automation, and notifications. | +| [Terraform](terraform/) | Deploy a starter farm with Terraform. | +| [Job bundles](job_bundles/) | Define OpenJD rendering, simulation, ML, scientific, and utility jobs. | +| [Conda recipes](conda_recipes/) | Build applications, adaptors, renderers, and plugins into versioned packages. | +| [Containers](containers/) | Build worker-compatible or application container images. | +| [Queue environments](queue_environments/) | Prepare Conda, Rez, pip, caching, and licensing once per worker session. | +| [Host configuration scripts](host_configuration_scripts/) | Install privileged software and configure service-managed fleet worker hosts. | +| [Submission hooks](submission_hooks/) | Inspect or modify job bundles immediately before submission. | +| [Utility scripts](utility_scripts/) | Automate supporting tasks such as uploading job attachments. | +| [Agent skills](skills/) | Give coding agents repeatable instructions for authoring jobs, packages, and host configs. | +| [Contributor documentation](docs/) | Use the adaptable sample README starting point. | +| [Repository validation](scripts/) | Run unit, local-link, and live external-link checks. | + +Each sample area README declares its tracked scope and provides a complete local index. Nested collection +READMEs provide their own complete tables, while the root routes users to recommended paths rather than +duplicating every sample. + +## Documentation -This repository contains a set of samples to use with [AWS Deadline Cloud](https://aws.amazon.com/deadline-cloud/). - -## CloudFormation template samples - -The [cloudformation](cloudformation) directory contains sample CloudFormation templates you can use to -deploy a Deadline Cloud farm or other infrastructure to work with your farm. The [starter_farm sample](cloudformation/farm_templates/starter_farm/) -is a good place to start. Other samples include event notification and health checks for customer-managed fleets. - -## Job bundle samples - -The [job_bundles](job_bundles) directory contains sample jobs that you can submit to your Deadline Cloud queue. You can use the -[Deadline Cloud CLI](https://github.com/aws-deadline/deadline-cloud) to submit these jobs to your queues. - -The [Open Job Description Specifications](https://github.com/OpenJobDescription/openjd-specifications) repository -has more samples that you can use with [AWS Deadline Cloud](https://aws.amazon.com/deadline-cloud/). - -### CLI job submission - -``` -$ deadline bundle submit job_bundles/cli_job -p DataDir=~/data_dir -``` - -### GUI job submission -``` -$ deadline bundle gui-submit job_bundles/gui_control_showcase -``` - -![deadline bundle gui-submit showcase](.images/deadline-bundle-gui-submit-showcase.png) - -## Container samples - -The [containers](containers) directory contains Dockerfiles for building -container images compatible with Deadline Cloud worker environments. The -[al2023-deadline](containers/al2023-deadline/) sample replicates the -service-managed fleet worker AMI package set on Amazon Linux 2023, useful for -building and testing conda packages or other software locally. - -## Conda recipes - -The [conda_recipes](conda_recipes) directory contains samples and tooling for building conda packages for your -Deadline Cloud queues. You can use the `submit-package-job` tool to submit -build jobs to your queue. See [this blog post](https://aws.amazon.com/blogs/media/create-a-conda-package-and-channel-for-aws-deadline-cloud/) -for instructions on how to configure your Deadline Cloud farm for building and -using an Amazon S3 conda channel. - -## Queue environment samples - -The [queue_environments](queue_environments) directory contains -sample [queue environments](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/create-queue-environment.html) -you can attach to your Deadline Cloud queue, to provide software applications to your jobs from -[Conda](https://docs.conda.io/projects/conda/) or [Rez](https://rez.readthedocs.io/). - -## Utility scripts - -The [utility_scripts](utility_scripts) directory contains sample scripts to help you work with -AWS Deadline Cloud. This directory contains command-line tools assist with common tasks like managing job attachments, working with queues, and automating workflows. - -## Submission hook samples - -The [submission_hooks](submission_hooks) directory contains sample -[submission hooks](https://github.com/aws-deadline/deadline-cloud/blob/mainline/docs/submission-hooks.md) -that run custom logic during job submission. The [license_limits](submission_hooks/license_limits/) sample -demonstrates how to enforce fixed license limits (e.g., V-Ray) using Deadline Cloud's Limits feature -combined with a pre-submission hook that automatically injects host requirements into job templates. - -## Additional resources - -* [AWS Deadline Cloud user guide](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/index.html) * [AWS Deadline Cloud developer guide](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/index.html) +* [AWS Deadline Cloud user guide](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/index.html) * [AWS Deadline Cloud API reference](https://docs.aws.amazon.com/deadline-cloud/latest/APIReference/index.html) -* [Open Job Description](https://github.com/OpenJobDescription/openjd-specifications/wiki) +* [Open Job Description specification](https://github.com/OpenJobDescription/openjd-specifications/wiki) +* [Contributing a sample](CONTRIBUTING.md#adding-or-updating-a-sample) ## Security -We take all security reports seriously. When we receive such reports, we will -investigate and subsequently address any potential vulnerabilities as quickly -as possible. If you discover a potential security issue in this project, please -notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) -or directly via email to [AWS Security](aws-security@amazon.com). Please do not -create a public GitHub issue in this project. +If you discover a potential security issue, notify AWS Security through the +[vulnerability reporting page](https://aws.amazon.com/security/vulnerability-reporting/) or +[email AWS Security](mailto:aws-security@amazon.com). Do not create a public GitHub issue. ## License -This library is licensed under the MIT-0 License. See the LICENSE file. +This repository is licensed under the [MIT-0 License](LICENSE). diff --git a/cloudformation/README.md b/cloudformation/README.md index 67d39059..8d98ef87 100644 --- a/cloudformation/README.md +++ b/cloudformation/README.md @@ -1,53 +1,20 @@ # AWS Deadline Cloud sample CloudFormation templates -With [AWS CloudFormation](https://aws.amazon.com/cloudformation/), you can use infrastructure as code to deploy infrastructure -such as a Deadline Cloud farm to your AWS account. Use the samples provided here directly or as a starting point -to create your own custom templates. +With [AWS CloudFormation](https://aws.amazon.com/cloudformation/), you can deploy Deadline Cloud infrastructure as code. Use these samples directly or as starting points for custom templates. -## Starter farm +## Sample index -The [starter_farm](farm_templates/starter_farm/) sample CloudFormation template deploys a Deadline Cloud farm you can use to run jobs that render images, -reconstruct 3D scenes, or transform your data in custom ways. Sample jobs to submit are available in the deadline-cloud-samples on GitHub, Deadline Cloud -provides many integrated submitter plugins for applications, and you can build your own jobs. The deployed farm includes the ability to -[build custom conda packages](../conda_recipes/README.md) for providing additional application support. +This table covers all deployable leaf samples below `cloudformation/`. The two subcategory READMEs provide the same samples grouped by purpose. -## Service-managed fleet with VPC resource endpoint +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [Starter farm](farm_templates/starter_farm/) | A farm, queue, service-managed fleets, and package-build support | You need a general-purpose Deadline Cloud starting environment | +| [CUDA farm](farm_templates/cuda_farm/) | A farm with a CUDA-capable fleet and package-build queue | You need GPU compute for CUDA jobs | +| [SMF with VPC and FSx](farm_templates/smf_vpc_fsx/) | VPC resource endpoints and FSx for OpenZFS shared storage | Service-managed workers need private VPC resources | +| [SMF capacity manager](farm_templates/smf_capacity_manager/) | Balancing Wait and Save and Spot capacity with Lambda and EventBridge Scheduler | A hybrid fleet should maintain target capacity cost-effectively | +| [Fleet standby scheduling](farm_templates/fleet_standby_scheduling/) | Time-based changes to a fleet's warm standby worker count | You want faster business-hours starts without full-time idle capacity | +| [CMF fleet health check](farm_templates/cmf_templates/) | Lambda, EventBridge, CloudWatch alarms, and optional SNS for fleet health | A customer-managed autoscaling fleet needs continuous monitoring | +| [Budget event notifications](notification_templates/budget_events_notification/) | Deadline budget events delivered through SNS and AWS Chatbot | You need email or Slack alerts when budget thresholds are reached | +| [Job event Slack notifications](notification_templates/job_events_slack_lambda/) | EventBridge invoking Lambda to post completion and failure messages | Studio automation should react to Deadline Cloud job state changes | -The [smf_vpc_fsx](farm_templates/smf_vpc_fsx/) sample CloudFormation template demonstrates how to connect -a service-managed fleet to private VPC resources using VPC Lattice. It deploys an FSx for OpenZFS file system -and configures workers to mount it via a VPC resource endpoint. This pattern is useful for accessing shared -storage, license servers, or other private resources from Deadline Cloud workers. - -## Service-managed fleet capacity manager - -The [smf_capacity_manager](farm_templates/smf_capacity_manager/) sample CloudFormation template implements automated -capacity management for hybrid fleet setups that combine Wait and Save and Spot fleets. It uses AWS Lambda and -Amazon EventBridge Scheduler to dynamically balance fleet sizes while maintaining constant total capacity, -optimizing for cost-effective Wait and Save capacity while ensuring any deficit is covered by Spot instances. - -## Budget events notification - -The [budget_events_notification](notification_templates/budget_events_notification/) CloudFormation template sets up an integration -to receive notifications via email and Slack when a budget threshold is reached in the aws.deadline service. It creates an SNS topic, -an EventBridge rule, and a Chatbot configuration to send the notifications. - -## Job event Slack notifications with Lambda - -The [job_events_slack_lambda](notification_templates/job_events_slack_lambda/) CloudFormation template demonstrates -how to connect an AWS Lambda function to Deadline Cloud job events through Amazon EventBridge. It creates an -EventBridge rule that matches job completion and failure events and invokes a Lambda function that posts a -notification to a Slack channel via an incoming webhook. Use it as a starting point for reacting to job events -in your own automation. - -## Scheduled standby workers - -The [fleet_standby_scheduling](farm_templates/fleet_standby_scheduling/) sample CloudFormation template schedules -standby worker count changes on a Deadline Cloud fleet based on a time schedule. It sets a warm standby pool -during business hours and resets it outside business hours to save cost. Works with any existing fleet created -via the console, CLI, or CloudFormation. - -## Customer-managed fleet health checks - -The [cmf_templates](farm_templates/cmf_templates/) collection includes a fleet health check CloudFormation template that sets up -continuous health check monitoring for a single Deadline Cloud customer-managed fleet with autoscaling. It creates a Lambda function, -an EventBridge rule, and a CloudWatch alarm that can be configured with an SNS topic. +Browse the [farm templates](farm_templates/) or [notification templates](notification_templates/) category for a focused index and setup context. diff --git a/cloudformation/farm_templates/README.md b/cloudformation/farm_templates/README.md new file mode 100644 index 00000000..641f2e63 --- /dev/null +++ b/cloudformation/farm_templates/README.md @@ -0,0 +1,18 @@ +# AWS Deadline Cloud farm CloudFormation templates + +These deployable CloudFormation samples create farms or add fleet infrastructure and automation to an existing Deadline Cloud deployment. + +## Sample index + +This table covers every immediate deployable sample directory in `farm_templates/`. + +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [Starter farm](starter_farm/) | A general-purpose farm, queue, service-managed fleets, and package-build support | You need a complete first farm | +| [CUDA farm](cuda_farm/) | A farm with a CUDA-capable fleet and package-build queue | You need GPU workers for CUDA workloads | +| [SMF with VPC and FSx](smf_vpc_fsx/) | Private VPC resource access and FSx for OpenZFS storage | Service-managed workers need shared storage or private services | +| [SMF capacity manager](smf_capacity_manager/) | Automated balancing of Wait and Save and Spot fleet capacity | You operate hybrid service-managed fleets | +| [Fleet standby scheduling](fleet_standby_scheduling/) | Scheduled warm standby worker counts | Worker startup latency matters during predictable hours | +| [CMF fleet health check](cmf_templates/) | Continuous health monitoring for an autoscaling customer-managed fleet | You need alarms for fleet capacity or health problems | + +[`apply-conda-queue-env.py`](apply-conda-queue-env.py) is support tooling used to apply a queue environment; it is not a separately deployable sample and is intentionally excluded from the table. diff --git a/cloudformation/farm_templates/starter_farm/README.md b/cloudformation/farm_templates/starter_farm/README.md index 92f9166d..f6bd3370 100644 --- a/cloudformation/farm_templates/starter_farm/README.md +++ b/cloudformation/farm_templates/starter_farm/README.md @@ -5,7 +5,7 @@ This CloudFormation template deploys an [AWS Deadline Cloud](https://aws.amazon.com/deadline-cloud/) farm you can use to run jobs that render images, reconstruct 3D scenes, or transform your data in custom ways. Sample jobs to submit are available in the -[deadline-cloud-samples on GitHub](https://github.com/aws-deadline/deadline-cloud-samples/tree/mainline/job_bundles#readme), +[deadline-cloud-samples on GitHub](../../../job_bundles/), Deadline Cloud provides many [integrated submitter plugins for applications](https://github.com/aws-deadline/#integrations), and you can [build your own jobs](https://docs.aws.amazon.com/en_us/deadline-cloud/latest/developerguide/building-jobs.html). @@ -53,7 +53,7 @@ your AWS Account. The AWS region should be the same as the one you use to deploy prerequisites, and any parameter customizations: 1. If you want to use the [conda-forge channel](https://conda-forge.org/), change the parameter value for ProdCondaChannels to "deadline-cloud conda-forge". The sample job bundle - [Turntable with Maya/Arnold](https://github.com/aws-deadline/deadline-cloud-samples/tree/mainline/job_bundles/turntable_with_maya_arnold) + [Turntable with Maya/Arnold](../../../job_bundles/turntable_with_maya_arnold/) shows how you can use the FFmpeg provided by conda-forge to encode a video. 2. Edit the fleet configuration parameters if you need a higher vCPU count, more RAM, more EBS bandwidth, etc. 5. Follow the CloudFormation console steps to complete stack creation. @@ -99,10 +99,10 @@ added `conda-forge` to the ProdCondaChannels parameter to the CloudFormation tem [download it as a ZIP](https://github.com/aws-deadline/deadline-cloud-samples/archive/refs/heads/mainline.zip). 2. From the `conda_recipes` directory of `deadline-cloud-samples`, run the following command. If you deployed different fleets than the default, you may need to adjust the conda platforms expression. See - [the conda recipe samples README](https://github.com/aws-deadline/deadline-cloud-samples/tree/mainline/conda_recipes#readme) + [the conda recipe samples README](../../../conda_recipes/) to learn more about this command. ``` - $ ./submit-package-build deadline -p "linux-64*" + $ ./submit-package-job deadline -p "linux-64*" ``` 3. From Deadline Cloud monitor, navigate to the package build queue to watch the job you submitted. When it is running, right click on the task and select "View logs". It may take several minutes as Deadline Cloud @@ -222,10 +222,10 @@ such as [bioconda](https://bioconda.github.io/). The CloudFormation template includes a queue environment that creates conda virtual environments for jobs to use. By default, this is the sample queue environment -[conda_queue_env_improved_caching.yaml](https://github.com/aws-deadline/deadline-cloud-samples/blob/mainline/queue_environments/conda_queue_env_improved_caching.yaml). You can run the provided Python script +[conda_queue_env_improved_caching.yaml](../../../queue_environments/conda_queue_env_improved_caching.yaml). You can run the provided Python script [apply-conda-queue-env.py](../apply-conda-queue-env.py) to substitute a different queue environment. For example, the following command would switch it to the sample -[conda_queue_env_from_console.yaml](https://github.com/aws-deadline/deadline-cloud-samples/blob/mainline/queue_environments/conda_queue_env_from_console.yaml): +[conda_queue_env_from_console.yaml](../../../queue_environments/conda_queue_env_from_console.yaml): ``` $ python ../apply-conda-queue-env.py \ @@ -250,4 +250,4 @@ to dive deeper into this topic. When you strike out on your own from the starter farm sample, you may find it helpful to first delete the `Metadata` section that controls the user interface, and remove the `Conditions` along with where they are used. These sections make a single template more flexible, but when you're using the code to define one particular -farm, they add unnecessary complexity. \ No newline at end of file +farm, they add unnecessary complexity. diff --git a/cloudformation/notification_templates/README.md b/cloudformation/notification_templates/README.md index 180273ab..c20baf7b 100644 --- a/cloudformation/notification_templates/README.md +++ b/cloudformation/notification_templates/README.md @@ -1,14 +1,23 @@ -# AWS Deadline Cloud Event Notification Templates +# AWS Deadline Cloud event notification templates -## Introduction -AWS Deadline Cloud sends events to customer's default event bus https://docs.aws.amazon.com/deadline-cloud/latest/userguide/monitoring-eventbridge.html -This directory holds sample Cloud Formation Templates customers can use to setup notifications based on events received from Deadline Cloud -through mechanisms like email or slack. +AWS Deadline Cloud sends events to the account's default EventBridge event bus. See +[Monitoring Deadline Cloud events with EventBridge](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/monitoring-eventbridge.html) +for service behavior. These CloudFormation templates route selected events to email or Slack integrations. -## Setup Instructions -Each Cloud Formation Template will have its own instructions on how to set up integrations, but in general they follow the below scheme: +## Sample index -1. Download the specified YAML file -2. On the AWS console, go to CloudFormation, Create Stack, and add the downloaded template. -3. Follow the specific instructions for that file, as you may need to enter specific parameters. -4. After this is set up, Deadline cloud events will reach through your setup mechanism. +This table covers every immediate deployable sample directory in `notification_templates/`. + +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [Budget event notifications](budget_events_notification/) | Sending Deadline Cloud budget threshold events through SNS and AWS Chatbot | You need email or Slack budget alerts | +| [Job event Slack notifications](job_events_slack_lambda/) | Matching job events in EventBridge and invoking a Slack-posting Lambda | You need an example service-event integration for job completion or failure | + +## Setup + +Each sample README documents its parameters and integration-specific setup. In general: + +1. Download the sample YAML template. +2. In the AWS CloudFormation console, choose **Create stack** and upload the template. +3. Supply the parameters required by that sample, such as notification destinations or credentials. +4. Confirm that matching Deadline Cloud events reach the configured destination. diff --git a/conda_recipes/README.md b/conda_recipes/README.md index 4a00112d..462813bd 100644 --- a/conda_recipes/README.md +++ b/conda_recipes/README.md @@ -15,6 +15,71 @@ building new packages for either Linux or Windows into it on AWS Deadline Cloud. * Supports [rattler-build](https://prefix-dev.github.io/rattler-build/), and (as deprecated) [conda-build](https://docs.conda.io/projects/conda-build/). +## Recipe index + +This table covers all 49 immediate user-selectable recipe directories in `conda_recipes/`. + +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [After Effects 25.1](aftereffects-25.1/) | Packaging Adobe After Effects for Windows workers | You need the base After Effects 25 application | +| [After Effects plugin bundle](aftereffects-plugin-bundle/) | Bundling multiple supplied After Effects plugins | You want one versioned package for a studio plugin set | +| [After Effects Saber](aftereffects-saber/) | Installing the Saber plugin into an After Effects package | You need a small single-plugin recipe example | +| [AutoDock Vina 1.2.5](autodock-vina-1.2.5/) | Building the AutoDock Vina molecular docking application | You run virtual-screening or docking jobs | +| [AYON Launcher](ayon-launcher/) | Packaging the AYON pipeline runtime for headless publishing | Deadline Cloud workers must participate in an AYON pipeline | +| [Blender 4.2](blender-4.2/) | Packaging Blender 4.2 for Linux and Windows | Your jobs require Blender 4.2 | +| [Blender 4.3](blender-4.3/) | Packaging Blender 4.3 for Linux and Windows | Your jobs require Blender 4.3 | +| [Blender 4.4](blender-4.4/) | Packaging Blender 4.4 for Linux and Windows | Your jobs require Blender 4.4 | +| [Blender 4.5](blender-4.5/) | Packaging Blender 4.5 for Deadline Cloud | Your jobs require Blender 4.5 | +| [Blender 5.0](blender-5.0/) | Packaging Blender 5.0 with plugin-sync support | Your jobs require Blender 5.0 | +| [Blender 5.1](blender-5.1/) | Packaging Blender 5.1 with tested plugin-sync scripts | Your jobs require Blender 5.1 | +| [Blender FLIP Fluids](blender-flipfluids/) | Installing the FLIP Fluids add-on into Blender | You need a Blender simulation add-on recipe | +| [Blender plugin bundle](blender-plugin-bundle/) | Packaging multiple Blender add-on ZIP files together | You deliver a changing studio collection of Blender plugins | +| [Cinema 4D 2024](cinema4d-2024/) | Packaging Cinema 4D 2024 for Windows | Your jobs require Cinema 4D 2024 | +| [Cinema 4D 2025](cinema4d-2025/) | Packaging Cinema 4D 2025 for Windows | Your jobs require Cinema 4D 2025 | +| [Arnold for Cinema 4D 2025](cinema4d-c4dtoa-2025/) | Packaging the C4DtoA renderer plugin | Cinema 4D 2025 jobs render with Arnold | +| [INSYDIUM for Cinema 4D 2025](cinema4d-insydium-2025/) | Packaging supplied INSYDIUM plugin files | Cinema 4D jobs use X-Particles or related plugins | +| [Cinema 4D OpenJD adaptor](cinema4d-openjd/) | Packaging the Cinema 4D integration adaptor | Cinema 4D jobs need OpenJD session integration | +| [V-Ray for Cinema 4D 2025](cinema4d-vray-2025/) | Packaging the V-Ray plugin for Cinema 4D | Cinema 4D 2025 jobs render with V-Ray | +| [Deadline Cloud CLI](deadline/) | Building the `deadline` Python package and command line tools | Another package or worker environment needs the Deadline client | +| [Houdini 20.5](houdini-20.5/) | Packaging Houdini 20.5 with plugin activation support | Your jobs require Houdini 20.5 | +| [Houdini 21.0](houdini-21.0/) | Packaging Houdini 21.0 with Plugin Sync activation | Your jobs require Houdini 21 or frequently updated plugins | +| [Redshift for Houdini 2025](houdini-redshift-2025/) | Packaging Redshift for Houdini 2025 | Houdini 20.5 jobs render with Redshift | +| [Redshift for Houdini 2026](houdini-redshift-2026/) | Packaging Redshift for Houdini 2026 | Houdini 21 jobs render with Redshift | +| [V-Ray 7 for Houdini](houdini-vray-7/) | Packaging V-Ray for Houdini | Houdini jobs render with V-Ray 7 | +| [Infinigen 1.19.0](infinigen-1.19.0/) | Packaging the procedural scene generator and dependencies | You generate synthetic indoor or outdoor scenes | +| [KeyShot 2025](keyshot-2025/) | Packaging KeyShot 2025.2 for Windows | Your jobs render with KeyShot | +| [Maya 2025](maya-2025/) | Packaging Maya and configuring module/plugin search paths | Your jobs require Maya 2025 | +| [Maya 2026](maya-2026/) | Packaging Maya with Plugin Sync activation | Your jobs require Maya 2026 or frequently updated plugins | +| [Bifrost for Maya 2026](maya-bifrost-2026/) | Packaging Autodesk Bifrost for Maya | Maya 2026 jobs use Bifrost graphs or simulations | +| [Arnold for Maya 2025](maya-mtoa-2025/) | Packaging MtoA against the Maya 2025 package | Maya 2025 jobs render with Arnold | +| [Arnold for Maya 2026](maya-mtoa-2026/) | Packaging MtoA against the Maya 2026 package | Maya 2026 jobs render with Arnold | +| [Maya OpenJD adaptor](maya-openjd/) | Packaging the Maya integration adaptor | Maya jobs need OpenJD session integration | +| [Redshift for Maya 2025](maya-redshift-2025/) | Packaging Redshift 2025 for supported Maya versions | Maya jobs use Redshift 2025 | +| [Redshift for Maya 2026](maya-redshift-2026/) | Packaging Redshift 2026 for supported Maya versions | Maya jobs use Redshift 2026 | +| [V-Ray for Maya 2025](maya-vray-2025/) | Packaging V-Ray for Maya 2025 | Maya 2025 jobs render with V-Ray | +| [V-Ray for Maya 2026](maya-vray-2026/) | Packaging V-Ray for Maya 2026 | Maya 2026 jobs render with V-Ray | +| [V-Ray 7.2 for Maya 2025](maya-vray-7.2-2025/) | Pinning V-Ray 7.20.02 to Maya 2025 | You need the exact V-Ray 7.2/Maya 2025 combination | +| [V-Ray 7.2 for Maya 2026](maya-vray-7.2-2026/) | Pinning V-Ray 7.20.02 to Maya 2026 | You need the exact V-Ray 7.2/Maya 2026 combination | +| [Nerfstudio](nerfstudio/) | Packaging Nerfstudio and Gaussian Splatting extras | You train NeRF or Gaussian Splatting models | +| [Nuke 16.0](nuke-16.0/) | Packaging Nuke 16 with plugin activation support | Your compositing jobs require Nuke 16 | +| [Nuke 17.0](nuke-17.0/) | Packaging Nuke 17 with Plugin Sync activation | Your compositing jobs require Nuke 17 or changing plugins | +| [Nuke DENoise](nuke-denoise/) | Packaging the DENoise plugin for Nuke | Nuke jobs need the DENoise node on workers | +| [OpenJD adaptor runtime](openjd-adaptor-runtime/) | Packaging the shared runtime used by DCC adaptors | You are building Maya, Cinema 4D, or other adaptor packages | +| [Unreal Engine](unreal-engine/) | Packaging Unreal Engine, including custom source builds | Unreal workloads need an engine package on workers | +| [Unreal Engine OpenJD adaptor](unreal-engine-openjd/) | Packaging the Unreal integration adaptor | Unreal jobs need OpenJD session integration | +| [V-Ray standalone](vray/) | Packaging the standalone V-Ray renderer | Jobs render `.vrscene` files without a DCC | +| [VRED Core 2025](vredcore-2025/) | Packaging Autodesk VRED Core 2025 for Linux | Automotive visualization jobs require VRED 2025 | +| [VRED Core 2026](vredcore-2026/) | Packaging Autodesk VRED Core 2026 for Linux | Automotive visualization jobs require VRED 2026 | + +## Build and archive support + +[`conda_build_linux_package/`](conda_build_linux_package/) is the reusable OpenJD package-build job, +not a package recipe, so it is intentionally excluded from the recipe table. The top-level +`submit-package-job`, `submit-package-job.bat`, `submit-package-job-script.py`, and +`conda_platform_host_requirements.yaml` files are its submission and platform-support tooling. +[`archive_files/`](archive_files/) stores source or generated package archives and is also excluded; +it is not a user-selectable recipe. + ## Infrastructure setup prerequisites See the Deadline Cloud developer guide documentation @@ -22,7 +87,7 @@ See the Deadline Cloud developer guide documentation for instructions on how to set up a Deadline Cloud farm for building packages into an Amazon S3 conda channel. Name your package build queue "Package Build Queue" for the job submission command to select it by default. -To make this process faster and simpler, you can use our provided starter farm CloudFormation template [here](https://github.com/aws-deadline/deadline-cloud-samples/tree/mainline/cloudformation/farm_templates/starter_farm) to deploy your Deadline infrastructure along with +To make this process faster and simpler, you can use our provided [starter farm CloudFormation template](../cloudformation/farm_templates/starter_farm/) to deploy your Deadline infrastructure along with a configured package build queue as documented in the Deadline Cloud developer guide linked above. To submit package build jobs, you will need the @@ -34,8 +99,7 @@ the CLI using the standalone submitter installer. ## Submitting package build jobs -The command `submit-package-job` is a CLI command for submitting package job provided in this `conda_recipes` -directory. It runs the script [submit-package-job-script.py](submit-package-job-script.py) using the Python +The `submit-package-job` command submits package-build jobs from this `conda_recipes` directory. It runs the script [submit-package-job-script.py](submit-package-job-script.py) using the Python for the Deadline Cloud CLI so it can rely on the `deadline` library dependency being available without additional setup. By default it will submit the job to a queue whose name starts with "Package", and will @@ -190,9 +254,9 @@ This allows you to customize the build process with any supported conda-build or The build arguments are parsed as space-separated values and added to the build command. Use quotes to group arguments that contain spaces. -## Recipe directory structure for `submit-package-build` +## Recipe directory structure for `submit-package-job` -The `submit-package-build` command expects rattler-build recipes in a specific directory structure. It's inspired by the +The `submit-package-job` command expects rattler-build recipes in a specific directory structure. It's inspired by the [conda-forge feedstock repository structure](https://conda-forge.org/docs/maintainer/adding_pkgs/#feedstock-repository-structure). **recipe** @@ -201,7 +265,7 @@ This folder contains the rattler-build recipe, including `recipe.yaml` and packa **deadline-cloud.yaml** -This file is used by the `submit-package-build` command to configure how it submits package build jobs +This file is used by the `submit-package-job` command to configure how it submits package build jobs to Deadline Cloud. **other files** @@ -294,7 +358,7 @@ condaPlatforms: #### The jobParameters list -This list lets the recipe provide parameter values to the job bundle that the `submit-package-job` comamnd uses. +This list lets the recipe provide parameter values to the job bundle that the `submit-package-job` command uses. The format is the same as the [parameter_values.yaml](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle-parameters.html) file of a job bundle. diff --git a/conda_recipes/blender-4.5/README.md b/conda_recipes/blender-4.5/README.md index af5a70e3..21238c02 100644 --- a/conda_recipes/blender-4.5/README.md +++ b/conda_recipes/blender-4.5/README.md @@ -97,7 +97,7 @@ sets for the location of Blender and it's Python. ### Examples Blender Addons - [Blender - Flip Fluids](../blender-flipfluids/) -- [Blender - Plugin Bundle](../blender-plugin-build/) +- [Blender - Plugin Bundle](../blender-plugin-bundle/) ## Troubleshooting diff --git a/conda_recipes/blender-5.0/README.md b/conda_recipes/blender-5.0/README.md index de027b1d..d44fba98 100644 --- a/conda_recipes/blender-5.0/README.md +++ b/conda_recipes/blender-5.0/README.md @@ -97,7 +97,7 @@ sets for the location of Blender and it's Python. ### Examples Blender Addons - [Blender - Flip Fluids](../blender-flipfluids/) -- [Blender - Plugin Bundle](../blender-plugin-build/) +- [Blender - Plugin Bundle](../blender-plugin-bundle/) ## Troubleshooting diff --git a/conda_recipes/houdini-20.5/README.md b/conda_recipes/houdini-20.5/README.md index 3cf49345..ebca5fb2 100644 --- a/conda_recipes/houdini-20.5/README.md +++ b/conda_recipes/houdini-20.5/README.md @@ -46,7 +46,7 @@ Houdini supports plugins through the use of package files. A package is a json f [Houdini Plugin Reference](https://www.sidefx.com/docs/houdini/ref/plugins.html). Create your package files in `$PREFIX/opt/houdini/packages` and point them to the location of your plugins. See our Redshift for -Houdini recipe as [an example](conda_recipes/houdini-redshift-2025). +Houdini recipe as [an example](../houdini-redshift-2025/). ### Plugin Installation Paths diff --git a/conda_recipes/houdini-21.0/README.md b/conda_recipes/houdini-21.0/README.md index b225f23a..b4b87b45 100644 --- a/conda_recipes/houdini-21.0/README.md +++ b/conda_recipes/houdini-21.0/README.md @@ -45,7 +45,7 @@ Houdini supports plugins through the use of package files. A package is a json f [Houdini Plugin Reference](https://www.sidefx.com/docs/houdini/ref/plugins.html). Create your package files in `$PREFIX/opt/houdini/packages` and point them to the location of your plugins. See our Redshift for -Houdini recipe as [an example](conda_recipes/houdini-redshift-2025). +Houdini recipe as [an example](../houdini-redshift-2025/). ### Plugin Installation Paths diff --git a/conda_recipes/keyshot-2025/README.md b/conda_recipes/keyshot-2025/README.md index fa64363b..02b2c42f 100644 --- a/conda_recipes/keyshot-2025/README.md +++ b/conda_recipes/keyshot-2025/README.md @@ -94,7 +94,7 @@ keyshot-2025/ ## Resources - **KeyShot Documentation**: https://www.keyshot.com/resources/ -- **KeyShot Scripting**: https://www.keyshot.com/scripting/ +- **KeyShot Scripting**: https://manual.keyshot.com/manual/scripting-2/ - **AWS Deadline Cloud Developer Guide**: https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/ - **Conda Build Documentation**: https://docs.conda.io/projects/conda-build/ - **KeyShot Network Rendering**: https://www.keyshot.com/network-rendering/ diff --git a/conda_recipes/maya-2025/README.md b/conda_recipes/maya-2025/README.md index 91c27cf0..103a7986 100644 --- a/conda_recipes/maya-2025/README.md +++ b/conda_recipes/maya-2025/README.md @@ -14,7 +14,7 @@ package, place its `.mod` file in one of these so that Maya loads the plugin at Download the Autodesk_Maya_2025_Linux_64bit.tgz full download file from Autodesk, and place it in the `conda_recipes/archive_files` directory in your git clone of the -[https://github.com/aws-deadline/deadline-cloud-samples](deadline-cloud-samples) repository for +[deadline-cloud-samples](https://github.com/aws-deadline/deadline-cloud-samples) repository for submitting package build jobs, ## Creating an archive file for Windows diff --git a/conda_recipes/maya-2026/README.md b/conda_recipes/maya-2026/README.md index b2dc5bae..721aec37 100644 --- a/conda_recipes/maya-2026/README.md +++ b/conda_recipes/maya-2026/README.md @@ -14,6 +14,6 @@ package, place its `.mod` file in one of these so that Maya loads the plugin at Download the Autodesk_Maya_2026_ML_Linux_64bit.tgz full download file from Autodesk, and place it in the `conda_recipes/archive_files` directory in your git clone of the -[https://github.com/aws-deadline/deadline-cloud-samples](deadline-cloud-samples) repository for +[deadline-cloud-samples](https://github.com/aws-deadline/deadline-cloud-samples) repository for submitting package build jobs. diff --git a/conda_recipes/maya-redshift-2025/README.md b/conda_recipes/maya-redshift-2025/README.md index 7a845b62..ed27eb25 100644 --- a/conda_recipes/maya-redshift-2025/README.md +++ b/conda_recipes/maya-redshift-2025/README.md @@ -11,7 +11,7 @@ This package provides Redshift 2025.4.2 support for Maya versions 2025, and 2026 Download the redshift_2025.4.2_1782753868_linux_x64.run installer, or suitable alternate version from Maxon, and place it in the `conda_recipes/archive_files` directory in your git clone of the -[https://github.com/aws-deadline/deadline-cloud-samples](deadline-cloud-samples) repository for +[deadline-cloud-samples](https://github.com/aws-deadline/deadline-cloud-samples) repository for submitting package build jobs. Please note that if the installer version used differs from "redshift_2025.4.2_1782753868_linux_x64.run", version diff --git a/conda_recipes/maya-redshift-2026/README.md b/conda_recipes/maya-redshift-2026/README.md index 725466f5..2389af0f 100644 --- a/conda_recipes/maya-redshift-2026/README.md +++ b/conda_recipes/maya-redshift-2026/README.md @@ -11,7 +11,7 @@ This package provides Redshift 2026.2.1 support for Maya versions 2025, and 2026 Download the redshift_2026.2.1_2202748377_linux_x64.run installer, or suitable alternate version from Maxon, and place it in the `conda_recipes/archive_files` directory in your git clone of the -[https://github.com/aws-deadline/deadline-cloud-samples](deadline-cloud-samples) repository for +[deadline-cloud-samples](https://github.com/aws-deadline/deadline-cloud-samples) repository for submitting package build jobs. Please note that if the installer version used differs from "redshift_2026.2.1_2202748377_linux_x64.run", version diff --git a/conda_recipes/nuke-16.0/README.md b/conda_recipes/nuke-16.0/README.md index c54663f9..12a7f5a0 100644 --- a/conda_recipes/nuke-16.0/README.md +++ b/conda_recipes/nuke-16.0/README.md @@ -98,7 +98,6 @@ nuke-16.0/ - **Nuke Documentation**: https://learn.foundry.com/nuke/ - **AWS Deadline Cloud Developer Guide**: https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/ - **Conda Build Documentation**: https://docs.conda.io/projects/conda-build/ -- **Plugin Development**: https://learn.foundry.com/nuke/developers/ --- diff --git a/conda_recipes/nuke-17.0/README.md b/conda_recipes/nuke-17.0/README.md index 11554a34..35e400d0 100644 --- a/conda_recipes/nuke-17.0/README.md +++ b/conda_recipes/nuke-17.0/README.md @@ -101,7 +101,6 @@ nuke-17.0/ - **Nuke Documentation**: https://learn.foundry.com/nuke/ - **AWS Deadline Cloud Developer Guide**: https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/ - **Conda Build Documentation**: https://docs.conda.io/projects/conda-build/ -- **Plugin Development**: https://learn.foundry.com/nuke/developers/ --- diff --git a/conda_recipes/nuke-denoise/README.md b/conda_recipes/nuke-denoise/README.md index 207aea0c..c33c7056 100644 --- a/conda_recipes/nuke-denoise/README.md +++ b/conda_recipes/nuke-denoise/README.md @@ -113,7 +113,7 @@ nuke-denoise/ - **OpenFX Standard**: http://openeffects.org/ - **AWS Deadline Cloud Developer Guide**: https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/ - **Conda Build Documentation**: https://docs.conda.io/projects/conda-build/ -- **Nuke Plugin Development**: https://learn.foundry.com/nuke/developers/ +- **Nuke Documentation**: https://learn.foundry.com/nuke/ --- diff --git a/containers/README.md b/containers/README.md index f8bdc8d5..9051b375 100644 --- a/containers/README.md +++ b/containers/README.md @@ -1,18 +1,14 @@ # AWS Deadline Cloud container samples -The container samples in this directory provide Dockerfiles and related resources -for building container images compatible with -[AWS Deadline Cloud](https://aws.amazon.com/deadline-cloud/) worker environments. +These samples provide Dockerfiles and related resources for building container images compatible with [AWS Deadline Cloud](https://aws.amazon.com/deadline-cloud/) worker environments. -Use these to build and test software locally with the same system libraries, -toolchains, and runtime environment as Deadline Cloud workers. +## Sample index -## Samples +This table covers both user-selectable container samples below `containers/`; supporting scripts and image assets remain with their sample. -### AL2023 worker-equivalent image +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [AL2023 worker-equivalent image](al2023-deadline/) | Reproducing a point-in-time service-managed fleet package set on Amazon Linux 2023 | You need to test packages or software against worker-compatible system libraries | +| [Blender application container](blender/blender-aswf-ci-base/) | Packaging Blender, the Deadline Cloud adaptor, and GPU support in an application image | You want to render Blender workloads from a purpose-built container | -The [al2023-deadline](al2023-deadline/) sample provides a Dockerfile that -replicates the package set of the Deadline Cloud service-managed fleet (SMF) -worker AMI on top of the base Amazon Linux 2023 image. Use it to build and test -[conda packages](../conda_recipes/) or other software that must be compatible -with the worker runtime. +The worker-equivalent image is useful for local compatibility work and package builds. The Blender image is an application-container example and includes its own deployment resources and instructions. diff --git a/docs/SAMPLE_README_TEMPLATE.md b/docs/SAMPLE_README_TEMPLATE.md new file mode 100644 index 00000000..b1e86399 --- /dev/null +++ b/docs/SAMPLE_README_TEMPLATE.md @@ -0,0 +1,62 @@ +# Sample title + + + +One or two sentences explaining what the sample accomplishes and when a user should choose it. + +## What this sample demonstrates + +Describe the Deadline Cloud capability or OpenJD pattern, the important delivery or lifecycle choice, +and the expected result. Omit or combine this section if the introduction already makes that clear. + +## Prerequisites + +Document required AWS resources and permissions, local tools and versions, and any application, +plugin, or license access users must provide. + +## How it works + +Explain the components and data flow needed to operate the sample safely. Link detailed architecture +to canonical documentation or a focused design document rather than repeating it here. + +## Setup + +Provide deterministic setup instructions and identify configuration values users must replace. + +## Run or submit + +Show the shortest working command first, followed by meaningful variants when useful. + +```console +# command +``` + +## Parameters and outputs + +Describe important inputs, defaults, output locations, and artifacts or resources the sample creates. +A compact table is often useful, but use whichever format fits the sample. + +## Security, cost, and cleanup + +Call out permission boundaries, secret handling, network exposure, billable resources, licensing, +and cleanup steps that apply. Do not embed credentials or private data. + +## Troubleshooting + +List likely, diagnosable failures and where users can find relevant worker, job, or service logs. + +## Related resources + +Link canonical AWS Deadline Cloud documentation and closely related samples when those links help the +reader choose a next step. diff --git a/host_configuration_scripts/3dsmax/README.md b/host_configuration_scripts/3dsmax/README.md index b305ba5e..5f5adc50 100644 --- a/host_configuration_scripts/3dsmax/README.md +++ b/host_configuration_scripts/3dsmax/README.md @@ -1,47 +1,60 @@ -# Sample Host Configuration scripts to install 3ds Max to Service Managed Fleets for AWS Deadline Cloud +# 3ds Max host configuration scripts for AWS Deadline Cloud -This folder contains sample host configuration scripts you can use to configure your AWS Deadline Cloud Windows Service Managed Fleets to install and run 3ds Max jobs on your workers. -Please see the README.md in each sample script for more steps on how to set it up. +These Windows host configuration scripts install 3ds Max and selected renderers or plugins on AWS Deadline Cloud service-managed fleet workers. 3ds Max requires administrative installation, so host configuration is the recommended delivery boundary. -## 3ds Max -3ds Max is a popular Digital Content Creation tool provided by Autodesk. 3ds Max runs on Windows, and requires administrative access to install onto a host. Because of the administrative requirement, Deadline Cloud recommends installing 3ds Max on to the worker host using Host Configuration Scripts. +## Sample index -- Note: While the example installs 3ds Max 2024 and 2025, Deadline Cloud's submitter supports 3ds Max 2026 and 2027 as well. The installation script should work equivalently for 3ds Max 2026 and 2027. +This table covers every immediate sample directory in `host_configuration_scripts/3dsmax/`. -## Generating a script for your version using Kiro +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [3ds Max 2024](3dsmax-2024/) | Installing the base 2024 application | You need 3ds Max 2024 without bundled render plugins | +| [3ds Max 2025 and Corona 13](3dsmax-2025-and-corona-13/) | Installing 3ds Max with Corona Renderer | Your 2025 scenes render with Corona 13 | +| [3ds Max 2025 and V-Ray](3dsmax-2025-and-vray/) | Installing 3ds Max and V-Ray together | Your 2025 scenes use V-Ray | +| [3ds Max 2025, V-Ray, and AEC plugins](3dsmax-2025-vray-and-aec-plugins/) | Adding Forest Pack, RailClone, and architectural visualization plugins | A 2025 V-Ray workload uses common AEC plugins | +| [3ds Max 2025, V-Ray, and tyFlow](3dsmax-2025-vray-and-tyflow/) | Installing V-Ray and tyFlow with 3ds Max | A 2025 workload combines rendering and particle simulation | +| [3ds Max 2027](3dsmax-2027/) | Installing the base 2027 application | You need 3ds Max 2027 without bundled render plugins | +| [3ds Max 2027 and Corona 14](3dsmax-2027-and-corona-14/) | Installing the first Corona version supporting 3ds Max 2027 | Your 2027 scenes render with Corona 14 | +| [3ds Max 2027 and V-Ray](3dsmax-2027-and-vray/) | Installing 3ds Max 2027 and V-Ray | Your 2027 scenes use V-Ray | +| [3ds Max 2027, V-Ray, and tyFlow](3dsmax-2027-and-vray-and-tyflow/) | Installing V-Ray and tyFlow with 3ds Max 2027 | A 2027 workload combines rendering and particle simulation | +| [3ds Max 2027, V-Ray, and AEC plugins](3dsmax-2027-vray-and-aec-plugins/) | Adding Forest Pack, RailClone, FloorGenerator, and MultiTexture | A 2027 V-Ray workload uses architectural visualization plugins | -The sample scripts in this folder cover specific version combinations. If you need a script for a different version of 3ds Max, a different renderer, or a different plugin combination, you can use [Kiro](https://kiro.dev) to generate one for you. +The samples currently cover 3ds Max 2024, 2025, and 2027. The Deadline Cloud submitter also supports 3ds Max 2026; adapt the nearest script for that installer and verify all product-specific silent-install options. + +## Generate a script for another version with Kiro + +The samples cover specific combinations. To create a script for another 3ds Max version, renderer, or plugin combination, you can use [Kiro](https://kiro.dev) with this repository. ### Prerequisites -- [Kiro](https://kiro.dev) installed -- This repository cloned and opened as a workspace in Kiro +* Install [Kiro](https://kiro.dev). +* Clone this repository and open it as the Kiro workspace. ### Steps -1. In the Kiro chat, type a request like: - - `"Create a host configuration script for 3ds Max 2026"` - - `"Create a host configuration script for 3ds Max 2026 and V-Ray 8"` - - `"Create a host configuration script for 3ds Max 2027 and Corona 14"` - - `"Add a host configuration script for 3ds Max 2026 with Forest Pack 10"` -2. Kiro will read the skill in `skills/3dsmax-host-config/SKILL.md` and generate the correct script and README for your version combination. -3. Review the generated script, fill in the `TODO` variables at the top (your S3 bucket name, installer file names), and configure your fleet. +1. Ask for the combination you need, for example: + * `Create a host configuration script for 3ds Max 2026` + * `Create a host configuration script for 3ds Max 2026 and V-Ray 8` + * `Create a host configuration script for 3ds Max 2027 and Corona 14` + * `Add a host configuration script for 3ds Max 2026 with Forest Pack 10` +2. Kiro reads [`skills/3dsmax-host-config/SKILL.md`](../../skills/3dsmax-host-config/SKILL.md) and generates a script and README for the requested combination. +3. Review and test the generated script, replace its `TODO` values with your S3 bucket and installer names, and then configure the fleet. -## Common Prerequisites -- Each sample requires you to have the 3ds Max installer in an S3 bucket in your AWS account. You can download the 3ds Max installer directly from Autodesk. See the next section for instructions on how to properly package the installer files. -- The host configuration scripts will download the installers from your S3 bucket, so your Fleet roles will need to be granted s3:GetObject permissions for the installers in S3. +## Common prerequisites -## Creating a 3ds Max installer archive in .zip format -Autodesk provides 3ds Max as a .7z archive which cannot be easily extracted from the command line without 3rd party software like [7-zip](https://www.7-zip.org/). To get around this problem, the examples in this folder expect a .zip archive instead. You can create a .zip archive with the following steps: +* Download each licensed installer from its vendor and place it in an S3 bucket in your account. +* Grant the fleet role `s3:GetObject` for the installer objects. +* Review installer versions, checksums where available, silent flags, licensing, and restart requirements before using a script. -1. Navigate to the [Products and Services page on the Autodesk Website](https://manage.autodesk.com/products), sign into your Autodesk account, and click View details under 3ds Max. -image - -2. Select your version and then click the dropdown icon next to the **Download** button and choose **Direct Download**. Note that this dropdown has different options than the one on the previous page. This will download a .7z and a .exe file. -image +## Creating a 3ds Max installer archive in .zip format -3. With both the .7z and .exe file in the same folder, double-click the .exe file and wait for it to extract the .7z for you. When the extraction is done, choose **Open in folder**. -image +Autodesk distributes 3ds Max as a `.7z` archive plus an extraction executable. The samples expect a ZIP so Windows can extract it without third-party software such as [7-Zip](https://www.7-zip.org/). -4. Finally, select all files in the resulting folder and right-click them to bring up the context menu. Choose **Send to > Compressed (zipped) folder**. -image +1. Open the [Autodesk Products and Services page](https://manage.autodesk.com/products), sign in, and choose **View details** for 3ds Max. + Autodesk product details page +2. Select the version, open the menu beside **Download**, and choose **Direct Download**. Keep the downloaded `.7z` and `.exe` in the same folder. + Autodesk direct download menu +3. Run the `.exe`, wait for extraction, and choose **Open in folder**. + Autodesk extraction completion dialog +4. Select all extracted files and choose **Send to > Compressed (zipped) folder**. + Windows compressed folder menu diff --git a/host_configuration_scripts/README.md b/host_configuration_scripts/README.md index b7a61def..08fda840 100644 --- a/host_configuration_scripts/README.md +++ b/host_configuration_scripts/README.md @@ -1,45 +1,46 @@ -# Sample Host Configuration Scripts for AWS Deadline Cloud Service Managed Fleets +# Sample host configuration scripts for AWS Deadline Cloud service-managed fleets -## Summary +Host configuration scripts run with elevated privileges on service-managed fleet workers. Use them for administrative tasks such as software installation, system tuning, GPU container setup, and worker restart behavior. -This directory contains sample scripts for configuring Service Managed Fleets on Windows and Linux. +## Sample index -Host Configuration Scripts allow you to perform administrative tasks, such as software installation, on your service-managed fleet workers. These scripts run with elevated privileges, giving you the flexibility to configure your workers for your system. +This table covers every immediate user-selectable group or leaf directory in `host_configuration_scripts/`. The application groups link to their own installer examples; implementation scripts remain inside each sample. -## Examples +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [3ds Max](3dsmax/) | Windows installation for multiple 3ds Max, renderer, and plugin combinations | You need 3ds Max on service-managed workers | +| [After Effects and Red Giant](aftereffects/) | Installing After Effects with optional Red Giant plugins | You need Adobe and Maxon software installed as administrator | +| [Cinema 4D and Red Giant](cinema4d/) | Installing Cinema 4D with Red Giant plugins | You need Cinema 4D and Maxon plugins on Windows workers | +| [Docker and NVIDIA Container Toolkit](docker_nvidia_container_toolkit/) | Installing Docker and the NVIDIA runtime on Linux GPU workers | Jobs run GPU-accelerated containers | +| [Linux font installation](linux_font_installation/) | Downloading fonts from S3 and registering them system-wide | Render applications need studio fonts | +| [Memory overcommit override](overcommit_override_for_smf/) | Changing `vm.overcommit_memory` on Linux workers | Large attachments or allocations fail despite free memory | +| [Passwordless sudo for job user](sudo_for_job_user/) | Granting `job-user` unrestricted sudo | A trusted workload requires root commands during tasks | +| [Swap for SMF](swap_for_smf/) | Creating and enabling a Linux swap file | A workload can temporarily exceed physical memory | +| [Worker configuration](worker_configuration/) | Windows system configuration such as page-file sizing | Workers need OS-level tuning before jobs start | +| [Worker reboot](worker_reboot/) | Rebooting Linux or Windows after host setup | Drivers, domain joins, or other changes require restart | -| Example | Platform | Description | -|---------|----------|-------------| -| [3dsmax](3dsmax/) | Windows | Install and configure Autodesk 3ds Max with various renderer plugins (V-Ray, Corona, tyFlow, AEC) | -| [aftereffects](aftereffects/) | Windows | Install After Effects with Red Giant plugins | -| [cinema4d](cinema4d/) | Windows | Install Cinema 4D with Red Giant plugins | -| [docker_nvidia_container_toolkit](docker_nvidia_container_toolkit/) | Linux | Install Docker and NVIDIA Container Toolkit for GPU-accelerated container workloads | -| [linux_font_installation](linux_font_installation/) | Linux | Install custom fonts from S3 for rendering applications | -| [sudo_for_job_user](sudo_for_job_user/) | Linux | Grant passwordless sudo to `job-user` for workloads that require root access | -| [swap_for_smf](swap_for_smf/) | Linux | Enable swap for memory-intensive workloads like ComfyUI and large diffusion models | -| [worker_configuration](worker_configuration/) | Windows | Configure Windows page file settings | -| [worker_reboot](worker_reboot/) | Linux / Windows | Reboot the worker after host configuration (e.g. after driver installs or domain joins) | +## Common uses -## Common Uses -- Installing software that requires administrator access -- Installing Docker containers -- Configuring GPU runtimes for containerized workloads -- Enabling swap for memory-intensive jobs +* Install software that requires administrator access. +* Install and configure container runtimes. +* Configure GPU support for containerized workloads. +* Tune memory, swap, fonts, or other host-wide settings. ## Setup -Copy and paste the sample scripts into the AWS Deadline Cloud console or use the AWS Deadline Cloud CLI to update your fleet. Follow [Run scripts as an administrator to configure workers](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/smf-admin.html) or reference the [AWS Deadline Cloud update-fleet CLI](https://docs.aws.amazon.com/cli/latest/reference/deadline/update-fleet.html) for more details. +Copy the selected script into the Deadline Cloud console or use the Deadline Cloud CLI to update the fleet. Follow [Run scripts as an administrator to configure workers](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/smf-admin.html) and the [`update-fleet` CLI reference](https://docs.aws.amazon.com/cli/latest/reference/deadline/update-fleet.html). Review every script and replace its configuration values before deployment because it runs with administrator privileges. ## Debugging -### CloudWatch Logs -Fleet Host Configuration logs are streamed to the Fleet’s log group, and specifically to a worker’s log stream. For example, `/aws/deadline/farm-12345/fleet-09876` is the log group for farm-12345, fleet-09876. Each worker will provision a dedicated log stream, for example worker-13579. Notice in the logs the log banner “Running Host Configuration Script” and “Finished running Host Configuration Script, exit code: 0”. The exit code of the script is included in the finished banner, and can be queried using CloudWatch tools. +### CloudWatch Logs -### CloudWatch Log Insights +Host configuration logs are streamed to the fleet log group and a stream dedicated to each worker. For example, `/aws/deadline/farm-12345/fleet-09876` can contain a `worker-13579` stream. Look for the “Running Host Configuration Script” and “Finished running Host Configuration Script, exit code: 0” banners. -CloudWatch Log Insights offers advanced capabilities to datamine the log information. For example, the following log insight query parses for the host configuration exit code, sorted by time. +### CloudWatch Logs Insights -``` +This query extracts host configuration exit codes in reverse chronological order: + +```text fields @timestamp, @message, @logStream, @log | filter @message like /Finished running Host Configuration Script/ | parse @message /exit code: (?\d+)/ diff --git a/job_bundles/README.md b/job_bundles/README.md index d940b6fd..4613a46e 100644 --- a/job_bundles/README.md +++ b/job_bundles/README.md @@ -2,133 +2,83 @@ Job bundles are the easiest way to define your jobs for AWS Deadline Cloud. They encapsulate an [Open Job Description job template](https://github.com/OpenJobDescription/openjd-specifications/wiki) into a directory -with additional information such the files and directories that your Jobs need. Read more about +with additional information such as the files and directories that your jobs need. Read more about how to [build a job bundle](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle.html) in the Deadline Cloud developer guide. See the [example Blender job submission](#example-blender-job-submission) below for more about submitting these jobs to your farm. ## Job bundle index -This list highlights just a few of the available job bundles. Browse the directory directly to discover the rest! - -### Jobs for the Deadline Cloud developer guide - -The [simple_job](simple_job/template.yaml) job bundle supplements the Deadline Cloud developer guide -[set up a developer farm section](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/getting-started-dev.html). -You can step through its instructions to get a developer-focused overview, using the AWS and Deadline Cloud CLIs to create a farm, queue, -and fleets, submit jobs, and see the details of how job attachments work. - -The job bundles [job_env_vars](job_env_vars/template.yaml), [job_env_with_new_command](job_env_with_new_command/template.yaml), and -[job_env_daemon_process](job_env_daemon_process/template.yaml) supplement the developer guide -[control the job environment section](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/control-the-job-environment.html). -These jobs show how to use [Open Job Description environments](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#4-environment). -Look at the [queue environment samples](../queue_environments) for more ideas. - -The job bundles [job_attachments_devguide](job_attachments_devguide) and [job_attachments_devguide_output](job_attachments_devguide_output) -supplement the developer guide [job attachments section](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-attachments.html). -Learn how data flow metadata on path job parameters and the job bundle `asset_references.yaml` file work together to describe the files -a job needs as input, and produce as output. When job bundles specify this metadata, they can work with either job attachments or shared file systems. - -### CLI job - -The [cli_job](cli_job/template.yaml) job bundle is a way to submit a multi-line bash script to Deadline Cloud. The script job parameter uses a multi-line -edit control, and a data directory job parameter lets you select a directory of data for the script to read from and write to. - -### GUI control showcase - -The [gui_control_showcase](gui_control_showcase/template.yaml) job bundle shows every GUI control that user interface metadata -on [Open Job Description job parameters](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#2-jobparameterdefinition) -support. - -### Job development progression - -Developing a job bundle can start small and simple, then grow complex as you add more job parameters, steps, and scripts. -The [job_dev_progression](job_dev_progression) directory contains a sequence of four job bundle development stages to help -manage that growing complexity. Read through the code and run these jobs on your Deadline Cloud farm to get a feel for it. - -### Digital content creation (DCC) render jobs - -The [blender_render](blender_render/template.yaml) job bundle shows how to support a CLI application in about 100 lines of YAML. A majority of -the template is metadata for the job parameters, defining the parameter names, types, defaults, and user interface metadata. -The step definition includes a parameter space to define a task for each frame for a range expression in the Frames job parameter, -and a short script that substitutes job parameters and the Frame task parameter into a script command for each task. - -* [3dsmax_vray_denoiser_example](3dsmax_vray_denoiser_example) - 3ds Max V-Ray rendering with smart frame chunking and VRIMG to EXR conversion -* [arnold_standalone_render](arnold_standalone_render) - Arnold standalone rendering of .ass files using `kick` -* [blender_render](blender_render/template.yaml) -* [keyshot_standalone](keyshot_standalone) -* [afterfx_render_one_task](afterfx_render_one_task) -* [maya_arnold_ass_export_render](maya_arnold_ass_export_render) - Export .ass from Maya and render with Arnold `kick` -* [maya_cli_render](maya_cli_render) -* [houdini_husk_usd_render](houdini_husk_usd_render) -* [nuke_render](nuke_render) -* [vray_render](vray_render/template.yaml) - -If you've created a similar job for your favorite DCC, see [CONTRIBUTING.md](../CONTRIBUTING.md) for how to add it here. - -### 3D Gaussian Splatting pipeline - -The [gsplat_pipeline](gsplat_pipeline/README.md) job bundle can take a video file as input and train a 3D Gaussian Splatting point cloud. -This example shows Deadline Cloud running a 3D reconstruction workload that uses CUDA GPUs for acceleration. - -### LLM evaluation with vLLM and lm-evaluation-harness - -The [vllm_lm_eval_leaderboard](vllm_lm_eval_leaderboard/README.md) job bundle evaluates multiple LLMs on a set of benchmarks in a single Deadline Cloud job using -[vLLM](https://github.com/vllm-project/vllm) for inference and [EleutherAI's lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) for scoring. -Models are a STRING parameter sweep — each task starts vLLM for one model, runs every benchmark against it, then stops vLLM. A final aggregation step -produces a ranked leaderboard (CSV + Markdown). - -### Arnold standalone render - -The [arnold_standalone_render](arnold_standalone_render) job bundle renders Arnold `.ass` (Arnold Scene Source) files -using the `kick` command-line renderer from MtoA. This is useful for batch rendering pre-exported Arnold scenes without -requiring a full Maya session. It includes a sample Cornell box `.ass` file for testing. You can also download sample -scenes from the [Autodesk Arnold learning scenes page](https://help.autodesk.com/view/MAYAUL/2024/ENU/?guid=arnold_for_maya_tutorials_am_Learning_Scenes_html). - -### Maya Arnold export and render - -The [maya_arnold_ass_export_render](maya_arnold_ass_export_render) job bundle is a two-step pipeline that exports -`.ass` files from a Maya scene and renders them with Arnold `kick`. The export step opens the Maya scene once and -exports all frames, then the render step distributes per-frame `kick` tasks across workers. This is useful when you -want to render Arnold scenes directly from Maya files without pre-exporting. - -### Turntable job with Maya/Arnold - -The [turntable_with_maya_arnold](turntable_with_maya_arnold) job bundle is an example pipeline utility job for taking a 3D model -stored as an OBJ file, and creating a turntable render video. It demonstrates how someone comfortable with YAML -and scripting in a digital content creation (DCC) application can create utility jobs that are easy to submit from a GUI. - -### Tile render job with Maya/Arnold and FFmpeg - -The [tile_render_with_maya_arnold](tile_render_with_maya_arnold) job bundle demonstrates a two-step job that first renders -all the frames in tiles using a 3-dimensional task parameter space (Frame * TileNumberX * TileNumberY), and then assembles -all the tiles for each frame using FFmpeg. It includes a simple scene and default parameters to make it simple to try out. - -The [tile_render_maya_ffmpeg_for_blogpost](tile_render_maya_ffmpeg_for_blogpost) job bundle goes with the blog post -[Create a tile rendering job with modifications for AWS Deadline Cloud](https://aws.amazon.com/blogs/media/create-a-tile-rendering-job-with-modifications-for-aws-deadline-cloud/) -that walks through customizing one of the Deadline Cloud adaptors and writing a tile rendering job template. - -### Copy S3 prefix to job attachments - -The [copy_s3_prefix_to_job_attachments](copy_s3_prefix_to_job_attachments) job bundle can help you pre-populate a queue's -job attachment S3 bucket with data files by copying them from where they are already stored on S3. It scans the source -S3 prefix, then distributes the hashing and data copies across a number of workers you specify. Because job attachments -uses content-addressed storage for data files, users that later submit jobs with these files attached will not have to -upload them. - -### FFmpeg movie from job output - -The [ffmpeg_movie_from_job_output](ffmpeg_movie_from_job_output) job bundle downloads the rendered output of another -completed job in the same queue and uses FFmpeg to encode the image sequence into an MP4 video. This is useful as a -post-processing utility — after a render job completes, submit this job with the source Job ID to automatically -assemble the frames into a movie with configurable frame rate, quality, and resolution settings. - -### SSH via SSM Managed Node - -The [ssh_to_smf](ssh_to_smf/README.md) job bundle registers a Deadline Cloud worker as an -AWS Systems Manager hybrid managed node, enabling interactive SSH access via Session Manager for the duration of the job. -A submit script handles creating the SSM hybrid activation and passing the credentials as job parameters. Requires -one-time account setup (IAM role + advanced-instances tier). See the bundle's README for full setup instructions. +This table covers every immediate user-selectable sample directory or collection in `job_bundles/`. +Nested collections provide their own complete indexes where applicable. + +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [3ds Max V-Ray denoiser](3dsmax_vray_denoiser/) | V-Ray rendering, frame chunking, and VRIMG-to-EXR conversion with denoising data | You render 3ds Max scenes with V-Ray and need denoised EXR output | +| [After Effects one-task render](afterfx_render_one_task/) | Running an entire `aerender` frame range as one task | A composition must stay on one worker for the full render | +| [Arnold standalone render](arnold_standalone_render/) | Rendering Arnold `.ass` files with `kick` | Your scenes are already exported and do not need Maya at render time | +| [CARLA autonomous-driving simulation](autonomous_driving_carla/) | GPU container simulation, parameter sweeps, and multi-sensor capture | You want to distribute autonomous-driving scenarios | +| [Blender render](blender_render/) | A compact frame-parallel DCC render job with application packages | You need a minimal Blender or general CLI-render starting point | +| [Blender turntable to Flow](blender_turntable_to_flow/) | Rendering, encoding, thumbnail extraction, and publishing to Flow Production Tracking | You are building a render-to-review studio workflow | +| [Bash CLI job](cli_job/) | Submitting a multi-line shell script with an attached data directory | You want to run an ad hoc command-line workload | +| [Copy S3 prefix to job attachments](copy_s3_prefix_to_job_attachments/) | Distributed hashing and copying into content-addressable job attachment storage | Existing S3 datasets should be staged without workstation uploads | +| [Custom submitters](custom_submitters/) | A collection of in-application submission interfaces, including Maya | Artist context and DCC state require a custom submission UI | +| [ESMFold prediction](esmfold_predict/) | Parallel protein structure prediction, validation, and rendering | You need a GPU bioinformatics pipeline from FASTA to PDB outputs | +| [FFmpeg encode video](ffmpeg_encode_video/) | Encoding a numbered image sequence into MP4 | You need a standalone render-output encoding utility | +| [FFmpeg movie from job output](ffmpeg_movie_from_job_output/) | Downloading another job's output and encoding it downstream | Post-processing should be a separately submitted follow-up job | +| [FLUX.2 Klein LoRA](flux2_klein_lora/) | A collection for LoRA training and image generation on GPUs | You want to fine-tune FLUX.2 Klein and generate images | +| [GROMACS molecular dynamics](gromacs_md/) | Molecular-dynamics stages and scientific result visualization | You want to distribute protein simulation work | +| [Gaussian Splatting pipeline](gsplat_pipeline/) | Video-frame extraction, structure from motion, GPU training, and point-cloud output | You need a multi-step 3D reconstruction workflow | +| [GUI control showcase](gui_control_showcase/) | Every OpenJD job-parameter GUI control and UI metadata option | You are designing a bundle submission interface | +| [Houdini Husk USD render](houdini_husk_usd_render/) | USD dependency discovery and rendering with Husk/Karma | You need a concise USD render job with asset introspection | +| [Infinigen scene generation](infinigen_scene_gen/) | Procedural indoor and outdoor scene generation on GPU workers | You need synthetic photorealistic datasets | +| [Job attachments input guide](job_attachments_devguide/) | Input path metadata and attached script files | You are learning how job attachment inputs are materialized | +| [Job attachments output guide](job_attachments_devguide_output/) | Collecting declared job output files | You are learning how job attachment outputs are returned | +| [Job development progression](job_dev_progression/) | Four stages from inline commands to a tested bundled Python package | You want to grow a maintainable job without starting complex | +| [Daemon-process environment](job_env_daemon_process/) | Starting a background process once and sharing it across tasks | Application startup should be amortized within a session | +| [Environment variables](job_env_vars/) | Setting variables at job and step scope with OpenJD environments | Tasks need consistent runtime configuration | +| [Environment-provided command](job_env_with_new_command/) | Creating a command and adding it to `PATH` for job steps | Setup should expose reusable tooling to every step | +| [KeyShot standalone](keyshot_standalone/) | Frame-parallel KeyShot rendering on Windows | You render KeyShot scenes with the standalone interface | +| [List available Conda packages](list_available_conda_packages/) | Querying a Conda channel from a Deadline Cloud job | You need to inspect packages visible to workers | +| [Maya Arnold export and render](maya_arnold_ass_export_render/) | Exporting `.ass` once, then rendering frames with Arnold `kick` | You want separate DCC export and renderer-only steps | +| [Maya CLI render](maya_cli_render/) | Rendering a Maya scene with the CLI `Render` command | You need a small Maya command-line example | +| [Monte Carlo simulation](monte_carlo_simulation/) | Parallel financial simulation followed by result aggregation | You want a non-rendering fan-out/fan-in workload | +| [MuJoCo sim-to-policy](mujoco_sim_to_policy/) | Simulation data generation, policy training, and rendered evaluation | You need a multi-step robotics ML workflow | +| [Nuke render](nuke_render/) | Frame-parallel headless compositing with `nuke -x` | You need to render Nuke scripts on workers | +| [Pip package job](pip_package_job/) | Declaring Python dependencies for a pip queue environment | A shared queue environment should provide job packages | +| [Pip self-contained job](pip_self_contained_job/) | Creating and activating a pip environment inside one bundle | You cannot or do not want to configure the queue | +| [POV-Ray 3.7](povray-3.7/) | Raytracing with a Conda-provided command-line renderer | You want a portable, lightweight render example | +| [Redshift 2025](redshift-2025/) | Rendering Cinema 4D Redshift scenes with `redshiftCmdLine` | You need direct Windows Redshift command-line rendering | +| [Satellite classification](satellite_classification/) | Per-tile image classification followed by mosaic assembly | Independent input files should fan out and merge | +| [Simple job](simple_job/) | The smallest developer-guide OpenJD job bundle | You are submitting your first custom job | +| [SSH to SMF](ssh_to_smf/) | Temporary Linux SSH access through an SSM hybrid managed node | You need interactive debugging on a service-managed worker | +| [SSH to SMF on Windows](ssh_to_smf_windows/) | Temporary RDP, SSH, or PowerShell access through SSM | You need interactive debugging on a Windows worker | +| [Task chunking](task_chunking/) | A collection of contiguous and non-contiguous chunking patterns | Per-task startup overhead should be shared across frames | +| [Maya tile render blog sample](tile_render_maya_ffmpeg_for_blogpost/) | The adaptor customization and tiled-render workflow from the AWS blog | You are following the tile-rendering walkthrough | +| [Maya Arnold tiled render](tile_render_with_maya_arnold/) | A three-dimensional task space and FFmpeg tile assembly | Arnold images should render as distributed tiles | +| [Maya V-Ray tiled render](tile_render_with_maya_vray/) | V-Ray tile rendering followed by OpenImageIO assembly | You need tiled EXR output from Maya and V-Ray | +| [V-Ray Linux region render](tile_render_with_vray_linux/) | Region rendering, asset discovery, path mapping, and image merge | You render `.vrscene` files in parallel regions on Linux | +| [Maya Arnold turntable](turntable_with_maya_arnold/) | Building a scene around an OBJ, rendering frames, and encoding video | You need an easy-to-submit 3D asset review utility | +| [AutoDock Vina virtual screening](virtual_screening_vina/) | Parallel molecular docking and ranked result aggregation | You want to screen many ligands against a protein target | +| [vLLM evaluation leaderboard](vllm_lm_eval_leaderboard/) | Parallel model evaluation and final CSV/Markdown aggregation | You need to compare multiple LLMs across benchmarks | +| [V-Ray standalone render](vray_render/) | Rendering with a Conda-provided V-Ray executable | You need a basic standalone V-Ray bundle | +| [VRED render](vred_render/) | Headless VRED rendering, tiling, and Python API control | You render VRED scenes with VRED Core or Pro | +| [VTK visualization](vtk-latest/) | Running a VTK Python visualization script | You need a portable scientific visualization job | + +### Developer guide companion samples + +Several compact bundles are intended to be read alongside reference documentation. The [simple job](simple_job/template.yaml) +accompanies the developer guide's [developer farm walkthrough](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/getting-started-dev.html). +The [environment variables](job_env_vars/template.yaml), [environment-provided command](job_env_with_new_command/template.yaml), and +[daemon-process environment](job_env_daemon_process/template.yaml) bundles accompany [Control the job environment](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/control-the-job-environment.html) +and demonstrate [OpenJD environments](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#4-environment); +see the [queue environment samples](../queue_environments/) for session-wide alternatives. + +The [job attachments input](job_attachments_devguide/) and [output](job_attachments_devguide_output/) bundles accompany the developer guide's +[job attachments walkthrough](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-attachments.html). They show how OpenJD +`PATH` parameter data-flow metadata and `asset_references.yaml` jointly describe inputs and outputs, allowing the same bundle patterns to work with +job attachments or shared filesystems. The [GUI control showcase](gui_control_showcase/template.yaml) is the compact reference for every control +supported by [OpenJD job-parameter UI metadata](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#2-jobparameterdefinition). ## Example Blender job submission @@ -148,9 +98,7 @@ or use the `deadline.client.api.create_job_from_job_bundle` function in the [`de If you do not want to use the `deadline` Python package's support for features like job attachments, you can also submit the job template by calling the [deadline:CreateJob API](https://docs.aws.amazon.com/deadline-cloud/latest/APIReference/API_CreateJob.html) directly. +## Example Husk USD render with asset introspection -## Example Husk USD Render with Asset Introspection - -The [houdini_husk_usd_render](houdini_husk_usd_render) sample shows how to use the Houdini Husk CLI USD renderer using a short job template and service-provided Conda packages. -Additionally, this sample shows how to write a custom asset introspection tool for job attachments, ensuring that only the required data is uploaded when using job attachments -while removing manual steps of the artists having to attach the required files. +The [Houdini Husk USD render](houdini_husk_usd_render/) sample shows how to use the Houdini Husk CLI USD renderer using a short job template and service-provided Conda packages. +It also shows how to write a custom asset introspection tool for job attachments, ensuring that only the required data is uploaded while removing manual steps for artists. diff --git a/job_bundles/copy_s3_prefix_to_job_attachments/README.md b/job_bundles/copy_s3_prefix_to_job_attachments/README.md index 4dc38e15..51aa0f62 100644 --- a/job_bundles/copy_s3_prefix_to_job_attachments/README.md +++ b/job_bundles/copy_s3_prefix_to_job_attachments/README.md @@ -9,9 +9,9 @@ volume of data, or are submitting a job that depends on a lot of new data like a can be a longer wait for the job attachments upload. Because job attachments never re-uploads files that are already in job attachments, you can use alternative -upload tools like [AWS Snowball](https://aws.amazon.com/snowball/), [AWS DataSync](https://aws.amazon.com/datasync/), -or [Nimble Studio File Transfer](https://docs.aws.amazon.com/nimble-studio/latest/filetransfer-guide/what-is-file-transfer.html) -to first copy that data to S3, then use this job to copy it into the job attachments for your queue. +upload tools like [AWS Snowball](https://aws.amazon.com/snowball/) or +[AWS DataSync](https://aws.amazon.com/datasync/) to first copy that data to S3, then use this job to +copy it into the job attachments for your queue. ## How to submit this job diff --git a/job_bundles/flux2_klein_lora/README.md b/job_bundles/flux2_klein_lora/README.md index dff7bcee..a7deba98 100644 --- a/job_bundles/flux2_klein_lora/README.md +++ b/job_bundles/flux2_klein_lora/README.md @@ -17,7 +17,16 @@ These AWS Deadline Cloud job bundles use [diffusers](https://github.com/huggingf - AWS Deadline Cloud farm with a GPU-enabled queue (Linux fleet with NVIDIA GPU) - [Deadline Cloud CLI](https://github.com/aws-deadline/deadline-cloud) installed -## Job bundles +## Job bundle index + +This table covers every immediate job bundle in `flux2_klein_lora/`. + +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [LoRA training](lora_training/) | Fine-tuning FLUX.2 Klein from an image-and-caption dataset | You need to create a reusable adapter for a subject or style | +| [Image generation](image_generation/) | Parallel inference with a trained LoRA adapter | You have LoRA weights and want to generate a set of images | + +## Bundle details ### 1. lora_training diff --git a/job_bundles/job_dev_progression/README.md b/job_bundles/job_dev_progression/README.md index f4a3e4b7..da65a027 100644 --- a/job_bundles/job_dev_progression/README.md +++ b/job_bundles/job_dev_progression/README.md @@ -1,31 +1,33 @@ # Job Development Progression -When you're developing a job bundle to run on AWS Deadline Cloud, you will -likely start with something simple. As you add more options and split the workload -into smaller pieces that run in parallel, the complexity of your job -will grow. +When you're developing a job bundle to run on AWS Deadline Cloud, you will likely start with something simple. As you add more options and split the workload into smaller pieces that run in parallel, the complexity of your job will grow. -This directory documents four stages you can take your job bundle through as -you develop it. It starts with a single self-contained job template, and -ends at a Python package bundled with all the trappings like script entrypoints -and unit tests. +This directory documents four stages you can take your job bundle through as you develop it. It starts with a single self-contained job template and ends at a Python package bundled with script entry points and unit tests. -While this example is built around Python, the ideas are not Python-specific. -Feel free to adapt them to your language toolchain of choice. +While this example is built around Python, the ideas are not Python-specific. Adapt them to your language toolchain of choice. + +## Stage index + +This table covers every immediate sample directory in `job_dev_progression/`. + +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [Stage 1: self-contained template](stage_1_self_contained_template/) | Keeping parameters and Python commands in one OpenJD template | You are prototyping a small job with minimal files | +| [Stage 2: bundled scripts](stage_2_bundled_scripts/) | Moving executable logic into scripts carried with the bundle | Inline commands are becoming hard to read or reuse | +| [Stage 3: shared script library](stage_3_bundled_scripts_shared_lib/) | Sharing common code across multiple bundled entry points | Several steps need the same helper logic | +| [Stage 4: bundled Python package](stage_4_bundled_python_package/) | Packaging modules, entry points, and unit tests together | The workload needs maintainable, testable application structure | ## Running jobs on Deadline Cloud To run these jobs on Deadline Cloud, you need a farm in your AWS account. The [quickstart in the Deadline Cloud console](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/getting-started.html) -or the [starter_farm sample CloudFormation template](https://github.com/aws-deadline/deadline-cloud-samples/tree/mainline/cloudformation/farm_templates/starter_farm#readme) -are two ways to deploy one. In both cases, the farm will include a queue environment that can -provide a conda virtual environment for the jobs. +or the [starter farm CloudFormation sample](../../cloudformation/farm_templates/starter_farm/) are two ways to deploy one. +In both cases, the farm includes a queue environment that can provide a Conda virtual environment for the jobs. -With the Deadline Cloud CLI installed locally, e.g. from `pip install deadline`, -the following command will submit the first stage to your farm: +With the Deadline Cloud CLI installed locally, for example with `pip install deadline`, the following command submits the first stage to your farm: -```bash -$ deadline bundle submit stage_1_self_contained_template +```console +deadline bundle submit stage_1_self_contained_template ``` You can view your job and its log output from [Deadline Cloud monitor](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/working-with-deadline-monitor.html). @@ -33,36 +35,24 @@ You can view your job and its log output from [Deadline Cloud monitor](https://d ## Running jobs locally You can run jobs locally for development or as a way to use one code base locally and on your farm. -Use the [Open Job Description CLI](https://github.com/OpenJobDescription/openjd-cli#readme), -available to install from `pip install openjd-cli`. +Use the [Open Job Description CLI](https://github.com/OpenJobDescription/openjd-cli#readme), available from `pip install openjd-cli`. -In each job template, you'll find two parameters defined that specify the software -environment it expects to run in. You can either provide the necessary applications by -installing them yourself, or you can use an environment template to provide the conda packages. -See the [sample environment templates](https://github.com/aws-deadline/deadline-cloud-samples/tree/mainline/queue_environments#readme), -and note that you will need conda installed in order for them to work. +Each job template defines two parameters that specify the software environment it expects. You can install the applications yourself or use an environment template to provide the Conda packages. See the [sample queue environments](../../queue_environments/) and note that you need Conda installed for the inline Conda environments. -If you have set up all the required software in the `PATH` environment variable, you can run -the job directly. If `polars` is not installed, the job will fail with an error in the log -like `ModuleNotFoundError: No module named 'polars'`. +If the required software is already in `PATH`, run the job directly. If `polars` is unavailable, the log reports an error such as `ModuleNotFoundError: No module named 'polars'`. -```bash -$ openjd run stage_1_self_contained_template/template.yaml +```console +openjd run stage_1_self_contained_template/template.yaml ``` -If you run the jobs with the [conda_queue_env_console_equivalent](https://github.com/aws-deadline/deadline-cloud-samples/blob/mainline/queue_environments/conda_queue_env_console_equivalent.yaml) -sample, it will create a conda virtual environment within the job's session directory. -This creates a fresh conda environment every time you run the job. +The [console-equivalent Conda queue environment](../../queue_environments/conda_queue_env_from_console.yaml) creates a fresh virtual environment in the job session directory: -```bash -$ openjd run --environment ../../queue_environments/conda_queue_env_console_equivalent.yaml stage_1_self_contained_template/template.yaml +```console +openjd run --environment ../../queue_environments/conda_queue_env_from_console.yaml stage_1_self_contained_template/template.yaml ``` -If you run the jobs with the [conda_queue_env_improved_caching](https://github.com/aws-deadline/deadline-cloud-samples/blob/mainline/queue_environments/conda_queue_env_improved_caching.yaml) -sample, it will take the hash of requested channels and packages, and use a named channel based on that hash. When -the named channel already exists, it will reuse it directly. After a configurable delay, it will refresh the packages, -and after a longer delay, it will remove the environment. +The [improved-caching Conda queue environment](../../queue_environments/conda_queue_env_improved_caching.yaml) names environments from a hash of channels and packages, reuses them, refreshes them after a configurable delay, and eventually removes stale environments: -```bash -$ openjd run --environment ../../queue_environments/conda_queue_env_improved_caching.yaml stage_1_self_contained_template/template.yaml +```console +openjd run --environment ../../queue_environments/conda_queue_env_improved_caching.yaml stage_1_self_contained_template/template.yaml ``` diff --git a/job_bundles/ssh_to_smf_windows/README.md b/job_bundles/ssh_to_smf_windows/README.md index 59de2e52..e9c0c97e 100644 --- a/job_bundles/ssh_to_smf_windows/README.md +++ b/job_bundles/ssh_to_smf_windows/README.md @@ -285,4 +285,4 @@ Push a new host-config via `aws deadline update-fleet --host-configuration file: ## See Also - [`ssh_to_smf`](../ssh_to_smf/README.md) — the Linux version this bundle is cloned from -- [`.kiro/specs/ssh-to-smf-windows/design.md`](../../../.kiro/specs/ssh-to-smf-windows/design.md) — full design doc +- [Linux sibling design reference](../ssh_to_smf/DESIGN.md) — background for the shared SSM activation pattern diff --git a/job_bundles/task_chunking/README.md b/job_bundles/task_chunking/README.md index 8cfa92cc..57e1857b 100644 --- a/job_bundles/task_chunking/README.md +++ b/job_bundles/task_chunking/README.md @@ -2,34 +2,26 @@ These samples demonstrate the [Task Chunking](https://github.com/OpenJobDescription/openjd-specifications/blob/mainline/rfcs/0001-task-chunking.md) extension for Open Job Description, which improves resource utilization by grouping multiple frames or tasks into chunks instead of processing them individually. -## Why Use Task Chunking? +## Why use task chunking? Render jobs often spend significant time loading applications and scene files before rendering each frame. Chunking amortizes this overhead by processing multiple frames or tasks per chunk, reducing total job runtime. -## Samples +## Sample index -### 1. Basic Contiguous Chunks (`basic_contiguous_chunks/`) +This table covers every immediate sample directory in `task_chunking/`. -A minimal example using `rangeConstraint: CONTIGUOUS`. Each chunk expands to a range like `"1-10"` or `"11-20"`. The script parses and prints the start and end frame numbers. +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [Basic contiguous chunks](basic_contiguous_chunks/) | `CHUNK[INT]` with `rangeConstraint: CONTIGUOUS` and start/end parsing | Your command accepts consecutive frame ranges | +| [Basic non-contiguous chunks](basic_non_contiguous_chunks/) | `CHUNK[INT]` with arbitrary sparse frame sets | Your command accepts lists such as `1-3,5,7-20:2` | +| [Blender contiguous chunks](blender_render_with_contiguous_chunks/) | Applying contiguous task chunks to a frame render | Blender should load once for several consecutive frames | +| [Blender non-contiguous chunks](blender_render_with_non_contiguous_chunks/) | Applying scheduler-selected non-contiguous chunks to a frame render | Blender can render arbitrary frame lists per task | -### 2. Basic Non-Contiguous Chunks (`basic_non_contiguous_chunks/`) +The Blender variants add the `TASK_CHUNKING` extension and a `ChunkSize` parameter to the base +[Blender render](../blender_render/) sample. They change the frame task parameter from `INT` to +`CHUNK[INT]` and set a target runtime so the scheduler can adjust chunk size. -A minimal example using `rangeConstraint: NONCONTIGUOUS`. Chunks can be arbitrary frame sets like `"1-3,5,7-20:2"`. The script prints the frames assigned by the scheduler. - -### 3. Blender Render with Contiguous Chunks (`blender_render_with_contiguous_chunks/`) - -A real-world example converted from the existing [blender_render](../blender_render/template.yaml) job bundle to render job with contiguous chunks. - -### 4. Blender Render with Non-Contiguous Chunks (`blender_render_with_non_contiguous_chunks/`) - -A real-world example converted from the existing [blender_render](../blender_render/template.yaml) job bundle to render job with non-contiguous chunks. - -Changes from the original: -1. Added `extensions: [TASK_CHUNKING]` -2. Added `ChunkSize` parameter (default: 5) -3. Changed `Frame` from `type: INT` to `type: CHUNK[INT]` with `rangeConstraint: CONTIGUOUS` and `targetRuntimeSeconds: 600` - -## Template Structure +## Template structure ```yaml specificationVersion: 'jobtemplate-2023-09' @@ -45,18 +37,18 @@ steps: range: "{{Param.Frames}}" chunks: defaultTaskCount: 10 # Default frames per chunk - targetRuntimeSeconds: 600 # Optional: allows the scheduler to adjust the task count for chunks to match this runtime + targetRuntimeSeconds: 600 # Optional target used to adjust chunk size rangeConstraint: CONTIGUOUS # or NONCONTIGUOUS ``` -## Range Constraints +## Range constraints | Constraint | `{{Task.Param.Frame}}` expands to | Use when | -|------------|-----------------------------------|----------| -| `CONTIGUOUS` | `"1-10"`, `"11-20"` | App supports start/end frame arguments | -| `NONCONTIGUOUS` | `"1-3,5,7-10"` | App supports arbitrary frame lists | +|---|---|---| +| `CONTIGUOUS` | `"1-10"`, `"11-20"` | The application supports start/end frame arguments | +| `NONCONTIGUOUS` | `"1-3,5,7-10"` | The application supports arbitrary frame lists | ## References -- [RFC 0001: Task Chunking](https://github.com/OpenJobDescription/openjd-specifications/blob/mainline/rfcs/0001-task-chunking.md) -- [Open Job Description Specification](https://github.com/OpenJobDescription/openjd-specifications) +* [RFC 0001: Task Chunking](https://github.com/OpenJobDescription/openjd-specifications/blob/mainline/rfcs/0001-task-chunking.md) +* [Open Job Description specification](https://github.com/OpenJobDescription/openjd-specifications) diff --git a/job_bundles/tile_render_maya_ffmpeg_for_blogpost/README.md b/job_bundles/tile_render_maya_ffmpeg_for_blogpost/README.md index ae5ae84b..8929111c 100644 --- a/job_bundles/tile_render_maya_ffmpeg_for_blogpost/README.md +++ b/job_bundles/tile_render_maya_ffmpeg_for_blogpost/README.md @@ -15,7 +15,7 @@ to define task parameter values (TileNumberX and TileNumberY) for the tile numbe to the runData for the Maya render step. A second step is defined to have a dependency on the render step, which uses a bash script to call ffmpeg to assemble the tiles. -See also the job bundle [tiled_region_render_with_maya_arnold](tiled_region_render_with_maya_arnold) +See also the job bundle [tile_render_with_maya_arnold](../tile_render_with_maya_arnold/) that forms the tile bounds in the job and uses region render parameters when calling the Open Job Description application interface for rendering Maya. diff --git a/job_bundles/tile_render_with_maya_vray/README.md b/job_bundles/tile_render_with_maya_vray/README.md index c01d52a0..5f12fa41 100644 --- a/job_bundles/tile_render_with_maya_vray/README.md +++ b/job_bundles/tile_render_with_maya_vray/README.md @@ -4,4 +4,4 @@ This job bundle will submit a tile rendering job using Maya and V-Ray to create This job bundle relies on the V-Ray render handler in the Maya adaptor. The template defines a number of X and Y tiles which is used to split the output into evenly sized tiles that can be distributed across multiple render nodes. -See also the job bundle [tiled_region_render_with_maya_arnold](https://github.com/aws-deadline/deadline-cloud-samples/tree/mainline/job_bundles/tile_render_with_maya_arnold) for an example using Maya and Arnold and using FFMPG to assemble PNG files into the final image. +See also the job bundle [tile_render_with_maya_arnold](../tile_render_with_maya_arnold/) for an example using Maya and Arnold and using FFmpeg to assemble PNG files into the final image. diff --git a/queue_environments/README.md b/queue_environments/README.md index d4234ce7..5e2fb9e6 100644 --- a/queue_environments/README.md +++ b/queue_environments/README.md @@ -1,246 +1,139 @@ # AWS Deadline Cloud queue environments -## Introduction +Queue environments follow the [Open Job Description environment template specification](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas) and prepare software or licensing once per worker session. Jobs select packages through parameters such as `CondaPackages`, `RezPackages`, or `PipPackages`. -This directory holds sample queue environments you can use with Deadline Cloud. Queue environments -follow the environment template specification from -[Open Job Description](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas). +## Sample index -The Conda and Rez queue environments let you provide software applications to jobs in your -Deadline Cloud queue, so each job only needs a parameter value for `CondaPackages` or `RezPackages` -to tell it the list of packages to use. The pip queue environment does the same for Python -packages, so a job only needs to provide a value for `PipPackages`. +This table covers every queue environment YAML file in `queue_environments/`. + +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [Console Conda environment](conda_queue_env_from_console.yaml) | Service-managed fleet `conda-queue-env` commands backed by Rattler | You want the environment created by console onboarding | +| [Inline Conda environment](conda_queue_env_inline.yaml) | Creating and deleting a Conda environment with portable bash | A customer-managed fleet has Conda but not service-provided helper commands | +| [Py-rattler Conda environment](conda_queue_env_pyrattler.yaml) | Solving and activating packages with the `py-rattler` library | You want faster solving and can accept its compatibility differences | +| [Cached Conda environment](conda_queue_env_improved_caching.yaml) | Reusing hash-named environments with service-managed fleet commands | Repeated package sets should avoid relinking on every job | +| [Cached inline Conda environment](conda_queue_env_inline_improved_caching.yaml) | Portable named-environment reuse and expiration logic | Customer-managed fleets need reusable Conda environments | +| [Rez environment](rez_queue_env.yaml) | Resolving packages from a shared Rez repository | Your studio already distributes software with Rez | +| [Pip environment](pip_queue_env.yaml) | Creating a Python `venv` and installing job-selected pip packages | Jobs need Python packages without Conda or Rez | +| [Disconnect UBL](disconnect_ubl_queue_env.yaml) | Removing Deadline Cloud Usage Based License environment variables | A queue must use only a custom license server | ## Create a queue environment for your queue -Here are steps to set up one of the sample queue environments. - -1. For the queue environment sample you wish to use, modify the default value for the parameter `CondaChannels` - or `RezRepositories` to be the source of your packages. Both Conda and Rez support shared file system - paths for this, while Conda also supports channels hosted on [Anaconda.org](https://anaconda.org), - web servers, and S3 buckets. -2. In Deadline Cloud, create a queue environment for your queue using the template you have modified. Read the topic - [create a queue environment](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/create-queue-environment.html) - in the user guide to learn how to create or update them in your queue. You can also create a queue environment with a - CLI command similar to: - ``` - $ aws deadline create-queue-environment \ - --farm-id FARM_ID \ - --queue-id QUEUE_ID \ - --priority 1 \ - --template-type YAML \ - --template file://conda_queue_env_improved_caching.yaml - ``` - -## Install git bash on Windows worker hosts - -The sample queue environments are written using bash script code that is portable to Windows. -You can use them with Windows customer-managed fleets by installing [Git for Windows](https://gitforwindows.org/) -on the worker hosts. Make sure that the git binary is in the PATH. +1. In the selected sample, change `CondaChannels`, `RezRepositories`, or package-index defaults to your package source. Conda and Rez support shared file-system paths; Conda also supports Anaconda.org, web, and S3 channels. +2. Follow [Create a queue environment](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/create-queue-environment.html) to add or update it. A CLI invocation looks like: + + ```console + aws deadline create-queue-environment \ + --farm-id FARM_ID \ + --queue-id QUEUE_ID \ + --priority 1 \ + --template-type YAML \ + --template file://conda_queue_env_improved_caching.yaml + ``` + +## Install Git Bash on Windows worker hosts + +These samples use bash that is portable to Windows. On Windows customer-managed fleets, install [Git for Windows](https://gitforwindows.org/) and put its Git binary on `PATH`. ## Install Conda and Rez on worker hosts -To use the queue environment samples from customer-managed fleets, you need to -provide [Conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) -or [Rez](https://rez.readthedocs.io/en/stable/installation.html) on worker hosts, -for example by installing them on your Amazon machine image (AMI). +Customer-managed fleets must provide [Conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) or [Rez](https://rez.readthedocs.io/en/stable/installation.html), for example in the AMI. -For Conda you must also apply the following setup steps so that `conda activate` and `conda deactivate` -are available within non-interactive bash shells. The scripts assume an `/opt/conda` install location -on Linux and `C:\Programs\Conda` install location on Windows. +For Conda, also make `conda activate` and `conda deactivate` available in non-interactive bash. The samples assume `/opt/conda` on Linux and `C:\Programs\Conda` on Windows. -Here is an example bash script that does this for Amazon Linux 2023: +Amazon Linux 2023: ```bash -# Turn on pam_env so that `/etc/environment` is used in non-interactive scripts +# Use /etc/environment in non-interactive scripts. echo 'auth required pam_env.so' >> /etc/pam.d/su -# Enable `conda activate ` in non-interactive scripts, echo 'BASH_ENV=/etc/bash_env' >> /etc/environment echo 'source /opt/conda/etc/profile.d/conda.sh' > /etc/bash_env ``` -Here is an example bash script that does this for Ubuntu: +Ubuntu: ```bash -# Enable `conda activate ` in non-interactive scripts, echo 'source /opt/conda/etc/profile.d/conda.sh' >> /usr/share/modules/init/bash ``` -Here is an example PowerShell script that does this for Windows: +Windows PowerShell: -```bash -# Set BASH_ENV so that it sources the conda command +```powershell [Environment]::SetEnvironmentVariable("BASH_ENV", "/etc/bash_env", "Machine") $Env:BASH_ENV = [Environment]::GetEnvironmentVariable("BASH_ENV", "Machine") echo @' echo 'source "/c/Programs/Conda/etc/profile.d/conda.sh"' > /etc/bash_env '@ | & "C:\Programs\Git\bin\bash" - ``` -## Submit Jobs - -One of the cool parts of queue environments and the Deadline Cloud submitters, such as -[deadline-cloud-for-blender](https://github.com/aws-deadline/deadline-cloud-for-blender), is that the submitters -will automatically add parameters in the queue environment for selecting the right Conda or Rez packages. +## Submit jobs -When you write your own job bundles, you can get the same result by including parameter definitions -for `CondaPackages` and/or `RezPackages` with a default value of the needed packages. -The [blender_render](https://github.com/aws-deadline/deadline-cloud-samples/blob/mainline/job_bundles/blender_render/template.yaml) -job bundle sample illustrates this by providing them both with a default value `blender`. +Deadline Cloud submitters can add the package-selection parameters defined by a queue environment automatically. Custom bundles can get the same behavior by defining `CondaPackages`, `RezPackages`, or `PipPackages` with suitable defaults. The [Blender render template](../job_bundles/blender_render/template.yaml) demonstrates both Conda and Rez package parameters. -When your jobs run, these parameter values get passed to the queue environment, which creates -and activates a Conda or Rez virtual environment containing the requested packages and their dependencies. -The script commands for the job should run application binaries like `blender` without specifying -absolute paths. Entering the environment updates the `PATH` environment variable to provide -the correct binaries. +The queue environment creates and activates the selected virtual environment, so task commands should invoke applications such as `blender` from `PATH` instead of using absolute paths. -## The sample queue environments +## Environment behavior details -### Console Conda queue environment +### Console Conda environment -The file [conda_queue_env_from_console.yaml](conda_queue_env_from_console.yaml) is a copy of the queue environment -added by Deadline Cloud console onboarding flows. Its onEnter and onExit actions run the commands -`conda-queue-env-enter` and `conda-queue-env-exit` respectively. These commands are provided on -the workers of Deadline Cloud service-managed fleets. They are written using [Rattler](https://github.com/conda/rattler), which -generally runs faster than equivalent operations with Conda. +The console environment runs `conda-queue-env-enter` and `conda-queue-env-exit`, which are available on service-managed fleet workers and implemented with [Rattler](https://github.com/conda/rattler). Their relevant options are: -Here are the CLI options the enter and exit commands provide: -``` +```text Usage: conda-queue-env-enter [OPTIONS] [ENV_DIR] Arguments: [ENV_DIR] The location of the environment to be created Options: - -p, --packages - Space-separated list of Conda packages to install - -c, --channels - Space-separated list of Conda channels - --channel-priority - Channel priority: "strict" or "disabled" [default: strict] - --persist-envs-hashed - Persist environments in hash-named subdirectories under this root dir. Enables environment reuse across jobs - --update-after-minutes - Minutes before updating a persisted environment (default: 600 = 10 hours) [default: 600] - -v, --verbose... - Increase logging verbosity (-v for debug, -vv for trace) - --windows-activation-shell - Shell to use for conda activation on Windows: "bash" (default) or "cmd" [default: bash] - --print-env0 - Print all environment variables as null-delimited KEY=VALUE pairs and exit. Used internally by bash activation to capture native OS paths + -p, --packages Space-separated packages + -c, --channels Space-separated channels + --channel-priority "strict" or "disabled" + --persist-envs-hashed Reuse hash-named environments + --update-after-minutes Refresh age; default 600 + -v, --verbose... Increase logging verbosity + --windows-activation-shell "bash" or "cmd" + --print-env0 Print null-delimited environment values -h, --help - Print help -``` ``` + +```text Usage: conda-queue-env-exit [OPTIONS] Options: - --persist-envs-hashed - Root directory containing hash-named persisted environments - --cleanup-after-hours - Remove persisted environments not updated within this many hours (default: 96) [default: 96] - -v, --verbose... - Increase logging verbosity (-v for debug, -vv for trace) + --persist-envs-hashed Root containing persistent environments + --cleanup-after-hours Stale cleanup age; default 96 + -v, --verbose... Increase logging verbosity -h, --help - Print help ``` -The `conda-queue-env` commands on service-managed fleets support creating persistent environments that can be reused across -multiple jobs, but this functionality is not enabled by default on the console queue environment. See the `conda_queue_env_improved_caching.yaml` -queue environment for a sample that enables this functionality. +Persistent reuse is not enabled in the console template by default. The cached Conda sample enables it. -To get similar functionality as the `conda_queue_env_from_console.yaml` environment on customer-managed fleets, -see the next sample `conda_queue_env_inline.yaml`. +### Inline Conda environment -### Conda queue environment using Conda written inline +The inline sample directly runs Conda and works on customer-managed fleets. It creates one environment per OpenJD session and deletes it afterward. Conda still caches downloaded and expanded packages, but each session pays the cost of linking a new environment. Unlike the console environment's strict channel priority, it uses Conda's flexible priority for multiple channels. -The file [conda_queue_env_inline.yaml](conda_queue_env_inline.yaml) has nearly the same behavior -as the console Conda queue environment, but it does not use Rattler and directly runs Conda to create the virtual environment. -There is a small difference in functionality when using multiple conda channels; the console queue environment uses `strict` channel priority, -whereas this queue environment, as well as other queue environments not using Rattler, use `flexible` channel priority. +### Py-rattler Conda environment -The behavior of this queue environment is to create a new Conda virtual environment for every Open Job -Description session that runs on a worker host, and then delete the environment when it is done. -Conda keeps a cache of the downloaded packages, and the expanded form of those packages, so it will not -repeatedly re-download the same applications, but each session will have the overhead of linking all -packages into the virtual environment. Look at the samples `conda_queue_env_improved_caching.yaml` and -`conda_queue_env_inline_improved_caching.yaml` for queue environments that can reuse virtual -environments across multiple jobs. +The py-rattler sample provides similar behavior through [py-rattler](https://conda.github.io/rattler/py-rattler/). It generally solves faster, but `pip` is not automatically included with `python`, it rejects some syntax accepted by Conda (for example `colmap=*=gpu*`), and solver errors can include less diagnostic detail. -### Conda queue environment using the py-rattler library +### Conda queue environment with improved caching -The file [conda_queue_env_pyrattler.yaml](conda_queue_env_pyrattler.yaml) provides the same functionality as -the above Conda queue environments, but uses the [py-rattler library](https://conda.github.io/rattler/py-rattler/). -Rattler is a library that provides common functionality used within the conda ecosystem. It's written -in Rust and tries to provide a clean API to its functionalities. The environments it creates are almost the same, -but we found that py-rattler does not include 'pip' alongside 'python' by default, so if you need pip you must -add it explicitly. It also raises an error for a subset of syntax that conda accepts, such as 'colmap=*=gpu*'. +The service-managed cached sample stores reusable environments under `~/.persistent_envs` by default. Change both enter and exit actions if you choose another path. -Testing has shown that this queue environment generally runs faster than the above when on the same instance types, -but the error messages it produces when failing to solve for a virtual environment do not include as -much detail to help diagnose what happened. +### Conda queue environment with improved caching using Conda written inline -### Rez queue environment +The cached inline sample implements the same idea with named Conda environments on customer-managed fleets. Its default name hashes channels and packages; jobs can also specify a name. Separate settings control how long an environment is reused before package refresh and when stale environments are deleted. -The file [rez_queue_env.yaml](rez_queue_env.yaml) provides the same functionality as -the above Conda queue environments, but for the Rez package manager. The queue environment will work in a -farm using customer-managed fleets that have a shared file system for the Rez package repository. +### Rez environment -### Conda queue environment with improved caching +The Rez sample resolves software from a shared package repository. Use it with customer-managed fleets that can access that repository. -The file [conda_queue_env_improved_caching.yaml](conda_queue_env_improved_caching.yaml) enables -the same virtual environments to be reused across multiple jobs via additional command line arguments to the `conda-queue-env-enter`, -and `conda-queue-env-exit` commands provided on service-managed fleets. This can give significant performance improvements when -running many jobs with the same package requirements. +### Pip environment -The queue environment is configured to store persistent environments under `~/.persistent_envs`. To store persistent environments -under a different directory, the `onEnter` and `onExit` actions can be modified to reference a different path. +The pip sample uses Python's standard-library `venv` module, installs `PipPackages`, and activates the environment for subsequent steps. If `PipPackages` is empty it does nothing, allowing mixed queues. `PipIndexUrl` and `PipExtraIndexUrls` support private indexes such as [AWS CodeArtifact](https://docs.aws.amazon.com/codeartifact/). -To get environment reuse functionality on customer-managed fleets, you can use the following sample. +Workers need `python3` or `python` on `PATH`; service-managed fleets provide one. Compare the [pip package job](../job_bundles/pip_package_job/) with the [self-contained pip job](../job_bundles/pip_self_contained_job/) when deciding whether configuration belongs on the queue or in one bundle. -### Conda queue environment with improved caching using Conda written inline +### Disconnect UBL -The file [conda_queue_env_inline_improved_caching.yaml](conda_queue_env_inline_improved_caching.yaml) extends the -capabilities of the Conda queue environment with a mechanism to reuse Conda virtual environments -across multiple jobs. This additional cache management is more complex, but the performance benefits -from environment reuse can be significant when running many jobs with the same package requirements. - -The core enhancement of this queue environment is to use named Conda environments that can be shared across -jobs. The default environment name uses the hash of the Conda channels and packages, or you can explicitly -set the name in the job. It also includes a parameter for how long to use an environment without running a package -update, so that most of the time it will take seconds to activate an environment that's being reused. - -### Pip queue environment - -The file [pip_queue_env.yaml](pip_queue_env.yaml) lets you provide Python packages to jobs using -[pip](https://pip.pypa.io/) and the standard library [venv](https://docs.python.org/3/library/venv.html) -module, rather than a package manager like Conda or Rez. When a job provides a `PipPackages` parameter -value, the queue environment creates a Python virtual environment in the session working directory, -installs the requested packages into it with pip, and activates it so subsequent steps run with those -packages available. If `PipPackages` is empty, the queue environment does nothing, so it is safe to add -to a queue that also runs jobs which do not use it. - -The `PipIndexUrl` and `PipExtraIndexUrls` parameters let jobs install from a private package index, such -as an [AWS CodeArtifact](https://docs.aws.amazon.com/codeartifact/) repository, instead of the default -[PyPI](https://pypi.org/) index. - -Unlike Conda and Rez, pip and venv are included with Python itself, so worker hosts only need a `python3` -(or `python`) interpreter on the `PATH`. Deadline Cloud service-managed fleets provide one. The -[pip_package_job](../job_bundles/pip_package_job) job bundle shows how to submit a job that uses this -queue environment, and [pip_self_contained_job](../job_bundles/pip_self_contained_job) shows the same -pip environment defined inline in a job bundle when you do not want to configure a queue environment. - -### Disconnect UBL queue environment - -The file [disconnect_ubl_queue_env.yaml](disconnect_ubl_queue_env.yaml) unsets Deadline Cloud Usage Based -License (UBL) environment variables. Use this queue environment if you want to turn off all connections to -Deadline Cloud UBL for your queue and force the use of a custom license server (see the -[Bring Your Own License documentation](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/smf-byol.html)). - -This queue environment should be run before any other queue environments (for example, by setting the priority to 0) -so that connections to your custom floating licenses (such as RLM) in other queue environments are not -accidentally removed. - -Please note that this is a sample, additional UBL environment variables may be -added in the future. +The disconnect environment unsets Deadline Cloud Usage Based License variables so jobs use a custom license server. Give it a higher-precedence position than other environments, for example priority `0`, so later licensing setup is not removed. Review [Bring Your Own License](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/smf-byol.html). Additional UBL variables can be introduced over time, so review the template against current service behavior before deployment. diff --git a/scripts/check_external_links.py b/scripts/check_external_links.py new file mode 100644 index 00000000..aa4f8ea6 --- /dev/null +++ b/scripts/check_external_links.py @@ -0,0 +1,523 @@ +#!/usr/bin/env python3 +"""Check live external links in every tracked Markdown file. + +The checker intentionally bypasses proxy settings and pins each connection to an IP +address from a validated DNS result. Redirect targets are independently validated. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import http.client +import ipaddress +import re +import socket +import ssl +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from urllib.parse import quote, urljoin, urlsplit, urlunsplit + +import check_markdown_links as markdown + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_IGNORE_FILE = REPOSITORY_ROOT / ".github" / "external-link-ignore.txt" +USER_AGENT = "deadline-cloud-samples-link-checker/1.0 (+https://github.com/aws-deadline/deadline-cloud-samples)" +REDIRECT_STATUSES = frozenset({301, 302, 303, 307, 308}) +RETRYABLE_STATUSES = frozenset({408, 425, 429}) +DATED_COMMENT = re.compile(r"^#\s*(\d{4}-\d{2}-\d{2}):\s*(\S.*)$") +DOMAIN = re.compile(r"^[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?$") + + +class UnsafeTarget(ValueError): + """A URL or DNS result is unsafe for requests from CI.""" + + +@dataclass(frozen=True) +class Settings: + timeout: float = 10.0 + retries: int = 2 + backoff: float = 1.0 + max_redirects: int = 5 + + +@dataclass(frozen=True) +class ParsedTarget: + url: str + scheme: str + hostname: str + port: int + host_header: str + request_target: str + + +@dataclass(frozen=True) +class Response: + status: int + location: str | None + + +@dataclass(frozen=True) +class ChainResult: + success: bool + final_url: str + status: int | None = None + error: str | None = None + hard_failure: bool = False + + @property + def retryable(self) -> bool: + return not self.hard_failure and ( + self.error is not None + or self.status in RETRYABLE_STATUSES + or (self.status is not None and self.status >= 500) + ) + + def describe(self) -> str: + if self.error: + return self.error + return f"HTTP {self.status} at {self.final_url}" + + +@dataclass(frozen=True) +class CheckResult: + url: str + success: bool + detail: str + + +def _require_public_unicast(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> None: + """Reject every address class that CI must not contact, including multicast.""" + if address.is_multicast: + raise UnsafeTarget(f"multicast IP address {address} is not allowed") + if not address.is_global: + raise UnsafeTarget(f"non-public IP address {address} is not allowed") + + +def _ascii_hostname(hostname: str) -> str: + if hostname.endswith("."): + raise UnsafeTarget("hostnames with a trailing dot are not allowed") + try: + address = ipaddress.ip_address(hostname) + except ValueError: + try: + ascii_hostname = hostname.encode("idna").decode("ascii").lower() + except UnicodeError as error: + raise UnsafeTarget(f"invalid internationalized hostname: {error}") from error + if "." not in ascii_hostname: + raise UnsafeTarget("single-label and localhost hostnames are not allowed") + if ascii_hostname == "localhost" or ascii_hostname.endswith(".localhost"): + raise UnsafeTarget("localhost targets are not allowed") + labels = ascii_hostname.split(".") + if any( + not label + or len(label) > 63 + or label.startswith("-") + or label.endswith("-") + or not re.fullmatch(r"[a-z0-9-]+", label) + for label in labels + ): + raise UnsafeTarget("malformed DNS hostname") + if len(ascii_hostname) > 253: + raise UnsafeTarget("DNS hostname is too long") + return ascii_hostname + _require_public_unicast(address) + return address.compressed + + +def parse_target(url: str) -> ParsedTarget: + if any(character.isspace() or ord(character) < 32 or ord(character) == 127 for character in url): + raise UnsafeTarget("URL contains whitespace or control characters") + if "\\" in url: + raise UnsafeTarget("URL contains a backslash") + try: + parsed = urlsplit(url) + port = parsed.port + except ValueError as error: + raise UnsafeTarget(f"malformed URL: {error}") from error + scheme = parsed.scheme.lower() + if scheme not in {"http", "https"}: + raise UnsafeTarget("only http and https URLs are allowed") + if not parsed.netloc or parsed.hostname is None: + raise UnsafeTarget("URL has no hostname") + if parsed.username is not None or parsed.password is not None or "@" in parsed.netloc: + raise UnsafeTarget("URL credentials are not allowed") + expected_port = 80 if scheme == "http" else 443 + if port is not None and port != expected_port: + raise UnsafeTarget(f"non-standard port {port} is not allowed for {scheme}") + + hostname = _ascii_hostname(parsed.hostname) + effective_port = port or expected_port + host_header = f"[{hostname}]" if ":" in hostname else hostname + if port is not None: + host_header = f"{host_header}:{port}" + path = quote(parsed.path or "/", safe="/%:@!$&'()*+,;=-._~") + query = quote(parsed.query, safe="%/?@!$&'()*+,;=:-._~") + request_target = f"{path}?{query}" if query else path + normalized_url = urlunsplit((scheme, parsed.netloc, parsed.path, parsed.query, "")) + return ParsedTarget(normalized_url, scheme, hostname, effective_port, host_header, request_target) + + +def _public_addresses(hostname: str, port: int) -> list[tuple[int, int, int, tuple[object, ...]]]: + try: + records = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except socket.gaierror as error: + raise OSError(f"DNS resolution failed for {hostname}: {error}") from error + if not records: + raise OSError(f"DNS resolution returned no addresses for {hostname}") + + addresses: list[tuple[int, int, int, tuple[object, ...]]] = [] + seen: set[tuple[int, tuple[object, ...]]] = set() + for family, socktype, protocol, _, sockaddr in records: + raw_address = str(sockaddr[0]).split("%", 1)[0] + try: + address = ipaddress.ip_address(raw_address) + except ValueError as error: + raise UnsafeTarget(f"DNS returned malformed address {raw_address!r}") from error + try: + _require_public_unicast(address) + except UnsafeTarget as error: + raise UnsafeTarget( + f"DNS for {hostname} returned unsafe address {address}; refusing all addresses: {error}" + ) from error + key = (family, sockaddr) + if key not in seen: + seen.add(key) + addresses.append((family, socktype, protocol, sockaddr)) + return addresses + + +def _connect( + addresses: list[tuple[int, int, int, tuple[object, ...]]], timeout: float +) -> socket.socket: + last_error: OSError | None = None + for family, socktype, protocol, sockaddr in addresses: + sock = socket.socket(family, socktype, protocol) + sock.settimeout(timeout) + try: + sock.connect(sockaddr) + return sock + except OSError as error: + last_error = error + sock.close() + raise OSError(f"could not connect to any validated address: {last_error}") + + +class DirectHTTPConnection(http.client.HTTPConnection): + def __init__( + self, + host: str, + port: int, + timeout: float, + addresses: list[tuple[int, int, int, tuple[object, ...]]], + ) -> None: + super().__init__(host, port, timeout=timeout) + self._addresses = addresses + + def connect(self) -> None: + self.sock = _connect(self._addresses, self.timeout) + + +class DirectHTTPSConnection(http.client.HTTPSConnection): + def __init__( + self, + host: str, + port: int, + timeout: float, + addresses: list[tuple[int, int, int, tuple[object, ...]]], + ) -> None: + super().__init__(host, port, timeout=timeout, context=ssl.create_default_context()) + self._addresses = addresses + + def connect(self) -> None: + raw_socket = _connect(self._addresses, self.timeout) + try: + # The original hostname is retained for TLS SNI and certificate verification. + self.sock = self._context.wrap_socket(raw_socket, server_hostname=self.host) + except BaseException: + raw_socket.close() + raise + + +def _request_once(url: str, method: str, settings: Settings) -> Response: + target = parse_target(url) + addresses = _public_addresses(target.hostname, target.port) + connection_class = DirectHTTPSConnection if target.scheme == "https" else DirectHTTPConnection + connection = connection_class(target.hostname, target.port, settings.timeout, addresses) + headers = { + "Accept": "*/*", + "Accept-Encoding": "identity", + "Connection": "close", + "Host": target.host_header, + "User-Agent": USER_AGENT, + } + if method == "GET": + headers["Range"] = "bytes=0-0" + try: + connection.request(method, target.request_target, headers=headers) + response = connection.getresponse() + return Response(response.status, response.getheader("Location")) + finally: + # Do not consume response bodies. Closing is sufficient for this one-shot connection. + connection.close() + + +def _request_chain(original_url: str, method: str, settings: Settings) -> ChainResult: + current_url = original_url + visited: set[str] = set() + for redirect_count in range(settings.max_redirects + 1): + try: + current_url = parse_target(current_url).url + if current_url in visited: + return ChainResult(False, current_url, error="redirect loop detected", hard_failure=True) + visited.add(current_url) + response = _request_once(current_url, method, settings) + except UnsafeTarget as error: + return ChainResult(False, current_url, error=f"unsafe target: {error}", hard_failure=True) + except (OSError, ssl.SSLError, http.client.HTTPException) as error: + return ChainResult(False, current_url, error=f"{type(error).__name__}: {error}") + + if 200 <= response.status < 300: + return ChainResult(True, current_url, status=response.status) + if response.status not in REDIRECT_STATUSES: + return ChainResult(False, current_url, status=response.status) + if not response.location: + return ChainResult( + False, + current_url, + status=response.status, + error=f"HTTP {response.status} redirect has no Location header", + hard_failure=True, + ) + if redirect_count == settings.max_redirects: + return ChainResult( + False, + current_url, + status=response.status, + error=f"more than {settings.max_redirects} redirects", + hard_failure=True, + ) + try: + current_url = parse_target(urljoin(current_url, response.location)).url + except UnsafeTarget as error: + return ChainResult( + False, + current_url, + error=f"unsafe redirect target {response.location!r}: {error}", + hard_failure=True, + ) + raise AssertionError("unreachable redirect state") + + +def _probe_once(url: str, settings: Settings) -> tuple[bool, str, bool]: + head = _request_chain(url, "HEAD", settings) + if head.success: + return True, f"HEAD {head.status} at {head.final_url}", False + if head.hard_failure: + return False, f"HEAD: {head.describe()}", False + + # Some servers reject or mishandle HEAD. Restart at the original URL with a one-byte GET. + get = _request_chain(url, "GET", settings) + if get.success: + return True, f"GET {get.status} at {get.final_url} (HEAD: {head.describe()})", False + return False, f"HEAD: {head.describe()}; GET: {get.describe()}", get.retryable + + +def check_url( + url: str, settings: Settings, sleep: Callable[[float], None] = time.sleep +) -> CheckResult: + try: + normalized = parse_target(url).url + except UnsafeTarget as error: + return CheckResult(url, False, f"unsafe target: {error}") + + detail = "" + for attempt in range(settings.retries + 1): + success, detail, retryable = _probe_once(normalized, settings) + if success: + return CheckResult(normalized, True, detail) + if not retryable or attempt == settings.retries: + break + delay = settings.backoff * (2**attempt) + sleep(delay) + return CheckResult(normalized, False, detail) + + +def _target_with_line_locations(text: str) -> list[tuple[str, int]]: + return markdown.extract_targets_with_lines(text) + + +def collect_external_links(paths: list[Path] | None = None) -> dict[str, list[str]]: + links: dict[str, set[str]] = {} + for source in paths if paths is not None else markdown.tracked_markdown(): + text = source.read_text(encoding="utf-8") + for raw_target, line in _target_with_line_locations(text): + target = markdown.normalize_target(raw_target) + if not re.match(r"^https?://", target, re.IGNORECASE): + continue + try: + network_url = parse_target(target).url + except UnsafeTarget: + network_url = target.split("#", 1)[0] + location = f"{source.relative_to(REPOSITORY_ROOT)}:{line}" + links.setdefault(network_url, set()).add(location) + return {url: sorted(locations) for url, locations in sorted(links.items())} + + +def load_ignore_file(path: Path) -> dict[str, str]: + rules: dict[str, str] = {} + dated_comment: tuple[str, str] | None = None + for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + line = raw_line.strip() + if not line: + dated_comment = None + continue + if line.startswith("#"): + match = DATED_COMMENT.match(line) + if match: + try: + date.fromisoformat(match.group(1)) + except ValueError as error: + raise ValueError( + f"{path}:{line_number}: invalid evidence date {match.group(1)!r}" + ) from error + dated_comment = (match.group(1), match.group(2)) + else: + dated_comment = None + continue + domain = line.lower().rstrip(".") + if ( + dated_comment is None + or not DOMAIN.fullmatch(domain) + or "*" in domain + or "/" in domain + or ".." in domain + ): + raise ValueError( + f"{path}:{line_number}: each exact domain needs an immediately preceding " + "'# YYYY-MM-DD: observed reason' comment" + ) + if domain in rules: + raise ValueError(f"{path}:{line_number}: duplicate domain {domain}") + rules[domain] = f"{dated_comment[0]}: {dated_comment[1]}" + dated_comment = None + return rules + + +def matching_ignore(hostname: str, rules: dict[str, str]) -> tuple[str, str] | None: + host = hostname.lower().rstrip(".") + for domain, reason in rules.items(): + if host == domain or host.endswith(f".{domain}"): + return domain, reason + return None + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--no-ignore", action="store_true", help="audit every URL, including ignored domains") + parser.add_argument("--ignore-file", type=Path, default=DEFAULT_IGNORE_FILE) + parser.add_argument("--workers", type=int, default=8) + parser.add_argument("--timeout", type=float, default=10.0) + parser.add_argument("--retries", type=int, default=2) + parser.add_argument("--backoff", type=float, default=1.0) + parser.add_argument("--max-redirects", type=int, default=5) + parser.add_argument( + "paths", + nargs="*", + type=Path, + help="Markdown files to check; defaults to every tracked Markdown file when omitted", + ) + return parser.parse_args() + + +def _selected_markdown(paths: list[Path]) -> list[Path]: + """Resolve requested Markdown paths, ignoring non-Markdown or untracked entries.""" + tracked = {path.resolve(): path for path in markdown.tracked_markdown()} + selected: list[Path] = [] + for path in paths: + resolved = (path if path.is_absolute() else (REPOSITORY_ROOT / path)).resolve() + tracked_path = tracked.get(resolved) + if tracked_path is not None and tracked_path not in selected: + selected.append(tracked_path) + return selected + + +def main() -> int: + arguments = _arguments() + if ( + arguments.workers < 1 + or arguments.timeout <= 0 + or arguments.retries < 0 + or arguments.backoff < 0 + or arguments.max_redirects < 0 + ): + print("invalid checker limits", file=sys.stderr) + return 2 + try: + ignore_rules = load_ignore_file(arguments.ignore_file) + except (OSError, ValueError) as error: + print(f"Cannot load external-link ignore file: {error}", file=sys.stderr) + return 2 + + if arguments.paths: + selected = _selected_markdown(arguments.paths) + if not selected: + print("No tracked Markdown files selected; nothing to check") + return 0 + links = collect_external_links(selected) + else: + links = collect_external_links() + ignored: dict[str, tuple[str, str]] = {} + to_check: list[str] = [] + malformed: list[CheckResult] = [] + for url in links: + try: + target = parse_target(url) + except UnsafeTarget as error: + malformed.append(CheckResult(url, False, f"unsafe target: {error}")) + continue + rule = matching_ignore(target.hostname, ignore_rules) + if rule and not arguments.no_ignore: + ignored[url] = rule + else: + to_check.append(url) + + settings = Settings(arguments.timeout, arguments.retries, arguments.backoff, arguments.max_redirects) + results = list(malformed) + if to_check: + with concurrent.futures.ThreadPoolExecutor(max_workers=min(arguments.workers, len(to_check))) as executor: + results.extend(executor.map(lambda url: check_url(url, settings), to_check)) + failures = sorted((result for result in results if not result.success), key=lambda result: result.url) + + if ignored: + ignored_domains = sorted({domain for domain, _ in ignored.values()}) + print( + f"Ignored {len(ignored)} URL(s) on documented bot-blocking domain(s): " + f"{', '.join(ignored_domains)}" + ) + if failures: + print( + f"External Markdown link validation failed: {len(failures)} of {len(results)} checked URL(s)", + file=sys.stderr, + ) + for result in failures: + print(f" {result.url}\n {result.detail}", file=sys.stderr) + for location in links[result.url]: + print(f" linked from {location}", file=sys.stderr) + return 1 + + occurrence_count = sum(len(locations) for locations in links.values()) + print( + "External Markdown links valid " + f"({len(results)} unique URL(s) checked, {len(ignored)} ignored, " + f"{occurrence_count} source location(s))" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_markdown_links.py b/scripts/check_markdown_links.py new file mode 100644 index 00000000..acd69da6 --- /dev/null +++ b/scripts/check_markdown_links.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""Check local links in every tracked Markdown file.""" + +from __future__ import annotations + +import html +import os +import re +import subprocess +import sys +from pathlib import Path +from urllib.parse import unquote, urlsplit + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") +REFERENCE_DEFINITION = re.compile(r"^\s*\[[^]]+\]:\s*(?:<([^>]+)>|(\S+))", re.MULTILINE) +HTML_TARGET = re.compile( + r"\b(?:href|src)\s*=\s*(?:\"([^\"]*)\"|'([^']*)'|([^\s>'\"]+))", + re.IGNORECASE, +) +ANGLE_AUTOLINK = re.compile(r"<((?:https?)://[^<>\s]+)>", re.IGNORECASE) +EXTENDED_URL_AUTOLINK = re.compile( + r"(^|[\s*_~(])(https?://([A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)+)[^\s<]*)", + re.IGNORECASE | re.MULTILINE, +) +AUTOLINK_TRAILING_PUNCTUATION = "?!.,:*_~" +OPENING_FENCE = re.compile(r"^[ \t]*(`{3,}|~{3,})(.*)$") +ATX_HEADING = re.compile(r"^ {0,3}#{1,6}(?:[ \t]+|$)(.*)$") +SETEXT_HEADING = re.compile(r"^ {0,3}(=+|-+)[ \t]*$") +HTML_ANCHOR = re.compile( + r"<(?:a\b[^>]*\b(?:id|name)|[A-Za-z][A-Za-z0-9:-]*\b[^>]*\bid)\s*=\s*" + r"(?:\"([^\"]+)\"|'([^']+)'|([^\s>'\"]+))", + re.IGNORECASE, +) + + +def tracked_markdown() -> list[Path]: + output = subprocess.check_output( + ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], + cwd=REPOSITORY_ROOT, + ).decode("utf-8") + paths = [REPOSITORY_ROOT / path for path in output.split("\0") if path.endswith(".md")] + return [path for path in paths if path.is_file()] + + +def strip_html_comments(line: str, in_comment: bool) -> tuple[str, bool]: + visible: list[str] = [] + position = 0 + while position < len(line): + if in_comment: + end = line.find("-->", position) + if end == -1: + return "".join(visible), True + position = end + 3 + in_comment = False + continue + start = line.find("\n" + ) + self.assertEqual(["https://visible.example/path"], checker.extract_targets(text)) + + def test_indented_code_url_is_ignored_but_paragraph_continuation_is_rendered(self) -> None: + text = ( + " https://indented-code.example/path\n" + "\n" + "Paragraph\n" + " https://continuation.example/path\n" + ) + self.assertEqual(["https://continuation.example/path"], checker.extract_targets(text)) + + def test_external_link_syntaxes_deduplicate_on_the_same_line(self) -> None: + url = "https://example.com/guide" + targets = checker.extract_targets_with_lines( + f"[Markdown]({url}) <{url}> HTML {url}\n" + ) + self.assertEqual([(url, 1)], targets) + + def test_explicit_link_destination_is_not_rescanned_as_bare_text(self) -> None: + url = "https://example.com/download" + self.assertEqual([(url, 1)], checker.extract_targets_with_lines(f"[Download]({url})\\\n")) + + def test_existing_same_document_fragment_is_accepted(self) -> None: + self.source.write_text("# Overview\n\n## Render content\n", encoding="utf-8") + self.assertIsNone(checker.check_target(self.source, "#render-content")) + + def test_missing_same_document_fragment_is_rejected(self) -> None: + self.source.write_text("# Overview\n", encoding="utf-8") + error = checker.check_target(self.source, "#missing-heading") + self.assertIn("broken local fragment", error or "") + + def test_empty_links_are_accepted(self) -> None: + self.source.write_text("# Overview\n", encoding="utf-8") + self.assertIsNone(checker.check_target(self.source, "")) + self.assertIsNone(checker.check_target(self.source, "#")) + + def test_multiline_link_label_is_scanned(self) -> None: + targets = checker.extract_targets("[a useful\nmultiline label](docs/guide.md)\n") + self.assertEqual(["docs/guide.md"], targets) + + def test_unquoted_html_target_is_scanned(self) -> None: + targets = checker.extract_targets("Example\n") + self.assertEqual(["images/example.png"], targets) + + def test_links_inside_html_comments_are_ignored(self) -> None: + text = "before\n\nafter\n" + self.assertEqual([], checker.extract_targets(text)) + + def test_brackets_followed_by_spaced_parenthetical_are_not_links(self) -> None: + targets = checker.extract_targets("Region [0,0,960,540] (top-left)\n") + self.assertEqual([], targets) + + def test_shorter_fence_does_not_close_four_backtick_block(self) -> None: + text = ( + "````markdown\n" + "[hidden](missing-one.md)\n" + "```\n" + "[still hidden](missing-two.md)\n" + "````\n" + "[visible](present.md)\n" + ) + self.assertEqual(["present.md"], checker.extract_targets(text)) + + def test_duplicate_headings_use_github_suffixes(self) -> None: + fragments = checker.heading_fragments("# Example\n## Example\n## Example\n") + self.assertEqual({"example", "example-1", "example-2"}, fragments) + + def test_explicit_unquoted_html_anchor_is_supported(self) -> None: + fragments = checker.heading_fragments("\n") + self.assertIn("custom-anchor", fragments) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py new file mode 100644 index 00000000..ee3ec3a0 --- /dev/null +++ b/scripts/validate_repository.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Run all repository-wide static validation.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CHECKS = ( + ( + "unit tests", + sys.executable, + "-m", + "unittest", + "discover", + "-s", + str(REPOSITORY_ROOT / "scripts" / "tests"), + "-p", + "test_*.py", + ), + ("Markdown links", sys.executable, str(REPOSITORY_ROOT / "scripts" / "check_markdown_links.py")), +) + + +def main() -> int: + for label, *command in CHECKS: + print(f"==> {label}", flush=True) + result = subprocess.run(command, cwd=REPOSITORY_ROOT, check=False) + if result.returncode: + return result.returncode + print("Repository validation passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/submission_hooks/README.md b/submission_hooks/README.md new file mode 100644 index 00000000..40681739 --- /dev/null +++ b/submission_hooks/README.md @@ -0,0 +1,13 @@ +# AWS Deadline Cloud submission hooks + +Submission hooks inspect or modify job bundles immediately before the Deadline Cloud CLI submits them. Use them for workstation-side policy that should apply consistently across jobs. + +## Sample index + +This table covers every immediate sample directory in `submission_hooks/`. + +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [License limits](license_limits/) | Injecting fixed-license host requirements into every step before submission | Artists should receive centrally enforced license scheduling without editing job templates | + +Read the sample README for workstation deployment, security implications, Deadline Cloud Limit setup, and testing instructions. diff --git a/terraform/README.md b/terraform/README.md index 505db7e3..edfa94d2 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -1,28 +1,23 @@ # AWS Deadline Cloud sample Terraform configurations -With [Terraform](https://www.terraform.io/), you can use infrastructure as code to deploy infrastructure -such as a Deadline Cloud farm to your AWS account. Use the samples provided here directly or as a starting point -to create your own custom configurations. +With [Terraform](https://www.terraform.io/), you can deploy Deadline Cloud infrastructure as code. These configurations use the [AWS Cloud Control (AWSCC) provider](https://registry.terraform.io/providers/hashicorp/awscc/latest), which supports AWS Deadline Cloud resource types. -These Terraform configurations use the [AWS Cloud Control (AWSCC) provider](https://registry.terraform.io/providers/hashicorp/awscc/latest) -for Deadline Cloud resources, which offers full support for AWS Deadline Cloud resource types. +## Sample index -## Starter farm +This table covers every immediate sample directory below `terraform/farm_templates/`. -The [starter_farm](farm_templates/starter_farm/) sample Terraform configuration deploys a Deadline Cloud farm you can use to run jobs that render images, -reconstruct 3D scenes, or transform your data in custom ways. This is the Terraform equivalent of the -[CloudFormation starter_farm template](../cloudformation/farm_templates/starter_farm/). -Sample jobs to submit are available in the [deadline-cloud-samples on GitHub](https://github.com/aws-deadline/deadline-cloud-samples). -Deadline Cloud provides many integrated submitter plugins for applications, and you can build your own jobs. The deployed farm includes the ability to -[build custom conda packages](../conda_recipes/README.md) for providing additional application support. +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [Starter farm](farm_templates/starter_farm/) | A farm, queue, service-managed fleets, and package-build support using AWSCC resources | Your team manages infrastructure with Terraform | + +The starter farm can run rendering, reconstruction, and data-transformation jobs from the [job bundle index](../job_bundles/). It is the Terraform equivalent of the [CloudFormation starter farm](../cloudformation/farm_templates/starter_farm/) and includes support for [building custom Conda packages](../conda_recipes/). ## CloudFormation vs Terraform -Both CloudFormation and Terraform configurations in this repository create equivalent infrastructure. -Choose based on your team's preferences and existing tooling: +Both starter configurations create equivalent infrastructure. Choose based on your team's existing tooling and state-management practices. | Aspect | CloudFormation | Terraform | -|--------|---------------|-----------| -| Provider | AWS native | HashiCorp | +|---|---|---| +| Provider | AWS native | HashiCorp AWSCC | | Deadline resources | `AWS::Deadline::*` | `awscc_deadline_*` | | State | Managed by AWS | Local or remote backend | diff --git a/utility_scripts/README.md b/utility_scripts/README.md index b7c17605..03826157 100644 --- a/utility_scripts/README.md +++ b/utility_scripts/README.md @@ -1,40 +1,44 @@ # AWS Deadline Cloud utility scripts -This directory contains sample utility scripts to help you work with AWS Deadline Cloud. These scripts provide -command-line tools for common tasks like managing job attachments, working with queues, and automating workflows. +These standalone command-line tools support common Deadline Cloud workflows outside a job bundle. ## Script index -### Upload to Job Attachments +This table covers every immediate user-selectable sample directory in `utility_scripts/`. -The [upload_to_job_attachments](upload_to_job_attachments) script uploads files and directories from your local -workstation or server to AWS Deadline Cloud job attachments storage. It uploads files to the job attachments S3 bucket in -content-addressable storage format, allowing subsequent Deadline Cloud jobs to use the data without re-uploading. This is useful -for pre-populating job attachments with large datasets that multiple jobs will use. +| Sample | What it demonstrates | Start here when | +|---|---|---| +| [Upload to job attachments](upload_to_job_attachments/) | Uploading files into content-addressable job attachment storage with deduplication | Large or reused datasets should be staged before job submission | + +## Upload to job attachments + +The uploader accepts files and directories from a workstation or server and places them in a queue's job attachments S3 bucket. Subsequent jobs can use the data without uploading unchanged content again. Key features: -- Upload individual files or entire directories recursively -- Multi-threaded uploads for better performance -- Automatic deduplication (skips files already in S3) -- Generates JSON manifest of uploaded files -- Two configuration modes: direct S3 specification or queue lookup - -Example usage: -```bash -# Upload using direct S3 specification -python upload_to_job_attachments.py \ + +* Upload individual files or directories recursively. +* Use multiple upload threads. +* Skip content that is already present in S3. +* Generate a JSON manifest. +* Configure storage directly or discover it from a farm and queue. + +```console +# Upload using an explicit S3 location +python upload_to_job_attachments/upload_to_job_attachments.py \ --s3-bucket my-bucket \ --s3-prefix job-attachments \ --paths /path/to/files /path/to/directory -# Upload using queue lookup -python upload_to_job_attachments.py \ +# Discover storage from a queue +python upload_to_job_attachments/upload_to_job_attachments.py \ --farm-id farm-1234567890abcdef \ --queue-id queue-1234567890abcdef \ --paths /path/to/files /path/to/directory ``` -## Additional Resources +See the [sample README](upload_to_job_attachments/) for installation, permissions, options, and manifest details. + +## Additional resources * [AWS Deadline Cloud user guide](https://docs.aws.amazon.com/deadline-cloud/latest/userguide/index.html) * [AWS Deadline Cloud developer guide](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/index.html)