template.yaml

specificationVersion: jobtemplate-2023-09

name: "CondaBuild: {{Param.RecipeName}}"
description: |
  This job uses conda-build to build a Conda package for the conda build recipe and optional attached source archive.

  https://github.com/aws-deadline/deadline-cloud-samples/tree/mainline/conda_recipes#readme

  Use the submit-package-job script in the parent directory as follows:

  $ ./submit-package-job blender-4.2

parameterDefinitions:
# Group: Conda Package Recipe
- name: BuildTool
  type: STRING
  userInterface:
    control: DROPDOWN_LIST
    label: Package Build Tool
    groupLabel: Conda Package Recipe
  description: Choose the build tool to use for the recipe (conda-build or rattler-build).
  default: conda-build
  allowedValues: ["conda-build", "rattler-build"]
- name: RecipeName
  description: The name of the recipe being built.
  userInterface:
    control: LINE_EDIT
    label: Recipe Name
    groupLabel: Conda Package Recipe
  type: STRING
- name: RecipeDir
  description: The directory containing the conda-build recipe.
  userInterface:
    control: CHOOSE_DIRECTORY
    label: Conda-build Recipe Dir
    groupLabel: Conda Package Recipe
  type: PATH
  objectType: DIRECTORY
  dataFlow: IN
- name: OverrideSourceArchive1
  description: |
    This replaces the recipe's first source URL with this archive file. Use "" or leave the
    parameter out to use the recipe's first source URL.
  userInterface:
    control: CHOOSE_INPUT_FILE
    label: Override Source Archive File 1
    groupLabel: Conda Package Recipe
  type: PATH
  objectType: FILE
  dataFlow: IN
  default: ""
- name: OverrideSourceArchive2
  description: |
    If the source URLs are a list, this replaces the recipe's second source URL with this archive file. Use "" or leave the
    parameter out to use the recipe's second source URL.
  userInterface:
    control: CHOOSE_INPUT_FILE
    label: Override Source Archive File 2
    groupLabel: Conda Package Recipe
  type: PATH
  objectType: FILE
  dataFlow: IN
  default: ""
- name: OverridePackageName
  description: If provided, overrides the package name in the recipe's meta.yaml file.
  type: STRING
  default: ""
  userInterface:
    control: LINE_EDIT
    label: Override Package Name
    groupLabel: Conda Package Recipe
- name: OverridePrefixLength
  description: |
    The --prefix-length to override in the package build.

    An example use case is to use conda itself to incept a conda environment
    into a subdirectory of the package. The interior prefix needs to be
    subtracted from the prefix length for the build to work. For
    example, if using the interior prefix "/opt/nerfstudio/", set this to
    239 = 255 - len("/opt/nerfstudio/").
  type: INT
  default: 0
  minValue: 0
  maxValue: 255
  userInterface:
    control: SPIN_BOX
    label: Override Prefix Length
    groupLabel: Conda Package Recipe
- name: VariantConfigFile
  description: |
    If provided, this is used as the variant config for the conda package build.
  type: PATH
  objectType: FILE
  dataFlow: IN
  default: ""
  userInterface:
    control: CHOOSE_INPUT_FILE
    label: Variant config YAML file
    groupLabel: Conda Package Recipe

# Group: Conda Channel
- name: CondaChannelName
  description: The name of the Conda channel to update.
  userInterface:
    control: LINE_EDIT
    label: Channel Name
    groupLabel: Conda Channel
  type: STRING
  default: User Conda Channel
- name: S3CondaChannel
  description: The S3 URL of the destination conda channel. E.g. s3://<s3-bucket-name>/channel/prefix
  userInterface:
    control: LINE_EDIT
    label: S3 Conda Channel
    groupLabel: Conda Channel
  type: STRING
- name: CondaPlatform
  description: |
    The Conda platform we're building the package for, e.g.
    linux-64, win-64.
  userInterface:
    control: LINE_EDIT
    label: Conda Platform
    groupLabel: Conda Channel
  type: STRING
  default: linux-64
- name: CondaChannels
  description: A space-separated list of conda channels to read packages from.
  userInterface:
    control: LINE_EDIT
    label: Additional Channels
    groupLabel: Conda Channel
  type: STRING
  default: ""

# Group: Package Build Env
- name: PackageBuildEnvName
  description: The name of the Conda virtual environment with conda-build and rattler-build installed.
  userInterface:
    control: LINE_EDIT
    label: Package Build Env Name
    groupLabel: Package Build Env
  type: STRING
  default: "PackageBuildEnv"
- name: ReusePackageBuildEnv
  description: Whether to reuse the Conda virtual environment for conda-build or not.
  userInterface:
    control: CHECK_BOX
    label: Reuse the named Conda Build environment.
    groupLabel: Package Build Env
  type: STRING
  default: "1"
  allowedValues: ["1", "0"]

# Hidden Controls
- name: JobScriptDir
  description: Directory containing embedded scripts.
  userInterface:
    control: HIDDEN
  type: PATH
  objectType: DIRECTORY
  dataFlow: IN
  default: scripts

jobEnvironments:

- name: Package Build Env
  variables:
    # Tell Qt applications to run off-screen.
    QT_QPA_PLATFORM: offscreen
    # Turn off buffering in Python
    PYTHONUNBUFFERED: "True"
  script:
    actions:
      onEnter:
        command: "bash"
        args:
        - '{{Param.JobScriptDir}}/enter-package-build-env.sh'
        - '--env-name'
        - '{{Param.PackageBuildEnvName}}'
        - '--conda-bld-dir'
        - '{{Session.WorkingDirectory}}/conda-bld'
        - '--reuse-env'
        - '{{Param.ReusePackageBuildEnv}}'
        - '--temp-dir'
        - '{{Session.WorkingDirectory}}/temp'

steps:

- name: PackageBuild
  description: |
    Builds a conda package and uploads it to an S3 conda channel.

    The following metadata controls how the submit-package-job-script.py script
    decomposes and re-assembles this job to support multiple conda platforms.

    meta:
      perStepParameters:
      - BuildTool
      - OverrideSourceArchive1
      - OverrideSourceArchive2
      - CondaPlatform
      - VariantConfigFile

  stepEnvironments:
  - name: S3 Conda Channels
    script:
      actions:
        onEnter:
          command: "bash"
          args:
          - '{{Param.JobScriptDir}}/enter-proxy-s3-conda-channels.sh'
          - '--build-tool'
          - '{{Param.BuildTool}}'
          - '--conda-channels'
          - '{{Param.CondaChannels}}'
          - '--s3-conda-channel'
          - '{{Param.S3CondaChannel}}'
        onExit:
          command: "bash"
          args:
          - '{{Param.JobScriptDir}}/exit-proxy-s3-conda-channels.sh'

  script:
    actions:
      onRun:
        command: "bash"
        args: ["{{Task.File.Run}}"]
    embeddedFiles:
    - name: Run
      type: TEXT
      data: |
        #!/bin/env bash
        set -xeuo pipefail

        python '{{Param.JobScriptDir}}/build-package.py' \
            --build-tool '{{Param.BuildTool}}' \
            --conda-bld-dir '{{Session.WorkingDirectory}}/conda-bld' \
            --conda-platform '{{Param.CondaPlatform}}' \
            --conda-channels "$CONDA_CHANNELS" \
            --s3-conda-channel '{{Param.S3CondaChannel}}' \
            --proxy-s3-conda-channel "$S3_CONDA_CHANNEL" \
            --override-prefix-length '{{Param.OverridePrefixLength}}' \
            --recipe-dir '{{Param.RecipeDir}}' \
            --override-package-name '{{Param.OverridePackageName}}' \
            --override-source-archive1 '{{Param.OverrideSourceArchive1}}' \
            --override-source-archive2 '{{Param.OverrideSourceArchive2}}' \
            --variant-config-file '{{Param.VariantConfigFile}}'

  hostRequirements:
    # Host requirements for the linux-64 conda platform
    attributes:
    - name: attr.worker.os.family
      anyOf:
      - linux
    - name: attr.worker.cpu.arch
      anyOf:
      - x86_64

- name: ReindexCondaChannel
  description: Reindexes an S3 conda channel.

  dependencies:
  - dependsOn: PackageBuild

  stepEnvironments:
  - name: ReindexingMutex
    script:
      actions:
        onEnter:
          command: python
          args:
          - '{{Param.JobScriptDir}}/s3-object-mutex.py'
          - 'enter'
          - '{{Param.S3CondaChannel}}/'
        onExit:
          command: python
          args:
          - '{{Param.JobScriptDir}}/s3-object-mutex.py'
          - 'exit'
          - '{{Param.S3CondaChannel}}/'

  script:
    actions:
      onRun:
        command: bash
        args:
        - '{{Param.JobScriptDir}}/reindex-conda-channel.sh'
        - '--reindexing-dir'
        - '{{Session.WorkingDirectory}}/reindex'
        - '--s3-conda-channel'
        - '{{Param.S3CondaChannel}}'
        - '--conda-channel-name'
        - '{{Param.CondaChannelName}}'

  hostRequirements:
    # Always reindex on Linux, because reindexing uses the Linux-only mountpoint-s3.
    attributes:
    - name: attr.worker.os.family
      anyOf:
      - linux