From 6a230dcfe7645373bc76b3a6685b86016a74df13 Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 10:47:15 -0700 Subject: [PATCH 01/11] feat(tensorflow): add TF Serving 2.20 inference DLC on AL2023 First TF inference DLC on the v2 (main) branch. Mirrors master's TF 2.19 inference image but ports to AL2023 + CUDA 12.9.1 + Python 3.12, switches to uv-driven dependency management, and rebuilds nginx-mod-njs from source (no AL2023 RPM exists). SageMaker only, x86 only, single-model + MME support preserved. - Dockerfile.cuda + Dockerfile.cpu with builder-njs source-build stage - Ports SageMaker handler scripts (Falcon + nginx + njs + multi_model_utils) from master TF 2.19 build_artifacts/sagemaker/ byte-for-byte - GitHub Actions workflows mirror PR #6107 (TF 2.21 training) shape - SageMaker integration tests for single-model and MME endpoints - ECR scan allowlist overlay at tensorflow/tensorflow-2.20.json TFS 2.20.0 binary copied from tensorflow/serving:2.20.0-devel-gpu image. tensorflow-serving-api installed with --no-deps to avoid pulling 600 MB of TF framework. framework: "tensorflow" + job_type: "inference" matches the cross-team release-logic convention (training and inference share the framework slug, distinguished by job_type). Known follow-ups (not blocking this PR): - PR #6107 (TF 2.21 training) will add tensorflow/framework_allowlist.json (shared base) once it merges. CI security-test may fail until then. - Separate change against the release-logic config needed to add tensorflow entry to frameworks.yml before image can release to prod. --- ...ensorflow-2.20-inference-sagemaker-cpu.yml | 22 + ...nsorflow-2.20-inference-sagemaker-cuda.yml | 23 + .../pr-tensorflow-inference-sagemaker-cpu.yml | 360 +++++++++ ...pr-tensorflow-inference-sagemaker-cuda.yml | 366 ++++++++++ .../tensorflow/inference/2.20/Dockerfile.cpu | 227 ++++++ .../tensorflow/inference/2.20/Dockerfile.cuda | 265 +++++++ .../inference/2.20/cpu/pyproject.toml | 67 ++ docker/tensorflow/inference/2.20/cpu/uv.lock | 550 ++++++++++++++ .../inference/2.20/cuda/pyproject.toml | 90 +++ docker/tensorflow/inference/2.20/cuda/uv.lock | 550 ++++++++++++++ .../inference/2.20/versions-cpu.env | 32 + .../inference/2.20/versions-cuda.env | 46 ++ .../inference/dockerd_entrypoint.sh | 12 + .../inference/sagemaker/__init__.py | 12 + .../inference/sagemaker/multi_model_utils.py | 53 ++ .../inference/sagemaker/nginx.conf.template | 67 ++ .../inference/sagemaker/python_service.py | 689 ++++++++++++++++++ scripts/tensorflow/inference/sagemaker/serve | 3 + .../tensorflow/inference/sagemaker/serve.py | 522 +++++++++++++ .../inference/sagemaker/tensorflowServing.js | 239 ++++++ .../inference/sagemaker/tfs_utils.py | 337 +++++++++ .../inference/tf_serving_entrypoint.sh | 6 + .../tensorflow/tensorflow-2.20.json | 1 + .../tensorflow/integration/inference/.gitkeep | 0 .../integration/inference/__init__.py | 0 .../integration/inference/conftest.py | 107 +++ .../integration/inference/requirements.txt | 5 + .../inference/resources/__init__.py | 0 .../inference/resources/build_sample_model.py | 60 ++ .../inference/test_multi_model_endpoint.py | 129 ++++ .../inference/test_single_model_endpoint.py | 89 +++ 31 files changed, 4929 insertions(+) create mode 100644 .github/config/image/tensorflow-2.20-inference-sagemaker-cpu.yml create mode 100644 .github/config/image/tensorflow-2.20-inference-sagemaker-cuda.yml create mode 100644 .github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml create mode 100644 .github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml create mode 100644 docker/tensorflow/inference/2.20/Dockerfile.cpu create mode 100644 docker/tensorflow/inference/2.20/Dockerfile.cuda create mode 100644 docker/tensorflow/inference/2.20/cpu/pyproject.toml create mode 100644 docker/tensorflow/inference/2.20/cpu/uv.lock create mode 100644 docker/tensorflow/inference/2.20/cuda/pyproject.toml create mode 100644 docker/tensorflow/inference/2.20/cuda/uv.lock create mode 100644 docker/tensorflow/inference/2.20/versions-cpu.env create mode 100644 docker/tensorflow/inference/2.20/versions-cuda.env create mode 100755 scripts/tensorflow/inference/dockerd_entrypoint.sh create mode 100644 scripts/tensorflow/inference/sagemaker/__init__.py create mode 100644 scripts/tensorflow/inference/sagemaker/multi_model_utils.py create mode 100644 scripts/tensorflow/inference/sagemaker/nginx.conf.template create mode 100644 scripts/tensorflow/inference/sagemaker/python_service.py create mode 100755 scripts/tensorflow/inference/sagemaker/serve create mode 100644 scripts/tensorflow/inference/sagemaker/serve.py create mode 100644 scripts/tensorflow/inference/sagemaker/tensorflowServing.js create mode 100644 scripts/tensorflow/inference/sagemaker/tfs_utils.py create mode 100755 scripts/tensorflow/inference/tf_serving_entrypoint.sh create mode 100644 test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json create mode 100644 test/tensorflow/integration/inference/.gitkeep create mode 100644 test/tensorflow/integration/inference/__init__.py create mode 100644 test/tensorflow/integration/inference/conftest.py create mode 100644 test/tensorflow/integration/inference/requirements.txt create mode 100644 test/tensorflow/integration/inference/resources/__init__.py create mode 100644 test/tensorflow/integration/inference/resources/build_sample_model.py create mode 100644 test/tensorflow/integration/inference/test_multi_model_endpoint.py create mode 100644 test/tensorflow/integration/inference/test_single_model_endpoint.py diff --git a/.github/config/image/tensorflow-2.20-inference-sagemaker-cpu.yml b/.github/config/image/tensorflow-2.20-inference-sagemaker-cpu.yml new file mode 100644 index 000000000000..0ab46d6128b1 --- /dev/null +++ b/.github/config/image/tensorflow-2.20-inference-sagemaker-cpu.yml @@ -0,0 +1,22 @@ +image: + name: "tensorflow-sagemaker-cpu" + description: "TensorFlow Serving 2.20 CPU inference for SageMaker" +common: + framework: "tensorflow" + framework_version: "2.20.0" + job_type: "inference" + python_version: "py312" + os_version: "amzn2023" + customer_type: "sagemaker" + platform: "sagemaker" + arch_type: "x86" + prod_image: "tensorflow-inference:2.20-cpu-amzn2023-sagemaker" + device_type: "cpu" + contributor: "None" +release: + release: true + force_release: false + public_registry: true + private_registry: true + enable_soci: true + environment: production diff --git a/.github/config/image/tensorflow-2.20-inference-sagemaker-cuda.yml b/.github/config/image/tensorflow-2.20-inference-sagemaker-cuda.yml new file mode 100644 index 000000000000..1a29dee0202f --- /dev/null +++ b/.github/config/image/tensorflow-2.20-inference-sagemaker-cuda.yml @@ -0,0 +1,23 @@ +image: + name: "tensorflow-sagemaker-cuda" + description: "TensorFlow Serving 2.20 CUDA inference for SageMaker" +common: + framework: "tensorflow" + framework_version: "2.20.0" + job_type: "inference" + python_version: "py312" + cuda_version: "cu129" + os_version: "amzn2023" + customer_type: "sagemaker" + platform: "sagemaker" + arch_type: "x86" + prod_image: "tensorflow-inference:2.20-cu129-amzn2023-sagemaker" + device_type: "gpu" + contributor: "None" +release: + release: true + force_release: false + public_registry: true + private_registry: true + enable_soci: true + environment: production diff --git a/.github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml b/.github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml new file mode 100644 index 000000000000..8526b1057eb3 --- /dev/null +++ b/.github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml @@ -0,0 +1,360 @@ +name: PR - TensorFlow Inference SageMaker CPU + +# Mirrors .github/workflows/pr-tensorflow-sagemaker-cpu.yml from the +# tensorflow-2.21-currency branch (TF 2.21 training PR #6107). Inference deltas: +# - path triggers narrowed to docker/tensorflow/inference/** (the training +# workflow's globs would also fire on inference changes — flagged as a +# latent issue in the training workflow; not fixed here) +# - image config glob: tensorflow-*-inference-sagemaker-cpu.yml +# - version detection regex matches docker/tensorflow/inference// paths +# - ECR tag uses tensorflow-inference (mirrors master TF 2.19 inference repo +# naming) and TF_SERVING_VERSION from versions-cpu.env (2.20.0) — note this +# is the TFS binary version, not the framework version +# - --build-args swapped: drop OPEN_MPI/TF_VERSION (training-only), +# add TF_SERVING_VERSION/TFS_SHORT_VERSION/NGINX_VERSION/NJS_VERSION +# - drop unit-test (no test/tensorflow/unit/ inference suite exists; Phase 5 +# scope. TODO marker below if added later) +# - sagemaker-test points at test/tensorflow/integration/inference/ which +# Phase 5 will populate + +on: + pull_request: + branches: [main] + types: [opened, reopened, synchronize] + paths: + - ".github/config/image/tensorflow-*-inference-sagemaker-cpu.yml" + - ".github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml" + - "docker/tensorflow/inference/*/Dockerfile.cpu" + - "docker/tensorflow/inference/*/cpu/**" + - "docker/tensorflow/inference/*/versions-cpu.env" + - "scripts/common/**" + - "scripts/tensorflow/inference/**" + - "scripts/telemetry/**" + - "test/tensorflow/integration/inference/**" + - "test/sanity/**" + - "test/telemetry/**" + - "!docs/**" + +permissions: + contents: read + pull-requests: read + +env: + FORCE_COLOR: "1" + # TFS version (binary), not framework version. See versions-cpu.env header. + LATEST_TENSORFLOW_INFERENCE_VERSION: "2.20" + +jobs: + # ============================================================ + # Gate: permission check on base branch + # ============================================================ + gatekeeper: + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: + - name: Checkout base branch (safe) + uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 1 + + - name: Run permission gate (from base) + uses: ./.github/actions/pr-permission-gate + + # ============================================================ + # Detect changed TensorFlow inference versions + build per-version configs + # matrix + run pre-commit + path-based change detection + # ============================================================ + check-changes: + needs: [gatekeeper] + if: success() + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + outputs: + versions: ${{ steps.versions.outputs.versions }} + configs: ${{ steps.versions.outputs.configs }} + build-change: ${{ steps.changes.outputs.build-change }} + sanity-test-change: ${{ steps.changes.outputs.sanity-test-change }} + telemetry-test-change: ${{ steps.changes.outputs.telemetry-test-change }} + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Setup python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + with: + extra_args: --all-files + + - name: Install yq + run: | + if ! command -v yq &> /dev/null; then + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + fi + + - name: Detect TensorFlow inference versions and build configs matrix + id: versions + run: | + # Match either the docker path (docker/tensorflow/inference/2.20/...) + # or the config-file naming (tensorflow-2.20-inference-...). + VERSIONS=$(git diff --name-only origin/main...HEAD \ + | grep -oP '(?:docker/tensorflow/inference/|tensorflow-)\K[0-9]+\.[0-9]+' \ + | sort -u) + if [ -z "$VERSIONS" ]; then + VERSIONS="$LATEST_TENSORFLOW_INFERENCE_VERSION" + fi + JSON=$(echo "$VERSIONS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "versions=${JSON}" >> $GITHUB_OUTPUT + echo "Detected versions: ${JSON}" + + # Build a configs matrix: each entry carries all metadata fields + CONFIGS="[]" + for V in $VERSIONS; do + CONFIG_FILE=".github/config/image/tensorflow-${V}-inference-sagemaker-cpu.yml" + if [ -f "$CONFIG_FILE" ]; then + CONFIGS=$(echo "$CONFIGS" | jq -c \ + --arg v "$V" \ + --arg fw "$(yq '.common.framework' $CONFIG_FILE)" \ + --arg fwv "$(yq '.common.framework_version' $CONFIG_FILE)" \ + --arg py "$(yq '.common.python_version' $CONFIG_FILE)" \ + --arg cuda "$(yq '.common.cuda_version // ""' $CONFIG_FILE)" \ + --arg os "$(yq '.common.os_version' $CONFIG_FILE)" \ + --arg ct "$(yq '.common.job_type' $CONFIG_FILE)" \ + --arg dt "$(yq '.common.device_type // "cpu"' $CONFIG_FILE)" \ + --arg at "$(yq '.common.arch_type // "x86"' $CONFIG_FILE)" \ + --arg contrib "$(yq '.common.contributor // "None"' $CONFIG_FILE)" \ + --arg cust "$(yq '.common.customer_type // ""' $CONFIG_FILE)" \ + --arg prod "$(yq '.common.prod_image' $CONFIG_FILE)" \ + '. + [{"version": $v, "framework": $fw, "framework_version": $fwv, "python_version": $py, "cuda_version": $cuda, "os_version": $os, "container_type": $ct, "device_type": $dt, "arch_type": $at, "contributor": $contrib, "customer_type": $cust, "prod_image": $prod}]') + fi + done + echo "configs=${CONFIGS}" >> $GITHUB_OUTPUT + echo "Configs matrix: ${CONFIGS}" + + - name: Detect file changes + id: changes + uses: dorny/paths-filter@v4 + with: + filters: | + build-change: + - ".github/config/image/tensorflow-*-inference-sagemaker-cpu.yml" + - "docker/tensorflow/inference/*/Dockerfile.cpu" + - "docker/tensorflow/inference/*/cpu/**" + - "docker/tensorflow/inference/*/versions-cpu.env" + - "scripts/common/setup_oss_compliance.sh" + - "scripts/tensorflow/inference/**" + - "scripts/telemetry/bash_telemetry.sh.template" + sanity-test-change: + - "test/sanity/**" + telemetry-test-change: + - "test/telemetry/**" + + # ============================================================ + # Build CPU SageMaker inference images (matrix over detected versions) + # ============================================================ + build-image: + needs: [check-changes] + if: needs.check-changes.outputs.build-change == 'true' + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + concurrency: + group: ${{ github.workflow }}-build-${{ matrix.version }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Setup buildkitd + run: .github/scripts/buildkitd.sh + + - name: ECR login + uses: ./.github/actions/ecr-authenticate + with: + aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Build sagemaker inference image + id: build-sagemaker + run: | + VERSION="${{ matrix.version }}" + source docker/tensorflow/inference/${VERSION}/versions-cpu.env + # ECR tag uses TF Serving binary version (TF_SERVING_VERSION from + # versions-cpu.env) — mirrors master TF 2.19 inference repo naming. + CI_IMAGE_URI="${{ vars.CI_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/ci:tensorflow-inference-${TF_SERVING_VERSION}-cpu-py312-sagemaker-${{ matrix.version }}-pr-${{ github.event.pull_request.number }}" + + # Derive label values to match check_labels.py expectations + FRAMEWORK_LABEL=$(echo "${{ matrix.framework }}" | tr '_' '-') + FWK_VER_LABEL=$(echo "${{ matrix.framework_version }}" | tr '.' '-') + OS_LABEL=$(echo "${{ matrix.os_version }}" | tr '.' '-') + + docker buildx build --progress plain \ + --build-arg FRAMEWORK=${{ matrix.framework }} \ + --build-arg PYTHON_VERSION=${PYTHON_VERSION} \ + --build-arg TF_SERVING_VERSION=${TF_SERVING_VERSION} \ + --build-arg TFS_SHORT_VERSION=${VERSION} \ + --build-arg NGINX_VERSION=${NGINX_VERSION} \ + --build-arg NJS_VERSION=${NJS_VERSION} \ + --build-arg DLC_MAJOR_VERSION=${DLC_MAJOR_VERSION} \ + --build-arg DLC_MINOR_VERSION=${DLC_MINOR_VERSION} \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.framework.${FRAMEWORK_LABEL}.${FWK_VER_LABEL}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.device.cpu=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.job.${{ matrix.container_type }}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.arch.${{ matrix.arch_type }}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.os.${OS_LABEL}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.python.${{ matrix.python_version }}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.contributor.${{ matrix.contributor }}=true" \ + --cache-to=type=inline \ + --cache-from=type=registry,ref=${CI_IMAGE_URI} \ + --tag ${CI_IMAGE_URI} \ + --push \ + --target sagemaker \ + -f docker/tensorflow/inference/${VERSION}/Dockerfile.cpu . + + echo "image-uri=${CI_IMAGE_URI}" >> $GITHUB_OUTPUT + + # ============================================================ + # Sanity tests + # ============================================================ + sanity-test: + needs: [check-changes, build-image] + if: | + always() && !failure() && !cancelled() && + (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.sanity-test-change == 'true') + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + uses: ./.github/workflows/reusable-sanity-tests.yml + with: + image-uri: ${{ needs.build-image.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:tensorflow-inference-{2}-cpu-py312-sagemaker-{3}-pr-{4}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.framework_version, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} + aws-account-id: ${{ needs.build-image.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + framework: ${{ matrix.framework }} + framework-version: ${{ matrix.framework_version }} + python-version: ${{ matrix.python_version }} + cuda-version: ${{ matrix.cuda_version }} + os-version: ${{ matrix.os_version }} + customer-type: ${{ matrix.customer_type }} + arch-type: ${{ matrix.arch_type }} + device-type: ${{ matrix.device_type }} + contributor: ${{ matrix.contributor }} + container-type: ${{ matrix.container_type }} + + # ============================================================ + # Security tests + # ============================================================ + security-test: + needs: [check-changes, build-image] + if: success() + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + uses: ./.github/workflows/reusable-security-tests.yml + with: + image-uri: ${{ needs.build-image.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:tensorflow-inference-{2}-cpu-py312-sagemaker-{3}-pr-{4}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.framework_version, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} + aws-account-id: ${{ needs.build-image.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + framework: ${{ matrix.framework }} + framework-version: ${{ matrix.framework_version }} + + # ============================================================ + # Telemetry tests + # ============================================================ + telemetry-test: + needs: [check-changes, build-image] + if: | + always() && !failure() && !cancelled() && + (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.telemetry-test-change == 'true') + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + concurrency: + group: ${{ github.workflow }}-telemetry-test-${{ matrix.version }}-${{ github.event.pull_request.number }} + cancel-in-progress: false + uses: ./.github/workflows/reusable-telemetry-tests.yml + with: + image-uri: ${{ needs.build-image.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:tensorflow-inference-{2}-cpu-py312-sagemaker-{3}-pr-{4}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.framework_version, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} + aws-account-id: ${{ needs.build-image.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + framework: ${{ matrix.framework }} + framework-version: ${{ matrix.framework_version }} + container-type: ${{ matrix.container_type }} + + # TODO: unit-test job — training has one targeting test/tensorflow/unit/. + # Inference equivalent does not exist yet. Add in Phase 5 if a unit + # suite is created. + + # TODO: MME-specific test — image config sets + # com.amazonaws.sagemaker.capabilities.multi-models=true. Phase 5 should + # add an MME endpoint smoke test once the integration test scaffolding + # is in place. + + # ============================================================ + # SageMaker integration tests (launch real CPU SM endpoints) + # ============================================================ + # Phase 5 will populate test/tensorflow/integration/inference/. Until then this + # job will run pytest on an empty dir (pytest exits 5 = no tests collected; we + # treat that as success here so the workflow can land before Phase 5). + sagemaker-test: + needs: [check-changes, build-image, sanity-test, security-test] + if: success() + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + concurrency: + group: ${{ github.workflow }}-sagemaker-${{ matrix.version }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: ECR login + uses: ./.github/actions/ecr-authenticate + with: + aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Install test dependencies + run: | + # TODO: Phase 5 — replace with `pip install -r + # test/tensorflow/integration/inference/requirements.txt` once + # Phase 5 supplies that file. + pip install pytest sagemaker boto3 + + - name: Run SageMaker inference tests + env: + PYTHONPATH: ${{ github.workspace }}/test + TEST_IMAGE_URI: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/ci:tensorflow-inference-{2}-cpu-py312-sagemaker-{3}-pr-{4}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.framework_version, matrix.version, github.event.pull_request.number) }} + SM_ROLE_ARN: arn:aws:iam::${{ vars.CI_AWS_ACCOUNT_ID }}:role/SageMakerRole + run: | + # pytest exit 5 (no tests collected) is acceptable until Phase 5 adds + # tests under test/tensorflow/integration/inference/. + set +e + pytest test/tensorflow/integration/inference/ -v + rc=$? + set -e + if [ "$rc" != "0" ] && [ "$rc" != "5" ]; then + exit $rc + fi diff --git a/.github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml b/.github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml new file mode 100644 index 000000000000..c94fd9876ab3 --- /dev/null +++ b/.github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml @@ -0,0 +1,366 @@ +name: PR - TensorFlow Inference SageMaker CUDA + +# Mirrors .github/workflows/pr-tensorflow-sagemaker-cuda.yml from the +# tensorflow-2.21-currency branch (TF 2.21 training PR #6107). Inference deltas: +# - path triggers narrowed to docker/tensorflow/inference/** (the training +# workflow's globs would also fire on inference changes — flagged as a +# latent issue in the training workflow; not fixed here) +# - image config glob: tensorflow-*-inference-sagemaker-cuda.yml +# - version detection regex matches docker/tensorflow/inference// paths +# - ECR tag uses tensorflow-inference (mirrors master TF 2.19 inference repo +# naming) and TF_SERVING_VERSION from versions-cuda.env (2.20.0) — note this +# is the TFS binary version, not the framework version +# - --build-args swapped: drop NCCL/EFA/OpenMPI/TF_VERSION (training-only), +# add TF_SERVING_VERSION/TFS_SHORT_VERSION/NGINX_VERSION/NJS_VERSION +# - drop single-gpu-test (no in-container GPU smoke for an inference image — +# sanity already covers serving smoke; full SM endpoint test runs on real GPU) +# - drop unit-test (no test/tensorflow/unit/ inference suite exists; Phase 5 +# scope. TODO marker below if added later) +# - sagemaker-test points at test/tensorflow/integration/inference/ which +# Phase 5 will populate + +on: + pull_request: + branches: [main] + types: [opened, reopened, synchronize] + paths: + - ".github/config/image/tensorflow-*-inference-sagemaker-cuda.yml" + - ".github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml" + - "docker/tensorflow/inference/*/Dockerfile.cuda" + - "docker/tensorflow/inference/*/cuda/**" + - "docker/tensorflow/inference/*/versions-cuda.env" + - "scripts/common/**" + - "scripts/tensorflow/inference/**" + - "scripts/telemetry/**" + - "test/tensorflow/integration/inference/**" + - "test/sanity/**" + - "test/telemetry/**" + - "!docs/**" + +permissions: + contents: read + pull-requests: read + +env: + FORCE_COLOR: "1" + # TFS version (binary), not framework version. See versions-cuda.env header. + LATEST_TENSORFLOW_INFERENCE_VERSION: "2.20" + +jobs: + # ============================================================ + # Gate: permission check on base branch + # ============================================================ + gatekeeper: + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: + - name: Checkout base branch (safe) + uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 1 + + - name: Run permission gate (from base) + uses: ./.github/actions/pr-permission-gate + + # ============================================================ + # Detect changed TensorFlow inference versions + build per-version configs + # matrix + run pre-commit + path-based change detection + # ============================================================ + check-changes: + needs: [gatekeeper] + if: success() + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + outputs: + versions: ${{ steps.versions.outputs.versions }} + configs: ${{ steps.versions.outputs.configs }} + build-change: ${{ steps.changes.outputs.build-change }} + sanity-test-change: ${{ steps.changes.outputs.sanity-test-change }} + telemetry-test-change: ${{ steps.changes.outputs.telemetry-test-change }} + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Setup python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + with: + extra_args: --all-files + + - name: Install yq + run: | + if ! command -v yq &> /dev/null; then + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + fi + + - name: Detect TensorFlow inference versions and build configs matrix + id: versions + run: | + # Match either the docker path (docker/tensorflow/inference/2.20/...) + # or the config-file naming (tensorflow-2.20-inference-...). + VERSIONS=$(git diff --name-only origin/main...HEAD \ + | grep -oP '(?:docker/tensorflow/inference/|tensorflow-)\K[0-9]+\.[0-9]+' \ + | sort -u) + if [ -z "$VERSIONS" ]; then + VERSIONS="$LATEST_TENSORFLOW_INFERENCE_VERSION" + fi + JSON=$(echo "$VERSIONS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "versions=${JSON}" >> $GITHUB_OUTPUT + echo "Detected versions: ${JSON}" + + # Build a configs matrix: each entry carries all metadata fields + CONFIGS="[]" + for V in $VERSIONS; do + CONFIG_FILE=".github/config/image/tensorflow-${V}-inference-sagemaker-cuda.yml" + if [ -f "$CONFIG_FILE" ]; then + CONFIGS=$(echo "$CONFIGS" | jq -c \ + --arg v "$V" \ + --arg fw "$(yq '.common.framework' $CONFIG_FILE)" \ + --arg fwv "$(yq '.common.framework_version' $CONFIG_FILE)" \ + --arg py "$(yq '.common.python_version' $CONFIG_FILE)" \ + --arg cuda "$(yq '.common.cuda_version' $CONFIG_FILE)" \ + --arg os "$(yq '.common.os_version' $CONFIG_FILE)" \ + --arg ct "$(yq '.common.job_type' $CONFIG_FILE)" \ + --arg dt "$(yq '.common.device_type // "gpu"' $CONFIG_FILE)" \ + --arg at "$(yq '.common.arch_type // "x86"' $CONFIG_FILE)" \ + --arg contrib "$(yq '.common.contributor // "None"' $CONFIG_FILE)" \ + --arg cust "$(yq '.common.customer_type // ""' $CONFIG_FILE)" \ + --arg prod "$(yq '.common.prod_image' $CONFIG_FILE)" \ + '. + [{"version": $v, "framework": $fw, "framework_version": $fwv, "python_version": $py, "cuda_version": $cuda, "os_version": $os, "container_type": $ct, "device_type": $dt, "arch_type": $at, "contributor": $contrib, "customer_type": $cust, "prod_image": $prod}]') + fi + done + echo "configs=${CONFIGS}" >> $GITHUB_OUTPUT + echo "Configs matrix: ${CONFIGS}" + + - name: Detect file changes + id: changes + uses: dorny/paths-filter@v4 + with: + filters: | + build-change: + - ".github/config/image/tensorflow-*-inference-sagemaker-cuda.yml" + - "docker/tensorflow/inference/*/Dockerfile.cuda" + - "docker/tensorflow/inference/*/cuda/**" + - "docker/tensorflow/inference/*/versions-cuda.env" + - "scripts/common/setup_oss_compliance.sh" + - "scripts/common/start_cuda_compat.sh" + - "scripts/tensorflow/inference/**" + - "scripts/telemetry/bash_telemetry.sh.template" + sanity-test-change: + - "test/sanity/**" + telemetry-test-change: + - "test/telemetry/**" + + # ============================================================ + # Build SageMaker inference images (matrix over detected versions) + # ============================================================ + build-image: + needs: [check-changes] + if: needs.check-changes.outputs.build-change == 'true' + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:x86-build-runner + buildspec-override:true + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + concurrency: + group: ${{ github.workflow }}-build-${{ matrix.version }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Setup buildkitd + run: .github/scripts/buildkitd.sh + + - name: ECR login + uses: ./.github/actions/ecr-authenticate + with: + aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Build sagemaker inference image + id: build-sagemaker + run: | + VERSION="${{ matrix.version }}" + source docker/tensorflow/inference/${VERSION}/versions-cuda.env + # ECR tag uses TF Serving binary version (TF_SERVING_VERSION from + # versions-cuda.env) — mirrors master TF 2.19 inference repo naming. + CI_IMAGE_URI="${{ vars.CI_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/ci:tensorflow-inference-${TF_SERVING_VERSION}-gpu-py312-cu129-sagemaker-${{ matrix.version }}-pr-${{ github.event.pull_request.number }}" + + # Derive label values to match check_labels.py expectations + FRAMEWORK_LABEL=$(echo "${{ matrix.framework }}" | tr '_' '-') + FWK_VER_LABEL=$(echo "${{ matrix.framework_version }}" | tr '.' '-') + CUDA_LABEL="${{ matrix.cuda_version }}" + OS_LABEL=$(echo "${{ matrix.os_version }}" | tr '.' '-') + + docker buildx build --progress plain \ + --build-arg FRAMEWORK=${{ matrix.framework }} \ + --build-arg CUDA_VERSION=${CUDA_VERSION} \ + --build-arg PYTHON_VERSION=${PYTHON_VERSION} \ + --build-arg TF_SERVING_VERSION=${TF_SERVING_VERSION} \ + --build-arg TFS_SHORT_VERSION=${VERSION} \ + --build-arg NGINX_VERSION=${NGINX_VERSION} \ + --build-arg NJS_VERSION=${NJS_VERSION} \ + --build-arg DLC_MAJOR_VERSION=${DLC_MAJOR_VERSION} \ + --build-arg DLC_MINOR_VERSION=${DLC_MINOR_VERSION} \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.framework.${FRAMEWORK_LABEL}.${FWK_VER_LABEL}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.device.gpu.${CUDA_LABEL}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.job.${{ matrix.container_type }}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.arch.${{ matrix.arch_type }}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.os.${OS_LABEL}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.python.${{ matrix.python_version }}=true" \ + --label "com.amazonaws.ml.engines.sagemaker.dlc.contributor.${{ matrix.contributor }}=true" \ + --cache-to=type=inline \ + --cache-from=type=registry,ref=${CI_IMAGE_URI} \ + --tag ${CI_IMAGE_URI} \ + --push \ + --target sagemaker \ + -f docker/tensorflow/inference/${VERSION}/Dockerfile.cuda . + + echo "image-uri=${CI_IMAGE_URI}" >> $GITHUB_OUTPUT + + # ============================================================ + # Sanity tests (labels, filesystem, OSS compliance) + # ============================================================ + sanity-test: + needs: [check-changes, build-image] + if: | + always() && !failure() && !cancelled() && + (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.sanity-test-change == 'true') + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + uses: ./.github/workflows/reusable-sanity-tests.yml + with: + image-uri: ${{ needs.build-image.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:tensorflow-inference-{2}-gpu-py312-cu129-sagemaker-{3}-pr-{4}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.framework_version, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} + aws-account-id: ${{ needs.build-image.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + framework: ${{ matrix.framework }} + framework-version: ${{ matrix.framework_version }} + python-version: ${{ matrix.python_version }} + cuda-version: ${{ matrix.cuda_version }} + os-version: ${{ matrix.os_version }} + customer-type: ${{ matrix.customer_type }} + arch-type: ${{ matrix.arch_type }} + device-type: ${{ matrix.device_type }} + contributor: ${{ matrix.contributor }} + container-type: ${{ matrix.container_type }} + + # ============================================================ + # Security tests (ECR scan, CVE allowlist) + # ============================================================ + security-test: + needs: [check-changes, build-image] + if: success() + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + uses: ./.github/workflows/reusable-security-tests.yml + with: + image-uri: ${{ needs.build-image.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:tensorflow-inference-{2}-gpu-py312-cu129-sagemaker-{3}-pr-{4}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.framework_version, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} + aws-account-id: ${{ needs.build-image.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + framework: ${{ matrix.framework }} + framework-version: ${{ matrix.framework_version }} + + # ============================================================ + # Telemetry tests (opt-out, environment variables) + # ============================================================ + telemetry-test: + needs: [check-changes, build-image] + if: | + always() && !failure() && !cancelled() && + (needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.telemetry-test-change == 'true') + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + concurrency: + group: ${{ github.workflow }}-telemetry-test-${{ matrix.version }}-${{ github.event.pull_request.number }} + cancel-in-progress: false + uses: ./.github/workflows/reusable-telemetry-tests.yml + with: + image-uri: ${{ needs.build-image.result == 'success' && format('{0}.dkr.ecr.{1}.amazonaws.com/ci:tensorflow-inference-{2}-gpu-py312-cu129-sagemaker-{3}-pr-{4}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.framework_version, matrix.version, github.event.pull_request.number) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', vars.PROD_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.prod_image) }} + aws-account-id: ${{ needs.build-image.result == 'success' && vars.CI_AWS_ACCOUNT_ID || vars.PROD_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + framework: ${{ matrix.framework }} + framework-version: ${{ matrix.framework_version }} + container-type: ${{ matrix.container_type }} + + # TODO: unit-test job — training has one targeting test/tensorflow/unit/. + # Inference equivalent (e.g., test/tensorflow/integration/inference/unit/ + # exercising tfs_utils / multi_model_utils in-container) does not exist + # yet. Add in Phase 5 if a unit suite is created. + + # TODO: MME-specific test — image config sets + # com.amazonaws.sagemaker.capabilities.multi-models=true. Phase 5 should + # add an MME endpoint smoke test (multi-model load via tensorflowServing.js) + # once the integration test scaffolding is in place. + + # ============================================================ + # SageMaker integration tests (launch real SM endpoints on GPU) + # ============================================================ + # Phase 5 will populate test/tensorflow/integration/inference/. Until then this + # job will run pytest on an empty dir (pytest exits 5 = no tests collected; we + # treat that as success here so the workflow can land before Phase 5). + sagemaker-test: + needs: [check-changes, build-image, sanity-test, security-test] + if: success() + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + strategy: + matrix: + include: ${{ fromJson(needs.check-changes.outputs.configs) }} + fail-fast: false + concurrency: + group: ${{ github.workflow }}-sagemaker-${{ matrix.version }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: ECR login + uses: ./.github/actions/ecr-authenticate + with: + aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Install test dependencies + run: | + # TODO: Phase 5 — replace with `pip install -r + # test/tensorflow/integration/inference/requirements.txt` once + # Phase 5 supplies that file. + pip install pytest sagemaker boto3 + + - name: Run SageMaker inference tests + env: + PYTHONPATH: ${{ github.workspace }}/test + TEST_IMAGE_URI: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/ci:tensorflow-inference-{2}-gpu-py312-cu129-sagemaker-{3}-pr-{4}', vars.CI_AWS_ACCOUNT_ID, vars.AWS_REGION, matrix.framework_version, matrix.version, github.event.pull_request.number) }} + SM_ROLE_ARN: arn:aws:iam::${{ vars.CI_AWS_ACCOUNT_ID }}:role/SageMakerRole + run: | + # pytest exit 5 (no tests collected) is acceptable until Phase 5 adds + # tests under test/tensorflow/integration/inference/. + set +e + pytest test/tensorflow/integration/inference/ -v + rc=$? + set -e + if [ "$rc" != "0" ] && [ "$rc" != "5" ]; then + exit $rc + fi diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cpu b/docker/tensorflow/inference/2.20/Dockerfile.cpu new file mode 100644 index 000000000000..27566aecc723 --- /dev/null +++ b/docker/tensorflow/inference/2.20/Dockerfile.cpu @@ -0,0 +1,227 @@ +# ============================================================================ +# TensorFlow Serving 2.20 Inference DLC — Amazon Linux 2023 (CPU) +# Multi-stage build: +# builder-base ──┬── builder-oss (OSS license generation — isolated) +# └── builder-njs (compile nginx + njs dynamic module) +# runtime-base ──── sagemaker (SageMaker inference, MME-capable) +# +# All version defaults mirror docker/tensorflow/inference/2.20/versions-cpu.env. +# Workflows source versions-cpu.env and pass ARGs via --build-arg — single +# source of truth. +# +# Same structure as Dockerfile.cuda; deltas: +# - amazonlinux:2023 base (no CUDA). +# - tensorflow/serving:${TF_SERVING_VERSION}-devel (no -gpu) for binary copy. +# - tensorflow-serving-api (no -gpu suffix) installed via --no-deps. +# ============================================================================ + +# ── Global ARGs (available to all stages) ─────────────────────────────────── +ARG DLC_MAJOR_VERSION=1 +ARG DLC_MINOR_VERSION=0 +ARG PYTHON_VERSION=3.12 +ARG TF_SERVING_VERSION=2.20.0 +ARG TFS_SHORT_VERSION=2.20 +ARG NGINX_VERSION=1.30.2 +ARG NJS_VERSION=0.9.9 + + +# ── Stage: build_image (TFS upstream binary source) ────────────────────────── +FROM tensorflow/serving:${TF_SERVING_VERSION}-devel AS build_image + + +# ── Stage: builder-base (Python venv + lockfile deps) ─────────────────────── +FROM amazonlinux:2023 AS builder-base +ARG PYTHON_VERSION + +RUN dnf install -y --allowerasing \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \ + gcc gcc-c++ make cmake git openssl-devel ninja-build \ + tar xz curl wget \ + && dnf clean all + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +ENV UV_PROJECT_ENVIRONMENT="/opt/venv" +RUN python${PYTHON_VERSION} -m venv /opt/venv +ENV PATH="/opt/venv/bin:${PATH}" + +COPY docker/tensorflow/inference/2.20/cpu/pyproject.toml docker/tensorflow/inference/2.20/cpu/uv.lock /tmp/build/ +WORKDIR /tmp/build +RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --inexact + + +# ── Stage: builder-oss (generates license files in isolation) ─────────────── +FROM amazonlinux:2023 AS builder-oss +ARG PYTHON_VERSION +RUN dnf install -y --allowerasing python${PYTHON_VERSION} curl && dnf clean all +COPY --from=builder-base /opt/venv /opt/venv +COPY scripts/common/setup_oss_compliance.sh /tmp/setup_oss_compliance.sh +RUN PATH="/opt/venv/bin:${PATH}" bash /tmp/setup_oss_compliance.sh python${PYTHON_VERSION} \ + && touch /root/THIRD_PARTY_SOURCE_CODE_URLS + + +# ── Stage: builder-njs (compile nginx + njs dynamic module from source) ───── +# Output: /out/modules/ngx_http_js_module.so + /out/modules/ngx_stream_js_module.so +# Identical to the cuda variant's builder-njs — njs build is CPU-only and +# produces an architecture-matched .so that loads into the AL2023 nginx. +FROM amazonlinux:2023 AS builder-njs +ARG NGINX_VERSION +ARG NJS_VERSION + +RUN dnf install -y --allowerasing \ + gcc gcc-c++ make \ + pcre-devel pcre2-devel zlib-devel openssl-devel \ + curl tar xz \ + && dnf clean all + +WORKDIR /tmp/njs-build + +RUN curl -fsSLO "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ + && tar xzf nginx-${NGINX_VERSION}.tar.gz \ + && curl -fsSL "https://github.com/nginx/njs/archive/refs/tags/${NJS_VERSION}.tar.gz" -o njs-${NJS_VERSION}.tar.gz \ + && tar xzf njs-${NJS_VERSION}.tar.gz + +RUN cd nginx-${NGINX_VERSION} \ + && ./configure \ + --with-compat \ + --add-dynamic-module=/tmp/njs-build/njs-${NJS_VERSION}/nginx \ + && make -j$(nproc) modules \ + && mkdir -p /out/modules \ + && cp objs/ngx_http_js_module.so /out/modules/ \ + && cp objs/ngx_stream_js_module.so /out/modules/ + + +# ── Stage: runtime-base (shared base for output stages) ───────────────────── +FROM amazonlinux:2023 AS runtime-base +ARG PYTHON_VERSION +ARG TF_SERVING_VERSION +ARG TFS_SHORT_VERSION +ARG DLC_MAJOR_VERSION +ARG DLC_MINOR_VERSION + +# Labels live on runtime-base so all output stages inherit them. +LABEL maintainer="Amazon AI" +LABEL dlc_major_version="${DLC_MAJOR_VERSION}" +LABEL dlc_minor_version="${DLC_MINOR_VERSION}" +LABEL framework="tensorflow" +LABEL framework_version="${TF_SERVING_VERSION}" + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + DLC_CONTAINER_TYPE=inference \ + MODEL_BASE_PATH=/models \ + MODEL_NAME=model + +# Runtime system deps via dnf — equivalent of master TF 2.19 CPU apt-get block. +RUN dnf install -y --allowerasing \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel \ + nginx \ + gcc gcc-c++ make git \ + tar gzip xz which findutils util-linux \ + libpng-devel freetype-devel zlib-devel \ + openssl unzip jq curl wget \ + && dnf clean all + +# Copy venv from builder-base +COPY --from=builder-base /opt/venv /opt/venv + +# Copy njs dynamic modules +COPY --from=builder-njs /out/modules/ngx_http_js_module.so /usr/lib64/nginx/modules/ +COPY --from=builder-njs /out/modules/ngx_stream_js_module.so /usr/lib64/nginx/modules/ + +# TF Serving binary +COPY --from=build_image /usr/local/bin/tensorflow_model_server /usr/local/bin/tensorflow_model_server + +# python symlink — some TF tooling expects /usr/local/bin/python. +RUN ln -sf $(which python${PYTHON_VERSION}) /usr/local/bin/python \ + && ln -sf $(which pip3) /usr/local/bin/pip + +ENV PATH="/opt/venv/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" + +# Models dir +RUN mkdir -p ${MODEL_BASE_PATH} + +# License file (S3 bucket pre-provisioned for tensorflow-2.20) +RUN curl -fsSLo /license.txt "https://aws-dlc-licenses.s3.amazonaws.com/tensorflow-${TFS_SHORT_VERSION}/license.txt" \ + || echo "tensorflow-${TFS_SHORT_VERSION}/license.txt not yet provisioned in aws-dlc-licenses bucket; placeholder emitted" >/license.txt + +WORKDIR / + + +# ── Stage: sagemaker (SageMaker inference output, MME-capable) ────────────── +FROM runtime-base AS sagemaker +ARG TF_SERVING_VERSION +ARG TFS_SHORT_VERSION +ARG PYTHON_VERSION + +# SageMaker inference labels — accept-bind-to-port allows pipeline use of +# SAGEMAKER_BIND_TO_PORT; multi-models=true enables MME loading. +LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true +LABEL com.amazonaws.sagemaker.capabilities.multi-models=true + +ENV SAGEMAKER_TFS_VERSION="${TFS_SHORT_VERSION}" +ENV PATH="$PATH:/sagemaker" + +# SageMaker BYOC paths +RUN mkdir -p /opt/ml/input/data /opt/ml/model /opt/ml/output /opt/ml/code + +# tensorflow-serving-api — installed inline with --no-deps; see locked +# decision Q1 in cpu/pyproject.toml header. +RUN /opt/venv/bin/uv pip install --no-deps --no-cache "tensorflow-serving-api==${TF_SERVING_VERSION}" 2>/dev/null \ + || /opt/venv/bin/pip install --no-deps --no-cache-dir "tensorflow-serving-api==${TF_SERVING_VERSION}" + +# SageMaker handler artifacts (TFS toolkit ported in Phase 3 from master TF 2.19 +# build_artifacts/sagemaker/: serve/serve.py, python_service.py, tfs_utils.py, +# multi_model_utils.py, tensorflowServing.js, nginx.conf.template). +COPY scripts/tensorflow/inference/sagemaker /sagemaker + +# Telemetry +COPY scripts/telemetry/deep_learning_container.py /usr/local/bin/deep_learning_container.py +COPY scripts/telemetry/bash_telemetry.sh.template /tmp/bash_telemetry.sh.template +ARG FRAMEWORK="tensorflow" +ARG CONTAINER_TYPE="inference" +RUN chmod +x /usr/local/bin/deep_learning_container.py \ + && sed -e "s/{{FRAMEWORK}}/${FRAMEWORK}/g" \ + -e "s/{{FRAMEWORK_VERSION}}/${TF_SERVING_VERSION}/g" \ + -e "s/{{CONTAINER_TYPE}}/${CONTAINER_TYPE}/g" \ + /tmp/bash_telemetry.sh.template >/usr/local/bin/bash_telemetry.sh \ + && chmod +x /usr/local/bin/bash_telemetry.sh \ + && rm /tmp/bash_telemetry.sh.template + +# Security patch — run after all installers so every OS package is covered. +RUN dnf upgrade -y --security --releasever latest \ + && dnf upgrade -y libcurl libcurl-minimal --refresh \ + && dnf clean all + +# Telemetry bashrc hook — MUST be after `dnf upgrade --security`. +RUN echo 'source /usr/local/bin/bash_telemetry.sh' >>/etc/bashrc \ + && echo 'source /usr/local/bin/bash_telemetry.sh' >>/root/.bashrc + +# OSS compliance +COPY --from=builder-oss /root/THIRD_PARTY_SOURCE_CODE_URLS /root/THIRD_PARTY_SOURCE_CODE_URLS +COPY --from=builder-oss /root/PYTHON_PACKAGES_LICENSES /root/PYTHON_PACKAGES_LICENSES +COPY --from=builder-oss /root/LINUX_PACKAGES_LICENSES /root/LINUX_PACKAGES_LICENSES +COPY --from=builder-oss /root/BUILD_FROM_SOURCE_PACKAGES_LICENCES /root/BUILD_FROM_SOURCE_PACKAGES_LICENCES +COPY --from=builder-oss /usr/local/bin/testOSSCompliance /usr/local/bin/testOSSCompliance + +# SM entrypoint — start_cuda_compat.sh is a safe no-op on CPU (the compat .so +# does not exist), so we ship it for uniformity with the GPU image's entrypoint +# contract. PR #6107 training Dockerfile.cpu makes the same simplification. +COPY scripts/tensorflow/inference/dockerd_entrypoint.sh /usr/local/bin/dockerd_entrypoint.sh +COPY scripts/tensorflow/inference/tf_serving_entrypoint.sh /usr/local/bin/tf_serving_entrypoint.sh +COPY scripts/common/start_cuda_compat.sh /usr/local/bin/start_cuda_compat.sh +RUN chmod +x /usr/local/bin/dockerd_entrypoint.sh \ + /usr/local/bin/tf_serving_entrypoint.sh \ + /usr/local/bin/start_cuda_compat.sh + +RUN rm -rf /tmp/* /root/.cache + +# Expose ports — TF Serving native gRPC (8500) and REST (8501). +EXPOSE 8500 8501 + +ENTRYPOINT ["bash", "-m", "/usr/local/bin/dockerd_entrypoint.sh"] +CMD ["/usr/local/bin/tf_serving_entrypoint.sh"] diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cuda b/docker/tensorflow/inference/2.20/Dockerfile.cuda new file mode 100644 index 000000000000..c24e9d916f1b --- /dev/null +++ b/docker/tensorflow/inference/2.20/Dockerfile.cuda @@ -0,0 +1,265 @@ +# ============================================================================ +# TensorFlow Serving 2.20 Inference DLC — Amazon Linux 2023 (CUDA 12.9.1) +# Multi-stage build: +# builder-base ──┬── builder-oss (OSS license generation — isolated) +# └── builder-njs (compile nginx + njs dynamic module) +# runtime-base ──── sagemaker (SageMaker inference, MME-capable) +# +# All version defaults mirror docker/tensorflow/inference/2.20/versions-cuda.env. +# Workflows source versions-cuda.env and pass ARGs via --build-arg — single +# source of truth. +# +# This image is SageMaker-only (no `runtime` stage). It is patterned after +# PR #6107 (TF 2.21 training v2) for stage layout, with two key deltas: +# 1. No EFA / NCCL / OpenMPI — inference does not run multi-node MPI. +# 2. NEW `builder-njs` stage that compiles nginx-mod-njs from source because +# AL2023's core repo does not ship nginx-module-njs (master TF 2.19 used +# Ubuntu's nginx.org apt repo, which does not exist for AL2023). +# ============================================================================ + +# ── Global ARGs (available to all stages) ─────────────────────────────────── +ARG DLC_MAJOR_VERSION=1 +ARG DLC_MINOR_VERSION=0 +ARG CUDA_VERSION=12.9.1 +ARG PYTHON_VERSION=3.12 +ARG TF_SERVING_VERSION=2.20.0 +ARG TFS_SHORT_VERSION=2.20 +ARG NGINX_VERSION=1.30.2 +ARG NJS_VERSION=0.9.9 + + +# ── Stage: build_image (TFS upstream binary source) ────────────────────────── +# Pinned by tag; provides /usr/local/bin/tensorflow_model_server compiled by +# Google against CUDA 12.x. CUDA minor-version forward compatibility lets the +# 12.x-built binary run on the CUDA 12.9.1 runtime base below. +FROM tensorflow/serving:${TF_SERVING_VERSION}-devel-gpu AS build_image + + +# ── Stage: builder-base (Python venv + lockfile deps) ─────────────────────── +FROM nvidia/cuda:${CUDA_VERSION}-devel-amzn2023 AS builder-base +ARG PYTHON_VERSION + +RUN dnf install -y --allowerasing \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \ + gcc gcc-c++ make cmake git openssl-devel ninja-build \ + tar xz curl \ + && dnf clean all + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +ENV UV_PROJECT_ENVIRONMENT="/opt/venv" +RUN python${PYTHON_VERSION} -m venv /opt/venv +ENV PATH="/opt/venv/bin:${PATH}" + +COPY docker/tensorflow/inference/2.20/cuda/pyproject.toml docker/tensorflow/inference/2.20/cuda/uv.lock /tmp/build/ +WORKDIR /tmp/build +RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --inexact + + +# ── Stage: builder-oss (generates license files in isolation) ─────────────── +FROM nvidia/cuda:${CUDA_VERSION}-runtime-amzn2023 AS builder-oss +ARG PYTHON_VERSION +RUN dnf install -y --allowerasing python${PYTHON_VERSION} curl && dnf clean all +COPY --from=builder-base /opt/venv /opt/venv +COPY scripts/common/setup_oss_compliance.sh /tmp/setup_oss_compliance.sh +RUN PATH="/opt/venv/bin:${PATH}" bash /tmp/setup_oss_compliance.sh python${PYTHON_VERSION} \ + && touch /root/THIRD_PARTY_SOURCE_CODE_URLS + + +# ── Stage: builder-njs (compile nginx + njs dynamic module from source) ───── +# Output: /out/modules/ngx_http_js_module.so + /out/modules/ngx_stream_js_module.so +# We use the CUDA -devel base for build toolchain consistency with builder-base; +# this stage produces a CPU-only .so artifact that is identical for cuda and cpu +# variants (kept in this Dockerfile as well as Dockerfile.cpu for self-contained +# multi-stage builds). +FROM nvidia/cuda:${CUDA_VERSION}-devel-amzn2023 AS builder-njs +ARG NGINX_VERSION +ARG NJS_VERSION + +RUN dnf install -y --allowerasing \ + gcc gcc-c++ make \ + pcre-devel pcre2-devel zlib-devel openssl-devel \ + curl tar xz \ + && dnf clean all + +WORKDIR /tmp/njs-build + +# Fetch nginx + njs sources, both with checksum-validated release tarballs +# from the upstream maintainer. +RUN curl -fsSLO "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ + && tar xzf nginx-${NGINX_VERSION}.tar.gz \ + && curl -fsSL "https://github.com/nginx/njs/archive/refs/tags/${NJS_VERSION}.tar.gz" -o njs-${NJS_VERSION}.tar.gz \ + && tar xzf njs-${NJS_VERSION}.tar.gz + +# Configure nginx with the njs dynamic module, then build only the modules. +# --with-compat is REQUIRED so the resulting .so loads into a stock nginx +# binary (we install nginx via dnf in runtime-base; the .so must be ABI-compat). +RUN cd nginx-${NGINX_VERSION} \ + && ./configure \ + --with-compat \ + --add-dynamic-module=/tmp/njs-build/njs-${NJS_VERSION}/nginx \ + && make -j$(nproc) modules \ + && mkdir -p /out/modules \ + && cp objs/ngx_http_js_module.so /out/modules/ \ + && cp objs/ngx_stream_js_module.so /out/modules/ + + +# ── Stage: runtime-base (shared base for output stages) ───────────────────── +FROM nvidia/cuda:${CUDA_VERSION}-runtime-amzn2023 AS runtime-base +ARG CUDA_VERSION +ARG PYTHON_VERSION +ARG TF_SERVING_VERSION +ARG TFS_SHORT_VERSION +ARG DLC_MAJOR_VERSION +ARG DLC_MINOR_VERSION + +# Labels live on runtime-base so all output stages inherit them. +LABEL maintainer="Amazon AI" +LABEL dlc_major_version="${DLC_MAJOR_VERSION}" +LABEL dlc_minor_version="${DLC_MINOR_VERSION}" +LABEL framework="tensorflow" +LABEL framework_version="${TF_SERVING_VERSION}" + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + DLC_CONTAINER_TYPE=inference \ + CUDA_HOME=/usr/local/cuda \ + MODEL_BASE_PATH=/models \ + MODEL_NAME=model + +# Runtime system deps via dnf — equivalent of master TF 2.19 GPU apt-get block, +# minus the training-specific packages (NCCL/EFA/OpenMPI). nginx is installed +# from AL2023 core (1.24.x); the njs .so we built in builder-njs is dropped +# into /usr/lib64/nginx/modules. tar/gzip/which/findutils/util-linux are +# baseline shell utilities the SM handler scripts and TFS launcher rely on. +RUN dnf install -y --allowerasing \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel \ + nginx \ + gcc gcc-c++ make git \ + tar gzip xz which findutils util-linux \ + libpng-devel freetype-devel zlib-devel \ + openssl unzip jq curl wget \ + && dnf clean all + +# Copy venv from builder-base +COPY --from=builder-base /opt/venv /opt/venv + +# Copy njs dynamic module(s) into the system nginx modules dir. The package +# nginx (1.24.x) on AL2023 honors `load_module modules/ngx_http_js_module.so` +# in its conf, since we built with --with-compat against matching nginx headers. +COPY --from=builder-njs /out/modules/ngx_http_js_module.so /usr/lib64/nginx/modules/ +COPY --from=builder-njs /out/modules/ngx_stream_js_module.so /usr/lib64/nginx/modules/ + +# TF Serving binary — copied from the official upstream devel image. +COPY --from=build_image /usr/local/bin/tensorflow_model_server /usr/local/bin/tensorflow_model_server + +# python symlink — some TF tooling expects /usr/local/bin/python. +RUN ln -sf $(which python${PYTHON_VERSION}) /usr/local/bin/python \ + && ln -sf $(which pip3) /usr/local/bin/pip + +ENV PATH="/opt/venv/bin:/usr/local/cuda/bin:${PATH}" +ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/lib:${LD_LIBRARY_PATH}" + +# Models dir +RUN mkdir -p ${MODEL_BASE_PATH} + +# License file (S3 bucket pre-provisioned for tensorflow-2.20) +RUN curl -fsSLo /license.txt "https://aws-dlc-licenses.s3.amazonaws.com/tensorflow-${TFS_SHORT_VERSION}/license.txt" \ + || echo "tensorflow-${TFS_SHORT_VERSION}/license.txt not yet provisioned in aws-dlc-licenses bucket; placeholder emitted" >/license.txt + +WORKDIR / + + +# ── Stage: sagemaker (SageMaker inference output, MME-capable) ────────────── +FROM runtime-base AS sagemaker +ARG TF_SERVING_VERSION +ARG TFS_SHORT_VERSION +ARG PYTHON_VERSION +# (Labels are inherited from runtime-base — do not redeclare maintainer/framework.) + +# SageMaker inference labels — accept-bind-to-port allows pipeline use of +# SAGEMAKER_BIND_TO_PORT; multi-models=true enables MME loading via +# tensorflowServing.js (handler scripts ported in Phase 3). +LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true +LABEL com.amazonaws.sagemaker.capabilities.multi-models=true +LABEL com.amazonaws.sagemaker.inference.cuda.verified_versions=12.9 + +ENV SAGEMAKER_TFS_VERSION="${TFS_SHORT_VERSION}" +ENV PATH="$PATH:/sagemaker" + +# SageMaker BYOC paths +RUN mkdir -p /opt/ml/input/data /opt/ml/model /opt/ml/output /opt/ml/code + +# tensorflow-serving-api-gpu — installed inline with --no-deps so the ~600 MB +# `tensorflow` framework wheel is NOT pulled into an inference image. See +# locked decision Q1 in cuda/pyproject.toml header. The transitive runtime +# needs (numpy / protobuf / grpcio) are already in the venv from builder-base. +RUN /opt/venv/bin/uv pip install --no-deps --no-cache "tensorflow-serving-api-gpu==${TF_SERVING_VERSION}" 2>/dev/null \ + || /opt/venv/bin/pip install --no-deps --no-cache-dir "tensorflow-serving-api-gpu==${TF_SERVING_VERSION}" + +# SageMaker handler artifacts (TFS toolkit ported in Phase 3 from master TF 2.19 +# build_artifacts/sagemaker/: serve/serve.py, python_service.py, tfs_utils.py, +# multi_model_utils.py, tensorflowServing.js, nginx.conf.template). +COPY scripts/tensorflow/inference/sagemaker /sagemaker + +# Telemetry (PT/TF v2 cross-framework pattern) +COPY scripts/telemetry/deep_learning_container.py /usr/local/bin/deep_learning_container.py +COPY scripts/telemetry/bash_telemetry.sh.template /tmp/bash_telemetry.sh.template +ARG FRAMEWORK="tensorflow" +ARG CONTAINER_TYPE="inference" +RUN chmod +x /usr/local/bin/deep_learning_container.py \ + && sed -e "s/{{FRAMEWORK}}/${FRAMEWORK}/g" \ + -e "s/{{FRAMEWORK_VERSION}}/${TF_SERVING_VERSION}/g" \ + -e "s/{{CONTAINER_TYPE}}/${CONTAINER_TYPE}/g" \ + /tmp/bash_telemetry.sh.template >/usr/local/bin/bash_telemetry.sh \ + && chmod +x /usr/local/bin/bash_telemetry.sh \ + && rm /tmp/bash_telemetry.sh.template + +# Security patch — run after all installers so every OS package is covered. +# Force libcurl refresh — AL2023's --security filter sometimes misses libcurl +# CVE patches that ship as regular updates rather than security advisories. +RUN dnf upgrade -y --security --releasever latest \ + && dnf upgrade -y cuda-compat-* \ + && dnf upgrade -y libcurl libcurl-minimal --refresh \ + && dnf clean all + +# Telemetry bashrc hook — MUST be after `dnf upgrade --security` because dnf +# may replace /etc/bashrc during the upgrade, silently wiping out any `source` +# line added earlier (PT main pattern). +RUN echo 'source /usr/local/bin/bash_telemetry.sh' >>/etc/bashrc \ + && echo 'source /usr/local/bin/bash_telemetry.sh' >>/root/.bashrc + +# OSS compliance (copy artifacts from builder-oss) +COPY --from=builder-oss /root/THIRD_PARTY_SOURCE_CODE_URLS /root/THIRD_PARTY_SOURCE_CODE_URLS +COPY --from=builder-oss /root/PYTHON_PACKAGES_LICENSES /root/PYTHON_PACKAGES_LICENSES +COPY --from=builder-oss /root/LINUX_PACKAGES_LICENSES /root/LINUX_PACKAGES_LICENSES +COPY --from=builder-oss /root/BUILD_FROM_SOURCE_PACKAGES_LICENCES /root/BUILD_FROM_SOURCE_PACKAGES_LICENCES +COPY --from=builder-oss /usr/local/bin/testOSSCompliance /usr/local/bin/testOSSCompliance + +# SM entrypoint — dockerd_entrypoint.sh + tf_serving_entrypoint.sh ported in +# Phase 3 from master TF 2.19 build_artifacts/. ENTRYPOINT below invokes +# dockerd_entrypoint.sh which (1) runs telemetry, (2) sources start_cuda_compat +# when the installed tensorflow-serving-api wheel is the -gpu variant, then +# (3) `eval`s "$@" — so SM's `docker run … serve` resolves /sagemaker/serve +# (PATH includes /sagemaker), and a bare `docker run` falls through to the +# default CMD (tf_serving_entrypoint.sh — non-SM smoke path). +COPY scripts/tensorflow/inference/dockerd_entrypoint.sh /usr/local/bin/dockerd_entrypoint.sh +COPY scripts/tensorflow/inference/tf_serving_entrypoint.sh /usr/local/bin/tf_serving_entrypoint.sh +COPY scripts/common/start_cuda_compat.sh /usr/local/bin/start_cuda_compat.sh +RUN chmod +x /usr/local/bin/dockerd_entrypoint.sh \ + /usr/local/bin/tf_serving_entrypoint.sh \ + /usr/local/bin/start_cuda_compat.sh + +RUN rm -rf /tmp/* /root/.cache + +# Expose ports — TF Serving native gRPC (8500) and REST (8501). +# Master TF 2.19 SM image exposes the same pair; the SM frontend nginx (port +# 8080 from the SM contract) is set up at runtime by the handler scripts and +# does not need an EXPOSE here. +EXPOSE 8500 8501 + +ENTRYPOINT ["bash", "-m", "/usr/local/bin/dockerd_entrypoint.sh"] +CMD ["/usr/local/bin/tf_serving_entrypoint.sh"] diff --git a/docker/tensorflow/inference/2.20/cpu/pyproject.toml b/docker/tensorflow/inference/2.20/cpu/pyproject.toml new file mode 100644 index 000000000000..ea7afd5a9472 --- /dev/null +++ b/docker/tensorflow/inference/2.20/cpu/pyproject.toml @@ -0,0 +1,67 @@ +[project] +name = "tensorflow-serving-inference-dlc-cpu" +version = "2.20.0" +requires-python = ">=3.12,<3.13" +# ────────────────────────────────────────────────────────────────────────────── +# TF Serving 2.20 CPU AL2023 inference DLC — runtime-base dependencies +# +# The TF Serving binary itself is NOT installed via pip — it is COPYed out of +# tensorflow/serving:2.20.0-devel in the Dockerfile. The `tensorflow-serving-api` +# wheel is the matched gRPC stub (used by python_service.py to call the local +# serving binary on port 8500). +# +# IMPORTANT — locked decision Q1 (handoff): +# `tensorflow-serving-api==2.20.0` is INTENTIONALLY OMITTED from this +# `dependencies` list. The PyPI wheel declares a hard `tensorflow` runtime +# dep (≈600 MB framework wheel) which we do not want in an inference image. +# uv's resolver does not support per-package `--no-deps`, so the wheel is +# instead installed inline in the Dockerfile via: +# uv pip install --no-deps tensorflow-serving-api==2.20.0 +# The transitive runtime needs of tfs-api (numpy, protobuf, grpcio) are +# declared explicitly below so a coherent venv is locked. OSS license +# tracking still picks up tfs-api at compliance-scan time because the +# wheel is physically present in the image. +# +# Source for this dep list: master TF 2.19 inference Dockerfile.cpu +# (RUN pip install blocks), translated to uv/pyproject form per training +# PR #6107 pattern. +# ────────────────────────────────────────────────────────────────────────────── +dependencies = [ + # ── tfs-api transitive runtime needs (explicit pins; see header note) ───── + "numpy==1.26.4", + "protobuf>=3.20,<7", + "grpcio>=1.0", + + # ── AWS / network / common utilities (carried over from master 2.19) ───── + "awscli<2", + "boto3", + "botocore", + "requests", + "packaging", + "gevent", + + # ── Falcon-based SM handler stack ───────────────────────────────────────── + # Both pins user-locked per handoff §16. + "falcon==3.1.0", + "gunicorn>=22.0.0", + + # ── Compat / security pins inherited from training PR #6107 ─────────────── + "cython<3", + "aiohttp>=3.14.0", + "urllib3", +] + +[project.optional-dependencies] +sagemaker = [ + # See cuda/pyproject.toml for rationale on each entry; CPU mirrors CUDA + # for SM-stage deps. `sagemaker-tensorflow-serving-container` is NOT on + # PyPI; the `sagemaker` SDK is intentionally omitted to avoid pulling + # torch + nvidia-* into an inference image. Handler scripts use boto3 + # directly (matches master TF 2.19 inference behavior). + "pandas", + "scikit-learn", + "cloudpickle", +] + +[tool.uv] +environments = ["sys_platform == 'linux' and platform_machine == 'x86_64'"] diff --git a/docker/tensorflow/inference/2.20/cpu/uv.lock b/docker/tensorflow/inference/2.20/cpu/uv.lock new file mode 100644 index 000000000000..5325aa6d44ed --- /dev/null +++ b/docker/tensorflow/inference/2.20/cpu/uv.lock @@ -0,0 +1,550 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", +] +supported-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/c6/61a2d7b7572279226bb2e7f61d7a19ca7c90da0329c93fa0d560cbf288d8/aiohappyeyeballs-2.6.2.tar.gz", hash = "sha256:e202810ee718bd01fc6ef49e8ea53d023d5cb6b581076d7925aa499fa55dbe64", size = 22591, upload-time = "2026-05-20T15:12:24.631Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/fc/a7bf5b6e4e617b45f90f2d9d2a68519c249c81dd4fc2658c7a2a61c4f4b7/aiohappyeyeballs-2.6.2-py3-none-any.whl", hash = "sha256:4708045e2d7a6c6bdf8aafa8ed39649eaf926a4543b54560659129e3365953c4", size = 15062, upload-time = "2026-05-20T15:12:23.328Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "aiosignal", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "attrs", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "frozenlist", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "multidict", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "propcache", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "yarl", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "awscli" +version = "1.45.28" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "colorama", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "docutils", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pyyaml", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "rsa", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "s3transfer", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/32/68ff637efdba82e49fd12ddc068fc19edd7f6b4b3ee5bb5976379daaeb25/awscli-1.45.28.tar.gz", hash = "sha256:50e788ed5dc17eb916fd1b9c74d581e4b6a86147c0e60b8b2d25b60ceb32d458", size = 1896334, upload-time = "2026-06-11T19:28:58.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/41/d9a71dfafd7670e7a79fa4a0d9f073ebbecd33ff4d498f0715d39e4a4464/awscli-1.45.28-py3-none-any.whl", hash = "sha256:783a45041e364c1be9640f730cbcc18313767c592f555477cf0911b1c9511455", size = 4648370, upload-time = "2026-06-11T19:28:54.876Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.28" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "jmespath", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "s3transfer", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4e/f2/a976b2a81d8dc7ff675f4b614367a185727061130184a28da0f53f446b97/boto3-1.43.28.tar.gz", hash = "sha256:8391fdcc4d8e1d4e0bf96575a7e5610964a4d401dafa4dccb0a5bade8dd3fbb0", size = 113202, upload-time = "2026-06-11T19:29:01.464Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/a5/47db150ea6380f11569b87d3ad064e3c929e5abe227a549d472fab6f5f3a/boto3-1.43.28-py3-none-any.whl", hash = "sha256:4fe6df2163aea02b561eca0d685e2f41a059d71f03721a3e79c3b522e79a3b56", size = 140536, upload-time = "2026-06-11T19:29:00.143Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.28" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "python-dateutil", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "urllib3", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/dc/1b01808003f88f8a328732c979f20cb0456791048b4440fc4abcae08c1a0/botocore-1.43.28.tar.gz", hash = "sha256:9bbad501a68e4ffdbeff76a382507f5d7827abc316f34a218ab76f5293e6c78d", size = 15503514, upload-time = "2026-06-11T19:28:50.989Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/8c/14916c353ce8a29d14cf6308c2bef842bbb25dde6defc620e26e28063331/botocore-1.43.28-py3-none-any.whl", hash = "sha256:8147adea89b4c9324e842cd8c01ea1a0e17c92cb6ebeaa8cb774f821cb5a7629", size = 15188401, upload-time = "2026-06-11T19:28:47.244Z" }, +] + +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cython" +version = "0.29.37" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/97/8cc3fe7c6de4796921236a64d00ca8a95565772e57f0d3caae68d880b592/Cython-0.29.37.tar.gz", hash = "sha256:f813d4a6dd94adee5d4ff266191d1d95bf6d4164a4facc535422c021b2504cfb", size = 2099621, upload-time = "2023-12-19T09:22:39.397Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/aa/99a0eac01136c0c75feb3210d107c49f93d49d5cb97f19e99318b9ecefdd/Cython-0.29.37-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8af5975ecfae254d8c0051204fca995dda8f93cf9f0bbf7571e3cda2b0cef4d", size = 1951525, upload-time = "2023-12-19T09:57:00.464Z" }, + { url = "https://files.pythonhosted.org/packages/14/5f/f5efba4409474892d650ba7e57349afa5d4c41d06f5ba3356c32891c75a6/Cython-0.29.37-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:29415d8eb2fdc1ea518ca4810c50a2d062b387d4c9fbcfb3352346e93db22c6d", size = 1950936, upload-time = "2023-12-19T09:24:07.949Z" }, + { url = "https://files.pythonhosted.org/packages/7e/26/9d8de10005fedb1eceabe713348d43bae1dbab1786042ca0751a2e2b0f8c/Cython-0.29.37-py2.py3-none-any.whl", hash = "sha256:95f1d6a83ef2729e67b3fa7318c829ce5b07ac64c084cd6af11c228e0364662c", size = 989503, upload-time = "2023-12-19T09:22:32.861Z" }, +] + +[[package]] +name = "docutils" +version = "0.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/330ea8d383eb2ce973df34d1239b3b21e91cd8c865d21ff82902d952f91f/docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6", size = 2056383, upload-time = "2022-07-05T20:17:31.045Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/69/e391bd51bc08ed9141ecd899a0ddb61ab6465309f1eb470905c0c8868081/docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc", size = 570472, upload-time = "2022-07-05T20:17:26.388Z" }, +] + +[[package]] +name = "falcon" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/53/4fd90c6c841bc2e4be29ab92c65e5406df9096c421f138bef9d95d43afc9/falcon-3.1.0.tar.gz", hash = "sha256:f2760bd18c16393a6fb5e55f371f67921edb72febe693a82b3c5e82195d087b7", size = 645631, upload-time = "2022-03-25T16:13:19.137Z" } + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "gevent" +version = "26.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" }, + { name = "zope-event", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "zope-interface", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/cb/98aa3a299e2fc4a2372b5d124863e02965b64579ffc29fe54d0641e65b2f/gevent-26.5.0.tar.gz", hash = "sha256:1655eb04c1e20d71b2aa4a3c7528162dd58ff6cc46a037af1f01f534c80fefba", size = 6712354, upload-time = "2026-05-20T21:22:45.132Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/69/1559b1f6b5107a9118fccd300240879bd581b6d87b03d568d0d155ea702c/gevent-26.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bccff69c462e3650a0fd1d4e9cfc8b6effe15f3e9b1cad20a7bb5ce14b057efd", size = 2114915, upload-time = "2026-05-20T20:35:25.041Z" }, + { url = "https://files.pythonhosted.org/packages/f9/3c/2fe77ee6e3d381b3c50c0b7d6c4c08c08b8ff5e8c0d9dd51a3b426d61eec/gevent-26.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0bf57df54f1c66273bf3601c2a1e41b12138fe848933718369663bc54f177ca2", size = 2140806, upload-time = "2026-05-20T20:43:28.895Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/6e/802acd792aebb2256fbbee8cacf2727faaeb6f240ac11008f09eae4414bc/greenlet-3.5.1.tar.gz", hash = "sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829", size = 197356, upload-time = "2026-05-20T15:05:03.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/69/b91cda0647df839483201545913514c2827ebea5e5ccdf931842763bc127/greenlet-3.5.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:add5217d68b31130f0beca584d7fef4878327d2e31642b66618a14eef312b63b", size = 611358, upload-time = "2026-05-20T13:14:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/65/2c/18cece62045e74598c3c393f70dce4a63f56222015ba29a5d4eeb04f764c/greenlet-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c5551170cf4f5ff5623e9af81323751979fee2c731e2287b61f73cd27257b823", size = 1635625, upload-time = "2026-05-20T13:14:34.027Z" }, +] + +[[package]] +name = "grpcio" +version = "1.81.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/b5/1ff353970a87eda4c98251e34d2dfd214abd4982dc89119c9252a2a482d2/grpcio-1.81.1.tar.gz", hash = "sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b", size = 13026582, upload-time = "2026-06-11T12:46:51.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8c/dea020b6d91508cd84463917a63149ec196ee7db505d032ae43fcb3303b9/grpcio-1.81.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:766bc7c9a9c340342f4c864ccbda8e78111e4751f13b895812b9c148fb79e9d0", size = 6809167, upload-time = "2026-06-11T12:45:32.52Z" }, + { url = "https://files.pythonhosted.org/packages/25/7a/71437c7f3596e5246155c515852795a85a1a8d228190212432b13b97a95d/grpcio-1.81.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8ea1936c26b99999b27479853039a7f34713f56c49375ad52b38535ec93a796c", size = 7849660, upload-time = "2026-06-11T12:45:40.627Z" }, +] + +[[package]] +name = "gunicorn" +version = "26.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/b7/a4a3f632f823e432ce6bc65f62961b7980c898c77f075a2f7118cb3846fe/gunicorn-26.0.0.tar.gz", hash = "sha256:ca9346f85e3a4aeeb64d491045c16b9a35647abd37ea15efe53080eb8b090baf", size = 727286, upload-time = "2026-05-05T06:38:25.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/40/9c2384fc2be4ad25dd4a49decd5ad9ea5a3639814c11bd40ab77cb9f0a14/gunicorn-26.0.0-py3-none-any.whl", hash = "sha256:40233d26a5f0d1872916188c276e21641155111c2853f0c2cd55260aec0d24fc", size = 212009, upload-time = "2026-05-05T06:38:23.007Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +] + +[[package]] +name = "narwhals" +version = "2.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/62/3c/c4ef2164a71c1a63d7f1ae411c4082c5fa872405106db60a4b7114989ad7/narwhals-2.22.1.tar.gz", hash = "sha256:d62920805a0a43b7ff8b54b0c0d3142d796f8a9301836ada37e573d6a33cbcd9", size = 647493, upload-time = "2026-06-05T12:34:34.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl", hash = "sha256:60567d774edf77db53906f89d9fbd164e66e56d66d388e1e6990f17ac33cfb53", size = 454815, upload-time = "2026-06-05T12:34:32.289Z" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, + { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "python-dateutil", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" }, + { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "charset-normalizer", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "idna", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "urllib3", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rsa" +version = "4.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/b5/475c45a58650b0580421746504b680cd2db4e81bc941e94ca53785250269/rsa-4.7.2.tar.gz", hash = "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9", size = 39711, upload-time = "2021-02-24T10:55:05.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/93/0c0f002031f18b53af7a6166103c02b9c0667be528944137cc954ec921b3/rsa-4.7.2-py3-none-any.whl", hash = "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", size = 34505, upload-time = "2021-02-24T10:55:03.55Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/1f/12417f7f493fc45e1f9fd5d4a9b6c125cf8d2cf3f8ddbdfab3e76406e9d6/s3transfer-0.18.0.tar.gz", hash = "sha256:3760b8b7ec1315da54048b2d626276732bee4300d054d492d4e1d43e20d4ecbd", size = 160560, upload-time = "2026-05-28T19:39:09.124Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/58/a58fc997655386daa2e25784e30c288aa3e3819e401f77029ee4899fb55a/s3transfer-0.18.0-py3-none-any.whl", hash = "sha256:239c13b09e65ad0346e1be7348b8a202dcad44ac7ea7c6eb858fc881dce739b6", size = 88572, upload-time = "2026-05-28T19:39:07.999Z" }, +] + +[[package]] +name = "scikit-learn" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "narwhals", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "scipy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "threadpoolctl", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8", size = 9132097, upload-time = "2026-06-02T11:53:33.456Z" }, +] + +[[package]] +name = "scipy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, + { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "tensorflow-serving-inference-dlc-cpu" +version = "2.20.0" +source = { virtual = "." } +dependencies = [ + { name = "aiohttp", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "awscli", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "boto3", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "botocore", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "cython", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "falcon", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "gevent", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "grpcio", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "gunicorn", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "packaging", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "protobuf", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "requests", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "urllib3", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] + +[package.optional-dependencies] +sagemaker = [ + { name = "cloudpickle", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pandas", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "scikit-learn", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiohttp", specifier = ">=3.14.0" }, + { name = "awscli", specifier = "<2" }, + { name = "boto3" }, + { name = "botocore" }, + { name = "cloudpickle", marker = "extra == 'sagemaker'" }, + { name = "cython", specifier = "<3" }, + { name = "falcon", specifier = "==3.1.0" }, + { name = "gevent" }, + { name = "grpcio", specifier = ">=1.0" }, + { name = "gunicorn", specifier = ">=22.0.0" }, + { name = "numpy", specifier = "==1.26.4" }, + { name = "packaging" }, + { name = "pandas", marker = "extra == 'sagemaker'" }, + { name = "protobuf", specifier = ">=3.20,<7" }, + { name = "requests" }, + { name = "scikit-learn", marker = "extra == 'sagemaker'" }, + { name = "urllib3" }, +] +provides-extras = ["sagemaker"] + +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "yarl" +version = "1.24.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "multidict", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "propcache", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c", size = 105612, upload-time = "2026-05-19T21:29:04.247Z" }, + { url = "https://files.pythonhosted.org/packages/48/2d/1c8d89c7c5f9cad9fb2902445d94e2ab1d7aa35de029afbb8ae95c42d00f/yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1", size = 105719, upload-time = "2026-05-19T21:29:18.367Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, +] + +[[package]] +name = "zope-event" +version = "6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/93/41/faa10af34d48d9cd6fa0249a1162943ad84a9590bd1a06939981e6640416/zope_event-6.2.tar.gz", hash = "sha256:b97d5d6327067ee6b9dfcbdf606ade9ade70991e19c162e808ea39e5fcf0f8d3", size = 18958, upload-time = "2026-04-28T06:24:10.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/33/848922889e946d4befc415c219fe516af75c49555d8e736e183bfd30db42/zope_event-6.2-py3-none-any.whl", hash = "sha256:5e755153ac4faf64c10a4b6dd3307680166a3edf65b38df22df592610f8fa874", size = 6525, upload-time = "2026-04-28T06:24:09.176Z" }, +] + +[[package]] +name = "zope-interface" +version = "8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" }, +] diff --git a/docker/tensorflow/inference/2.20/cuda/pyproject.toml b/docker/tensorflow/inference/2.20/cuda/pyproject.toml new file mode 100644 index 000000000000..5e27349df5f1 --- /dev/null +++ b/docker/tensorflow/inference/2.20/cuda/pyproject.toml @@ -0,0 +1,90 @@ +[project] +name = "tensorflow-serving-inference-dlc" +version = "2.20.0" +requires-python = ">=3.12,<3.13" +# ────────────────────────────────────────────────────────────────────────────── +# TF Serving 2.20 CUDA AL2023 inference DLC — runtime-base dependencies +# +# The TF Serving binary itself is NOT installed via pip — it is COPYed out of +# tensorflow/serving:2.20.0-devel-gpu in the Dockerfile. The +# `tensorflow-serving-api-gpu` wheel is the matched gRPC stub (used by +# python_service.py to call the local serving binary on port 8500). +# +# IMPORTANT — locked decision Q1 (handoff): +# `tensorflow-serving-api-gpu==2.20.0` is INTENTIONALLY OMITTED from this +# `dependencies` list. The PyPI wheel declares a hard `tensorflow` runtime +# dep (≈600 MB framework wheel) which we do not want in an inference image. +# uv's resolver does not support per-package `--no-deps`, so the wheel is +# instead installed inline in the Dockerfile via: +# uv pip install --no-deps tensorflow-serving-api-gpu==2.20.0 +# The transitive runtime needs of tfs-api (numpy, protobuf, grpcio) are +# declared explicitly below so a coherent venv is locked. OSS license +# tracking still picks up tfs-api at compliance-scan time because the +# wheel is physically present in the image. +# +# Source for this dep list: master TF 2.19 inference Dockerfile.gpu +# (RUN pip install blocks at lines ~155 and SM-stage line ~265), translated +# to uv/pyproject form per training PR #6107 pattern. +# ────────────────────────────────────────────────────────────────────────────── +dependencies = [ + # ── tfs-api transitive runtime needs (explicit pins; see header note) ───── + # numpy 1.26.4 — TF SavedModel/numpy ABI compat. Must match the version + # the TF 2.21 training image uses to export SavedModels (handoff §16). + "numpy==1.26.4", + # protobuf — tfs-api uses protobuf for the gRPC stubs. Master 2.19 pinned + # protobuf==6.33.4; we relax to >=3.20,<7 here so the resolver can pick the + # latest line that satisfies the rest of the venv. Floor 3.20 matches the + # tfs-api wheel's metadata; ceiling 7 prevents future major-version churn. + "protobuf>=3.20,<7", + # grpcio — gRPC client used by python_service.py. Latest stable line. + "grpcio>=1.0", + + # ── AWS / network / common utilities (carried over from master 2.19) ───── + "awscli<2", + "boto3", + "botocore", + "requests", + "packaging", + "gevent", + + # ── Falcon-based SM handler stack ───────────────────────────────────────── + # Both pins user-locked per handoff §16. Master GPU used falcon==3.0.1, but + # we standardize on 3.1.0 (matches master CPU and is forward-compatible). + "falcon==3.1.0", + "gunicorn>=22.0.0", + + # ── Compat / security pins inherited from training PR #6107 ─────────────── + # cython<3 — TF/numpy ABI compat; same constraint as training image. + "cython<3", + # aiohttp>=3.14.0 — security pin (CVE-2026-34993, CVE-2026-47265). Pulled + # transitively by aiobotocore / boto3 ecosystem. Direct pin forces resolver + # to pick the patched line. + "aiohttp>=3.14.0", + "urllib3", +] + +[project.optional-dependencies] +sagemaker = [ + # NOTE: `sagemaker-tensorflow-serving-container` is NOT on PyPI (verified + # via `uv lock` failure on 2026-06-11; handoff §14.4 flagged this risk). + # Master TF 2.19 inference references it but never pip-installs it from + # PyPI — it is vendored as the local handler scripts under + # docker/build_artifacts/sagemaker/, which we port to scripts/tensorflow/ + # inference/sagemaker/ in Phase 3. + # + # Master TF 2.19 inference SM stage does NOT install the `sagemaker` SDK + # (verified by re-reading tensorflow/inference/docker/2.19/py3/cu122/Dockerfile.gpu + # lines 247-275). The handler scripts (python_service.py, multi_model_utils.py) + # use boto3 directly. Removing the SDK from this extra avoids transitively + # pulling torch + ~20 nvidia-cu13 packages we have no use for in an + # inference image. + # + # Lightweight data tooling — kept so customer inference handlers can use + # them without an extra pip install: + "pandas", + "scikit-learn", + "cloudpickle", +] + +[tool.uv] +environments = ["sys_platform == 'linux' and platform_machine == 'x86_64'"] diff --git a/docker/tensorflow/inference/2.20/cuda/uv.lock b/docker/tensorflow/inference/2.20/cuda/uv.lock new file mode 100644 index 000000000000..f63d2c8f949f --- /dev/null +++ b/docker/tensorflow/inference/2.20/cuda/uv.lock @@ -0,0 +1,550 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", +] +supported-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/c6/61a2d7b7572279226bb2e7f61d7a19ca7c90da0329c93fa0d560cbf288d8/aiohappyeyeballs-2.6.2.tar.gz", hash = "sha256:e202810ee718bd01fc6ef49e8ea53d023d5cb6b581076d7925aa499fa55dbe64", size = 22591, upload-time = "2026-05-20T15:12:24.631Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/fc/a7bf5b6e4e617b45f90f2d9d2a68519c249c81dd4fc2658c7a2a61c4f4b7/aiohappyeyeballs-2.6.2-py3-none-any.whl", hash = "sha256:4708045e2d7a6c6bdf8aafa8ed39649eaf926a4543b54560659129e3365953c4", size = 15062, upload-time = "2026-05-20T15:12:23.328Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "aiosignal", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "attrs", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "frozenlist", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "multidict", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "propcache", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "yarl", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "awscli" +version = "1.45.28" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "colorama", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "docutils", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pyyaml", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "rsa", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "s3transfer", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/32/68ff637efdba82e49fd12ddc068fc19edd7f6b4b3ee5bb5976379daaeb25/awscli-1.45.28.tar.gz", hash = "sha256:50e788ed5dc17eb916fd1b9c74d581e4b6a86147c0e60b8b2d25b60ceb32d458", size = 1896334, upload-time = "2026-06-11T19:28:58.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/41/d9a71dfafd7670e7a79fa4a0d9f073ebbecd33ff4d498f0715d39e4a4464/awscli-1.45.28-py3-none-any.whl", hash = "sha256:783a45041e364c1be9640f730cbcc18313767c592f555477cf0911b1c9511455", size = 4648370, upload-time = "2026-06-11T19:28:54.876Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.28" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "jmespath", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "s3transfer", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4e/f2/a976b2a81d8dc7ff675f4b614367a185727061130184a28da0f53f446b97/boto3-1.43.28.tar.gz", hash = "sha256:8391fdcc4d8e1d4e0bf96575a7e5610964a4d401dafa4dccb0a5bade8dd3fbb0", size = 113202, upload-time = "2026-06-11T19:29:01.464Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/a5/47db150ea6380f11569b87d3ad064e3c929e5abe227a549d472fab6f5f3a/boto3-1.43.28-py3-none-any.whl", hash = "sha256:4fe6df2163aea02b561eca0d685e2f41a059d71f03721a3e79c3b522e79a3b56", size = 140536, upload-time = "2026-06-11T19:29:00.143Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.28" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "python-dateutil", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "urllib3", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/dc/1b01808003f88f8a328732c979f20cb0456791048b4440fc4abcae08c1a0/botocore-1.43.28.tar.gz", hash = "sha256:9bbad501a68e4ffdbeff76a382507f5d7827abc316f34a218ab76f5293e6c78d", size = 15503514, upload-time = "2026-06-11T19:28:50.989Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/8c/14916c353ce8a29d14cf6308c2bef842bbb25dde6defc620e26e28063331/botocore-1.43.28-py3-none-any.whl", hash = "sha256:8147adea89b4c9324e842cd8c01ea1a0e17c92cb6ebeaa8cb774f821cb5a7629", size = 15188401, upload-time = "2026-06-11T19:28:47.244Z" }, +] + +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cython" +version = "0.29.37" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/97/8cc3fe7c6de4796921236a64d00ca8a95565772e57f0d3caae68d880b592/Cython-0.29.37.tar.gz", hash = "sha256:f813d4a6dd94adee5d4ff266191d1d95bf6d4164a4facc535422c021b2504cfb", size = 2099621, upload-time = "2023-12-19T09:22:39.397Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/aa/99a0eac01136c0c75feb3210d107c49f93d49d5cb97f19e99318b9ecefdd/Cython-0.29.37-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8af5975ecfae254d8c0051204fca995dda8f93cf9f0bbf7571e3cda2b0cef4d", size = 1951525, upload-time = "2023-12-19T09:57:00.464Z" }, + { url = "https://files.pythonhosted.org/packages/14/5f/f5efba4409474892d650ba7e57349afa5d4c41d06f5ba3356c32891c75a6/Cython-0.29.37-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:29415d8eb2fdc1ea518ca4810c50a2d062b387d4c9fbcfb3352346e93db22c6d", size = 1950936, upload-time = "2023-12-19T09:24:07.949Z" }, + { url = "https://files.pythonhosted.org/packages/7e/26/9d8de10005fedb1eceabe713348d43bae1dbab1786042ca0751a2e2b0f8c/Cython-0.29.37-py2.py3-none-any.whl", hash = "sha256:95f1d6a83ef2729e67b3fa7318c829ce5b07ac64c084cd6af11c228e0364662c", size = 989503, upload-time = "2023-12-19T09:22:32.861Z" }, +] + +[[package]] +name = "docutils" +version = "0.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/330ea8d383eb2ce973df34d1239b3b21e91cd8c865d21ff82902d952f91f/docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6", size = 2056383, upload-time = "2022-07-05T20:17:31.045Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/69/e391bd51bc08ed9141ecd899a0ddb61ab6465309f1eb470905c0c8868081/docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc", size = 570472, upload-time = "2022-07-05T20:17:26.388Z" }, +] + +[[package]] +name = "falcon" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/53/4fd90c6c841bc2e4be29ab92c65e5406df9096c421f138bef9d95d43afc9/falcon-3.1.0.tar.gz", hash = "sha256:f2760bd18c16393a6fb5e55f371f67921edb72febe693a82b3c5e82195d087b7", size = 645631, upload-time = "2022-03-25T16:13:19.137Z" } + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "gevent" +version = "26.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" }, + { name = "zope-event", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "zope-interface", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/cb/98aa3a299e2fc4a2372b5d124863e02965b64579ffc29fe54d0641e65b2f/gevent-26.5.0.tar.gz", hash = "sha256:1655eb04c1e20d71b2aa4a3c7528162dd58ff6cc46a037af1f01f534c80fefba", size = 6712354, upload-time = "2026-05-20T21:22:45.132Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/69/1559b1f6b5107a9118fccd300240879bd581b6d87b03d568d0d155ea702c/gevent-26.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bccff69c462e3650a0fd1d4e9cfc8b6effe15f3e9b1cad20a7bb5ce14b057efd", size = 2114915, upload-time = "2026-05-20T20:35:25.041Z" }, + { url = "https://files.pythonhosted.org/packages/f9/3c/2fe77ee6e3d381b3c50c0b7d6c4c08c08b8ff5e8c0d9dd51a3b426d61eec/gevent-26.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0bf57df54f1c66273bf3601c2a1e41b12138fe848933718369663bc54f177ca2", size = 2140806, upload-time = "2026-05-20T20:43:28.895Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/6e/802acd792aebb2256fbbee8cacf2727faaeb6f240ac11008f09eae4414bc/greenlet-3.5.1.tar.gz", hash = "sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829", size = 197356, upload-time = "2026-05-20T15:05:03.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/69/b91cda0647df839483201545913514c2827ebea5e5ccdf931842763bc127/greenlet-3.5.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:add5217d68b31130f0beca584d7fef4878327d2e31642b66618a14eef312b63b", size = 611358, upload-time = "2026-05-20T13:14:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/65/2c/18cece62045e74598c3c393f70dce4a63f56222015ba29a5d4eeb04f764c/greenlet-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c5551170cf4f5ff5623e9af81323751979fee2c731e2287b61f73cd27257b823", size = 1635625, upload-time = "2026-05-20T13:14:34.027Z" }, +] + +[[package]] +name = "grpcio" +version = "1.81.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/b5/1ff353970a87eda4c98251e34d2dfd214abd4982dc89119c9252a2a482d2/grpcio-1.81.1.tar.gz", hash = "sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b", size = 13026582, upload-time = "2026-06-11T12:46:51.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8c/dea020b6d91508cd84463917a63149ec196ee7db505d032ae43fcb3303b9/grpcio-1.81.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:766bc7c9a9c340342f4c864ccbda8e78111e4751f13b895812b9c148fb79e9d0", size = 6809167, upload-time = "2026-06-11T12:45:32.52Z" }, + { url = "https://files.pythonhosted.org/packages/25/7a/71437c7f3596e5246155c515852795a85a1a8d228190212432b13b97a95d/grpcio-1.81.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8ea1936c26b99999b27479853039a7f34713f56c49375ad52b38535ec93a796c", size = 7849660, upload-time = "2026-06-11T12:45:40.627Z" }, +] + +[[package]] +name = "gunicorn" +version = "26.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/b7/a4a3f632f823e432ce6bc65f62961b7980c898c77f075a2f7118cb3846fe/gunicorn-26.0.0.tar.gz", hash = "sha256:ca9346f85e3a4aeeb64d491045c16b9a35647abd37ea15efe53080eb8b090baf", size = 727286, upload-time = "2026-05-05T06:38:25.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/40/9c2384fc2be4ad25dd4a49decd5ad9ea5a3639814c11bd40ab77cb9f0a14/gunicorn-26.0.0-py3-none-any.whl", hash = "sha256:40233d26a5f0d1872916188c276e21641155111c2853f0c2cd55260aec0d24fc", size = 212009, upload-time = "2026-05-05T06:38:23.007Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +] + +[[package]] +name = "narwhals" +version = "2.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/62/3c/c4ef2164a71c1a63d7f1ae411c4082c5fa872405106db60a4b7114989ad7/narwhals-2.22.1.tar.gz", hash = "sha256:d62920805a0a43b7ff8b54b0c0d3142d796f8a9301836ada37e573d6a33cbcd9", size = 647493, upload-time = "2026-06-05T12:34:34.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl", hash = "sha256:60567d774edf77db53906f89d9fbd164e66e56d66d388e1e6990f17ac33cfb53", size = 454815, upload-time = "2026-06-05T12:34:32.289Z" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, + { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "python-dateutil", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" }, + { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "charset-normalizer", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "idna", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "urllib3", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rsa" +version = "4.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/b5/475c45a58650b0580421746504b680cd2db4e81bc941e94ca53785250269/rsa-4.7.2.tar.gz", hash = "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9", size = 39711, upload-time = "2021-02-24T10:55:05.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/93/0c0f002031f18b53af7a6166103c02b9c0667be528944137cc954ec921b3/rsa-4.7.2-py3-none-any.whl", hash = "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", size = 34505, upload-time = "2021-02-24T10:55:03.55Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/1f/12417f7f493fc45e1f9fd5d4a9b6c125cf8d2cf3f8ddbdfab3e76406e9d6/s3transfer-0.18.0.tar.gz", hash = "sha256:3760b8b7ec1315da54048b2d626276732bee4300d054d492d4e1d43e20d4ecbd", size = 160560, upload-time = "2026-05-28T19:39:09.124Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/58/a58fc997655386daa2e25784e30c288aa3e3819e401f77029ee4899fb55a/s3transfer-0.18.0-py3-none-any.whl", hash = "sha256:239c13b09e65ad0346e1be7348b8a202dcad44ac7ea7c6eb858fc881dce739b6", size = 88572, upload-time = "2026-05-28T19:39:07.999Z" }, +] + +[[package]] +name = "scikit-learn" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "narwhals", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "scipy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "threadpoolctl", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8", size = 9132097, upload-time = "2026-06-02T11:53:33.456Z" }, +] + +[[package]] +name = "scipy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, + { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "tensorflow-serving-inference-dlc" +version = "2.20.0" +source = { virtual = "." } +dependencies = [ + { name = "aiohttp", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "awscli", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "boto3", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "botocore", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "cython", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "falcon", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "gevent", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "grpcio", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "gunicorn", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "packaging", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "protobuf", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "requests", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "urllib3", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] + +[package.optional-dependencies] +sagemaker = [ + { name = "cloudpickle", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pandas", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "scikit-learn", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiohttp", specifier = ">=3.14.0" }, + { name = "awscli", specifier = "<2" }, + { name = "boto3" }, + { name = "botocore" }, + { name = "cloudpickle", marker = "extra == 'sagemaker'" }, + { name = "cython", specifier = "<3" }, + { name = "falcon", specifier = "==3.1.0" }, + { name = "gevent" }, + { name = "grpcio", specifier = ">=1.0" }, + { name = "gunicorn", specifier = ">=22.0.0" }, + { name = "numpy", specifier = "==1.26.4" }, + { name = "packaging" }, + { name = "pandas", marker = "extra == 'sagemaker'" }, + { name = "protobuf", specifier = ">=3.20,<7" }, + { name = "requests" }, + { name = "scikit-learn", marker = "extra == 'sagemaker'" }, + { name = "urllib3" }, +] +provides-extras = ["sagemaker"] + +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "yarl" +version = "1.24.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "multidict", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "propcache", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c", size = 105612, upload-time = "2026-05-19T21:29:04.247Z" }, + { url = "https://files.pythonhosted.org/packages/48/2d/1c8d89c7c5f9cad9fb2902445d94e2ab1d7aa35de029afbb8ae95c42d00f/yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1", size = 105719, upload-time = "2026-05-19T21:29:18.367Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, +] + +[[package]] +name = "zope-event" +version = "6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/93/41/faa10af34d48d9cd6fa0249a1162943ad84a9590bd1a06939981e6640416/zope_event-6.2.tar.gz", hash = "sha256:b97d5d6327067ee6b9dfcbdf606ade9ade70991e19c162e808ea39e5fcf0f8d3", size = 18958, upload-time = "2026-04-28T06:24:10.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/33/848922889e946d4befc415c219fe516af75c49555d8e736e183bfd30db42/zope_event-6.2-py3-none-any.whl", hash = "sha256:5e755153ac4faf64c10a4b6dd3307680166a3edf65b38df22df592610f8fa874", size = 6525, upload-time = "2026-04-28T06:24:09.176Z" }, +] + +[[package]] +name = "zope-interface" +version = "8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" }, +] diff --git a/docker/tensorflow/inference/2.20/versions-cpu.env b/docker/tensorflow/inference/2.20/versions-cpu.env new file mode 100644 index 000000000000..dde4c749e6e6 --- /dev/null +++ b/docker/tensorflow/inference/2.20/versions-cpu.env @@ -0,0 +1,32 @@ +# versions-cpu.env — Pinned versions for TensorFlow Serving 2.20 CPU AL2023 inference DLC +# Source this file: source versions-cpu.env + +# ── Image metadata ────────────────────────────────────────────── +export DLC_MAJOR_VERSION="1" +export DLC_MINOR_VERSION="0" +export BUILD_DATE="$(date +%Y%m%d)" + +# ── Python ────────────────────────────────────────────────────── +export PYTHON_VERSION="3.12" + +# ── TensorFlow Serving ───────────────────────────────────────── +# The inference image ships the tensorflow_model_server binary copied out of +# tensorflow/serving:2.20.0-devel. PyPI tensorflow-serving-api==2.20.0 is the +# matched gRPC stub (installed inline in the Dockerfile via --no-deps; see locked +# decision Q1 in cpu/pyproject.toml header). +# Note: this is TF Serving's version, NOT the framework version (2.21). TF Serving's +# release cadence trails the framework — 2.20 is the latest published serving release. +export TF_SERVING_VERSION="2.20.0" + +# Git commit pin on the tensorflow/serving repo (branch r2.20). Used by +# `setup.sources.sh` and OSS-compliance manifest references. Sourced from +# `git ls-remote https://github.com/tensorflow/serving refs/heads/r2.20` at the +# start of Phase 2 (2026-06-11). +export TF_SERVING_VERSION_GIT_COMMIT="bc7e9d2b9a419294c4526e293c2df8727b0d3116" + +# ── nginx + njs (compiled in builder-njs stage) ──────────────── +# AL2023 base repo ships nginx 1.24.x without nginx-module-njs; we therefore +# build njs as a dynamic module from source against a matched nginx source tree. +# Same versions as the CUDA variant — the njs build does not depend on CUDA. +export NGINX_VERSION="1.30.2" +export NJS_VERSION="0.9.9" diff --git a/docker/tensorflow/inference/2.20/versions-cuda.env b/docker/tensorflow/inference/2.20/versions-cuda.env new file mode 100644 index 000000000000..ee5d7f7cbd0f --- /dev/null +++ b/docker/tensorflow/inference/2.20/versions-cuda.env @@ -0,0 +1,46 @@ +# versions-cuda.env — Pinned versions for TensorFlow Serving 2.20 CUDA AL2023 inference DLC +# Source this file: source versions-cuda.env + +# ── Image metadata ────────────────────────────────────────────── +export DLC_MAJOR_VERSION="1" +export DLC_MINOR_VERSION="0" +export BUILD_DATE="$(date +%Y%m%d)" + +# ── CUDA ─────────────────────────────────────────────────────── +# CUDA 12.9.1 — matches the TF 2.21 training image base. tensorflow/serving:2.20.0-devel-gpu +# is compiled against CUDA 12.x (upstream-pinned by Google's TFS build matrix); we rely on +# CUDA minor-version forward compatibility to run the serving binary on CUDA 12.9.1 runtime. +# This is the same compat chain the training image uses for the TF 2.21 wheel built against +# CUDA 12.5 — verified empirically there. +export CUDA_VERSION="12.9.1" + +# ── Python ────────────────────────────────────────────────────── +export PYTHON_VERSION="3.12" + +# ── TensorFlow Serving ───────────────────────────────────────── +# The inference image ships the tensorflow_model_server binary copied out of +# tensorflow/serving:2.20.0-devel-gpu. PyPI tensorflow-serving-api-gpu==2.20.0 is the +# matched gRPC stub (installed inline in the Dockerfile via --no-deps; see locked +# decision Q1 in cuda/pyproject.toml header). +# Note: this is TF Serving's version, NOT the framework version (2.21). TF Serving's +# release cadence trails the framework — 2.20 is the latest published serving release. +export TF_SERVING_VERSION="2.20.0" + +# Git commit pin on the tensorflow/serving repo (branch r2.20). Used by +# `setup.sources.sh` for any apt source bootstrapping that the upstream TFS build +# scripts perform during the binary copy or by reference inside the OSS-compliance +# manifest. Sourced from `git ls-remote https://github.com/tensorflow/serving refs/heads/r2.20` +# at the start of Phase 2 (2026-06-11). +export TF_SERVING_VERSION_GIT_COMMIT="bc7e9d2b9a419294c4526e293c2df8727b0d3116" + +# ── nginx + njs (compiled in builder-njs stage) ──────────────── +# AL2023 base repo ships nginx 1.24.x without nginx-module-njs; we therefore +# build njs as a dynamic module from source against a matched nginx source tree +# (mainline 1.30.x, current stable line as of 2026-06). +# +# nginx 1.30.2 — current mainline stable; ships CVE-2026-9256 fix +# (heap buffer overflow in ngx_http_rewrite_module). +# njs 0.9.9 — latest stable; ships CVE-2026-8711 fix (heap buffer overflow +# in js_fetch_proxy directive). +export NGINX_VERSION="1.30.2" +export NJS_VERSION="0.9.9" diff --git a/scripts/tensorflow/inference/dockerd_entrypoint.sh b/scripts/tensorflow/inference/dockerd_entrypoint.sh new file mode 100755 index 000000000000..341ba60575a3 --- /dev/null +++ b/scripts/tensorflow/inference/dockerd_entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Execute telemetry script if it exists, suppress errors +bash /usr/local/bin/bash_telemetry.sh >/dev/null 2>&1 || true + +TF_SERVING_PACKAGE=$(pip list | grep tensorflow-serving | cut -d ' ' -f 1) + +if [[ ${TF_SERVING_PACKAGE} == *"gpu"* ]]; then + bash /usr/local/bin/start_cuda_compat.sh +fi + +eval '"$@"' diff --git a/scripts/tensorflow/inference/sagemaker/__init__.py b/scripts/tensorflow/inference/sagemaker/__init__.py new file mode 100644 index 000000000000..04fbf5d9a144 --- /dev/null +++ b/scripts/tensorflow/inference/sagemaker/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/scripts/tensorflow/inference/sagemaker/multi_model_utils.py b/scripts/tensorflow/inference/sagemaker/multi_model_utils.py new file mode 100644 index 000000000000..79bf7a88464c --- /dev/null +++ b/scripts/tensorflow/inference/sagemaker/multi_model_utils.py @@ -0,0 +1,53 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import fcntl +import signal +import time +from contextlib import contextmanager + +MODEL_CONFIG_FILE = "/sagemaker/model-config.cfg" +DEFAULT_LOCK_FILE = "/sagemaker/lock-file.lock" + + +@contextmanager +def lock(path=DEFAULT_LOCK_FILE): + f = open(path, "w", encoding="utf8") + fd = f.fileno() + fcntl.lockf(fd, fcntl.LOCK_EX) + + try: + yield + finally: + time.sleep(1) + fcntl.lockf(fd, fcntl.LOCK_UN) + + +@contextmanager +def timeout(seconds=60): + def _raise_timeout_error(signum, frame): + raise Exception(408, "Timed out after {} seconds".format(seconds)) + + try: + signal.signal(signal.SIGALRM, _raise_timeout_error) + signal.alarm(seconds) + yield + finally: + signal.alarm(0) + + +class MultiModelException(Exception): + def __init__(self, code, msg, pid): + Exception.__init__(self, code, msg) + self.pid = pid + self.code = code + self.msg = msg diff --git a/scripts/tensorflow/inference/sagemaker/nginx.conf.template b/scripts/tensorflow/inference/sagemaker/nginx.conf.template new file mode 100644 index 000000000000..3951fe96b939 --- /dev/null +++ b/scripts/tensorflow/inference/sagemaker/nginx.conf.template @@ -0,0 +1,67 @@ +load_module modules/ngx_http_js_module.so; + +worker_processes auto; +daemon off; +pid /tmp/nginx.pid; +error_log /dev/stderr %NGINX_LOG_LEVEL%; + +worker_rlimit_nofile 4096; + +events { + worker_connections 2048; +} + +http { + include /etc/nginx/mime.types; + default_type application/json; + access_log /dev/stdout combined; + js_import tensorflowServing.js; + + proxy_read_timeout %PROXY_READ_TIMEOUT%; + + upstream tfs_upstream { + %TFS_UPSTREAM%; + } + + upstream gunicorn_upstream { + server unix:/tmp/gunicorn.sock fail_timeout=1; + } + + server { + listen %NGINX_HTTP_PORT% deferred; + client_max_body_size 0; + client_body_buffer_size 100m; + subrequest_output_buffer_size 100m; + + set $tfs_version %TFS_VERSION%; + set $default_tfs_model %TFS_DEFAULT_MODEL_NAME%; + + location /tfs { + rewrite ^/tfs/(.*) /$1 break; + proxy_redirect off; + proxy_pass_request_headers off; + proxy_set_header Content-Type 'application/json'; + proxy_set_header Accept 'application/json'; + proxy_pass http://tfs_upstream; + } + + location /ping { + %FORWARD_PING_REQUESTS%; + } + + location /invocations { + %FORWARD_INVOCATION_REQUESTS%; + } + + location /models { + proxy_pass http://gunicorn_upstream/models; + } + + location / { + return 404 '{"error": "Not Found"}'; + } + + keepalive_timeout 3; + } +} + \ No newline at end of file diff --git a/scripts/tensorflow/inference/sagemaker/python_service.py b/scripts/tensorflow/inference/sagemaker/python_service.py new file mode 100644 index 000000000000..0cb2dc9ecabf --- /dev/null +++ b/scripts/tensorflow/inference/sagemaker/python_service.py @@ -0,0 +1,689 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +# monkey patching to ensure that all I/O operations are properly made asynchronous. +import gevent.monkey + +gevent.monkey.patch_all() + +import bisect +import argparse +import importlib.util +import json +import logging +import os +import signal +import subprocess +import grpc +import sys +import shutil +import copy +import pickle + +import falcon +import requests +import random + +from multi_model_utils import MultiModelException, lock +import tfs_utils + +SAGEMAKER_MULTI_MODEL_ENABLED = os.environ.get("SAGEMAKER_MULTI_MODEL", "false").lower() == "true" +INFERENCE_SCRIPT_PATH = ( + "/opt/ml/code/inference.py" + if SAGEMAKER_MULTI_MODEL_ENABLED + else "/opt/ml/model/code/inference.py" +) + +SAGEMAKER_BATCHING_ENABLED = os.environ.get("SAGEMAKER_TFS_ENABLE_BATCHING", "false").lower() +MODEL_CONFIG_FILE_PATH = "/sagemaker/model-config.cfg" +TFS_GRPC_PORTS = os.environ.get("TFS_GRPC_PORTS") +TFS_REST_PORTS = os.environ.get("TFS_REST_PORTS") +SAGEMAKER_TFS_PORT_RANGE = os.environ.get("SAGEMAKER_SAFE_PORT_RANGE") +TFS_INSTANCE_COUNT = int(os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", "1")) + +logging.basicConfig( + format="%(process)d %(asctime)s %(levelname)-8s %(message)s", force=True, level=logging.INFO +) +log = logging.getLogger(__name__) + +CUSTOM_ATTRIBUTES_HEADER = "X-Amzn-SageMaker-Custom-Attributes" +MME_TFS_INSTANCE_STATUS_FILE = "/sagemaker/tfs_instance.pickle" + + +def default_handler(data, context): + """A default inference request handler that directly send post request to TFS rest port with + un-processed data and return un-processed response + :param data: input data + :param context: context instance that contains tfs_rest_uri + :return: inference response from TFS model server + """ + data = data.read().decode("utf-8") + if not isinstance(data, str): + data = json.loads(data) + response = requests.post(context.rest_uri, data=data) + return response.content, context.accept_header + + +class TfsInstanceStatus: + def __init__(self, rest_port: str, grpc_port: str, pid: int): + self.rest_port = rest_port + self.grpc_port = grpc_port + self.pid = pid + + def __repr__(self): + return f"TFS Instance Status (rest_port : {self.rest_port}, grpc_port: {self.grpc_port}, pid: {self.pid}))" + + +class PythonServiceResource: + def __init__(self): + if SAGEMAKER_MULTI_MODEL_ENABLED: + self._mme_tfs_instances_status: dict[str, [TfsInstanceStatus]] = {} + self._tfs_ports = self._parse_sagemaker_port_range_mme(SAGEMAKER_TFS_PORT_RANGE) + self._tfs_available_ports = self._parse_sagemaker_port_range_mme( + SAGEMAKER_TFS_PORT_RANGE + ) + # If Multi-Model mode is enabled, dependencies/handlers will be imported + # during the _handle_load_model_post() + self.model_handlers = {} + else: + self._tfs_grpc_ports = self._parse_concat_ports(TFS_GRPC_PORTS) + self._tfs_rest_ports = self._parse_concat_ports(TFS_REST_PORTS) + + self._channels = {} + for grpc_port in self._tfs_grpc_ports: + # Initialize grpc channel here so gunicorn worker could have mapping + # between each grpc port and channel + self._setup_channel(grpc_port) + + self._default_handlers_enabled = False + if os.path.exists(INFERENCE_SCRIPT_PATH): + # Single-Model Mode & Multi-Model Mode both use one inference.py + self._handler, self._input_handler, self._output_handler = self._import_handlers() + self._handlers = self._make_handler( + self._handler, self._input_handler, self._output_handler + ) + else: + self._handlers = default_handler + self._default_handlers_enabled = True + + self._tfs_enable_batching = SAGEMAKER_BATCHING_ENABLED == "true" + self._tfs_default_model_name = os.environ.get("TFS_DEFAULT_MODEL_NAME", "None") + self._tfs_inter_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTER_OP_PARALLELISM", 0) + self._tfs_intra_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTRA_OP_PARALLELISM", 0) + self._tfs_instance_count = int(os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", 1)) + self._gunicorn_workers = int(os.environ.get("SAGEMAKER_GUNICORN_WORKERS", 1)) + self._tfs_wait_time_seconds = int( + os.environ.get("SAGEMAKER_TFS_WAIT_TIME_SECONDS", 55 // self._tfs_instance_count) + ) + + def on_post(self, req, res, model_name=None): + if model_name or "invocations" in req.uri: + self._handle_invocation_post(req, res, model_name) + else: + data = json.loads(req.stream.read().decode("utf-8")) + self._handle_load_model_post(res, data) + + def _parse_concat_ports(self, concat_ports): + return concat_ports.split(",") + + def _pick_port(self, ports): + return random.choice(ports) + + def _parse_sagemaker_port_range_mme(self, port_range): + lower, upper = port_range.split("-") + lower = int(lower) + upper = lower + int((int(upper) - lower) * 0.9) # only utilizing 90% of the ports + rest_port = lower + grpc_port = (lower + upper) // 2 + tfs_ports = { + "rest_port": [port for port in range(rest_port, grpc_port)], + "grpc_port": [port for port in range(grpc_port, upper)], + } + return tfs_ports + + def _ports_available(self): + rest_ports = self._tfs_available_ports["rest_port"] + grpc_ports = self._tfs_available_ports["grpc_port"] + return len(rest_ports) > 0 and len(grpc_ports) > 0 + + def _update_ports_available(self): + self._tfs_available_ports = copy.deepcopy(self._tfs_ports) + for _, tf_status_list in self._mme_tfs_instances_status.items(): + for tf_status in tf_status_list: + if tf_status.rest_port in self._tfs_available_ports["rest_port"]: + self._tfs_available_ports["rest_port"].remove(tf_status.rest_port) + if tf_status.grpc_port in self._tfs_available_ports["grpc_port"]: + self._tfs_available_ports["grpc_port"].remove(tf_status.grpc_port) + log.info(f"available ports : {self._tfs_available_ports}") + + def _load_model(self, model_name, base_path, rest_port, grpc_port, model_index): + if self.validate_model_dir(base_path): + try: + self._import_custom_modules(model_name) + tfs_config = tfs_utils.create_tfs_config_individual_model(model_name, base_path) + tfs_config_file = "/sagemaker/tfs-config/{}/{}/model-config.cfg".format( + model_name, model_index + ) + log.info("tensorflow serving model config: \n%s\n", tfs_config) + os.makedirs(os.path.dirname(tfs_config_file)) + with open(tfs_config_file, "w", encoding="utf8") as f: + f.write(tfs_config) + + batching_config_file = "/sagemaker/batching/{}/{}/batching-config.cfg".format( + model_name, model_index + ) + if self._tfs_enable_batching: + tfs_utils.create_batching_config(batching_config_file) + + cmd = tfs_utils.tfs_command( + grpc_port, + rest_port, + tfs_config_file, + self._tfs_enable_batching, + batching_config_file, + tfs_intra_op_parallelism=self._tfs_intra_op_parallelism, + tfs_inter_op_parallelism=self._tfs_inter_op_parallelism, + ) + log.info("MME starts tensorflow serving with command: {}".format(cmd)) + p = subprocess.Popen(cmd.split()) + + tfs_utils.wait_for_model(rest_port, model_name, self._tfs_wait_time_seconds, p.pid) + + log.info("started tensorflow serving (pid: %d)", p.pid) + + return { + "status": falcon.HTTP_200, + "body": json.dumps( + { + "success": "Successfully loaded model {}, " + "listening on rest port {} " + "and grpc port {}.".format(model_name, rest_port, grpc_port) + }, + ), + "pid": p.pid, + } + except MultiModelException as multi_model_exception: + if multi_model_exception.code == 409: + return { + "status": falcon.HTTP_409, + "body": multi_model_exception.msg, + "pid": multi_model_exception.pid, + } + elif multi_model_exception.code == 408: + cpu_memory_usage = tfs_utils.get_cpu_memory_util() + log.info(f"cpu memory usage {cpu_memory_usage}") + if cpu_memory_usage > 70: + return { + "status": falcon.HTTP_507, + "body": "Memory exhausted: not enough memory to start TFS instance", + "pid": multi_model_exception.pid, + } + return { + "status": falcon.HTTP_408, + "body": multi_model_exception.msg, + "pid": multi_model_exception.pid, + } + else: + return { + "status": falcon.HTTP_500, + "body": multi_model_exception.msg, + "pid": multi_model_exception.pid, + } + except FileExistsError as e: + return { + "status": falcon.HTTP_409, + "body": json.dumps( + {"error": "Model {} is already loaded. {}".format(model_name, str(e))} + ), + } + except OSError as os_error: + log.error(f"failed to load model with exception {os_error}") + if os_error.errno == 12: + return { + "status": falcon.HTTP_507, + "body": "Memory exhausted: not enough memory to start TFS instance", + } + else: + return { + "status": falcon.HTTP_500, + "body": os_error.strerror, + } + else: + return { + "status": falcon.HTTP_404, + "body": json.dumps( + { + "error": "Could not find valid base path {} for servable {}".format( + base_path, model_name + ) + } + ), + } + + def _handle_load_model_post(self, res, data): # noqa: C901 + with lock(): + model_name = data["model_name"] + base_path = data["url"] + + # sync sync_local_mme_instance_status & update available ports + self._sync_local_mme_instance_status() + self._update_ports_available() + self._sync_model_handlers() + + # model is already loaded + if model_name in self._mme_tfs_instances_status: + res.status = falcon.HTTP_409 + res.body = json.dumps({"error": "Model {} is already loaded.".format(model_name)}) + return + + is_load_successful = True + response = {} + for i in range(self._tfs_instance_count): + # check if there are available ports + if not self._ports_available(): + is_load_successful = False + response["status"] = falcon.HTTP_507 + response["body"] = json.dumps( + {"error": "Memory exhausted: no available ports to load the model."} + ) + break + tfs_rest_port = self._tfs_available_ports["rest_port"].pop() + tfs_grpc_port = self._tfs_available_ports["grpc_port"].pop() + + response = self._load_model(model_name, base_path, tfs_rest_port, tfs_grpc_port, i) + + if "pid" in response: + self._mme_tfs_instances_status.setdefault(model_name, []).append( + TfsInstanceStatus(tfs_rest_port, tfs_grpc_port, response["pid"]) + ) + + if response["status"] != falcon.HTTP_200: + log.info(f"Failed to load model : {model_name}") + is_load_successful = False + break + + if not is_load_successful: + log.info(f"Failed to load model : {model_name}, Starting to cleanup...") + self._delete_model(model_name) + self._remove_model_config(model_name) + else: + self._upload_mme_instance_status() + + res.status = response["status"] + res.body = response["body"] + + def _import_custom_modules(self, model_name): + inference_script_path = "/opt/ml/models/{}/model/code/inference.py".format(model_name) + python_lib_path = "/opt/ml/models/{}/model/code/lib".format(model_name) + if os.path.exists(python_lib_path): + log.info( + "Add Python code library for the model {} found at path {}.".format( + model_name, python_lib_path + ) + ) + sys.path.append(python_lib_path) + else: + log.info( + "Python code library for the model {} not found at path {}.".format( + model_name, python_lib_path + ) + ) + if os.path.exists(inference_script_path): + log.info( + "Importing handlers from model-specific inference script for the model {} found at path {}.".format( + model_name, inference_script_path + ) + ) + handler, input_handler, output_handler = self._import_handlers(inference_script_path) + model_handlers = self._make_handler(handler, input_handler, output_handler) + self.model_handlers[model_name] = model_handlers + else: + log.info( + "Model-specific inference script for the model {} not found at path {}.".format( + model_name, inference_script_path + ) + ) + + def _handle_invocation_post(self, req, res, model_name=None): + if SAGEMAKER_MULTI_MODEL_ENABLED: + if model_name: + if self._gunicorn_workers > 1: + if model_name not in self._mme_tfs_instances_status or not self._check_pid( + self._mme_tfs_instances_status[model_name][0].pid + ): + with lock(): + self._sync_local_mme_instance_status() + self._sync_model_handlers() + + if model_name not in self._mme_tfs_instances_status: + res.status = falcon.HTTP_404 + res.body = json.dumps( + {"error": "Model {} is not loaded yet.".format(model_name)} + ) + return + else: + log.info("model name: {}".format(model_name)) + rest_ports = [ + status.rest_port for status in self._mme_tfs_instances_status[model_name] + ] + rest_port = self._pick_port(rest_ports) + log.info("rest port: {}".format(str(rest_port))) + grpc_ports = [ + status.grpc_port for status in self._mme_tfs_instances_status[model_name] + ] + grpc_port = grpc_ports[rest_ports.index(rest_port)] + log.info("grpc port: {}".format(str(grpc_port))) + data, context = tfs_utils.parse_request( + req, + rest_port, + grpc_port, + self._tfs_default_model_name, + model_name=model_name, + ) + else: + res.status = falcon.HTTP_400 + res.body = json.dumps({"error": "Invocation request does not contain model name."}) + return + else: + # Randomly pick port used for routing incoming request. + grpc_port = self._pick_port(self._tfs_grpc_ports) + rest_port = self._pick_port(self._tfs_rest_ports) + data, context = tfs_utils.parse_request( + req, + rest_port, + grpc_port, + self._tfs_default_model_name, + channel=self._channels[grpc_port], + ) + + try: + res.status = falcon.HTTP_200 + handlers = self._handlers + if SAGEMAKER_MULTI_MODEL_ENABLED and model_name in self.model_handlers: + log.info( + "Model-specific inference script for the model {} exists, importing handlers.".format( + model_name + ) + ) + handlers = self.model_handlers[model_name] + elif not self._default_handlers_enabled: + log.info( + "Universal inference script exists at path {}, importing handlers.".format( + INFERENCE_SCRIPT_PATH + ) + ) + else: + log.info( + "Model-specific inference script and universal inference script both do not exist, using default handlers." + ) + res.body, res.content_type = handlers(data, context) + except Exception as e: # pylint: disable=broad-except + log.exception("exception handling request: {}".format(e)) + res.status = falcon.HTTP_500 + res.body = json.dumps({"error": str(e)}).encode("utf-8") # pylint: disable=E1101 + + def _setup_channel(self, grpc_port): + if grpc_port not in self._channels: + log.info("Creating grpc channel for port: %s", grpc_port) + self._channels[grpc_port] = grpc.insecure_channel("localhost:{}".format(grpc_port)) + + def _import_handlers(self, inference_script=INFERENCE_SCRIPT_PATH): + spec = importlib.util.spec_from_file_location("inference", inference_script) + inference = importlib.util.module_from_spec(spec) + spec.loader.exec_module(inference) + + _custom_handler, _custom_input_handler, _custom_output_handler = None, None, None + if hasattr(inference, "handler"): + _custom_handler = inference.handler + elif hasattr(inference, "input_handler") and hasattr(inference, "output_handler"): + _custom_input_handler = inference.input_handler + _custom_output_handler = inference.output_handler + else: + raise NotImplementedError("Handlers are not implemented correctly in user script.") + + return _custom_handler, _custom_input_handler, _custom_output_handler + + def _make_handler(self, custom_handler, custom_input_handler, custom_output_handler): + if custom_handler: + return custom_handler + + def handler(data, context): + processed_input = custom_input_handler(data, context) + response = requests.post(context.rest_uri, data=processed_input) + return custom_output_handler(response, context) + + return handler + + def on_get(self, req, res, model_name=None): # pylint: disable=W0613 + with lock(): + self._sync_local_mme_instance_status() + if model_name is None: + models_info = {} + uri = "http://localhost:{}/v1/models/{}" + for model, tfs_instance_status in self._mme_tfs_instances_status.items(): + try: + info = json.loads( + requests.get( + uri.format(tfs_instance_status[0].rest_port, model) + ).content + ) + models_info[model] = info + except ValueError as e: + log.exception("exception handling request: {}".format(e)) + res.status = falcon.HTTP_500 + res.body = json.dumps({"error": str(e)}).encode("utf-8") + res.status = falcon.HTTP_200 + res.body = json.dumps(models_info) + else: + if model_name not in self._mme_tfs_instances_status: + res.status = falcon.HTTP_404 + res.body = json.dumps( + {"error": "Model {} is loaded yet.".format(model_name)} + ).encode("utf-8") + else: + port = self._mme_tfs_instances_status[model_name].rest_port + uri = "http://localhost:{}/v1/models/{}".format(port, model_name) + try: + info = requests.get(uri) + res.status = falcon.HTTP_200 + res.body = json.dumps({"model": info}).encode("utf-8") + except ValueError as e: + log.exception("exception handling GET models request.") + res.status = falcon.HTTP_500 + res.body = json.dumps({"error": str(e)}).encode("utf-8") + + def on_delete(self, req, res, model_name): # pylint: disable=W0613 + with lock(): + self._sync_local_mme_instance_status() + if model_name not in self._mme_tfs_instances_status: + res.status = falcon.HTTP_404 + res.body = json.dumps({"error": "Model {} is not loaded yet".format(model_name)}) + else: + try: + self._delete_model(model_name) + self._remove_model_config(model_name) + del self._mme_tfs_instances_status[model_name] + self._upload_mme_instance_status() + res.status = falcon.HTTP_200 + res.body = json.dumps( + {"success": "Successfully unloaded model {}.".format(model_name)} + ) + except OSError as error: + res.status = falcon.HTTP_500 + res.body = json.dumps({"error": str(error)}).encode("utf-8") + + def _delete_model(self, model_name): + if model_name not in self._mme_tfs_instances_status: + return + for tfs_status in self._mme_tfs_instances_status[model_name]: + os.kill(tfs_status.pid, signal.SIGKILL) + + def _remove_model_config(self, model_name): + shutil.rmtree("/sagemaker/tfs-config/{}".format(model_name), ignore_errors=True) + shutil.rmtree("/sagemaker/batching/{}".format(model_name), ignore_errors=True) + + def validate_model_dir(self, model_path): + # model base path doesn't exits + if not os.path.exists(model_path): + return False + versions = [] + for _, dirs, _ in os.walk(model_path): + for dirname in dirs: + if dirname.isdigit(): + versions.append(dirname) + return self.validate_model_versions(versions) + + def validate_model_versions(self, versions): + if not versions: + return False + for v in versions: + if v.isdigit(): + # TensorFlow model server will succeed with any versions found + # even if there are directories that's not a valid model version, + # the loading will succeed. + return True + return False + + def _upload_mme_instance_status(self): + log.info( + "uploaded mme instance status file with content: {}".format( + self._mme_tfs_instances_status + ) + ) + with open(MME_TFS_INSTANCE_STATUS_FILE, "wb") as handle: + pickle.dump(self._mme_tfs_instances_status, handle, protocol=pickle.HIGHEST_PROTOCOL) + + def _sync_local_mme_instance_status(self): + if not os.path.exists(MME_TFS_INSTANCE_STATUS_FILE): + log.info("mme instance status file does not found.") + return + with open(MME_TFS_INSTANCE_STATUS_FILE, "rb") as handle: + self._mme_tfs_instances_status = pickle.load(handle) + log.info( + "updated local mme instance status with content: {}".format( + self._mme_tfs_instances_status + ) + ) + + def _sync_model_handlers(self): + for model_name, _ in self._mme_tfs_instances_status.items(): + if model_name not in self.model_handlers: + self._import_custom_modules(model_name) + + def _check_pid(self, pid): + """Check For the existence of a unix pid.""" + try: + os.kill(pid, 0) + except OSError: + return False + else: + return True + + +class PingResource: + def on_get(self, req, res): # pylint: disable=W0613 + res.status = falcon.HTTP_200 + + +class ServiceResources: + def __init__(self): + self._enable_model_manager = SAGEMAKER_MULTI_MODEL_ENABLED + self._python_service_resource = PythonServiceResource() + self._ping_resource = PingResource() + + def add_routes(self, application): + application.add_route("/ping", self._ping_resource) + application.add_route("/invocations", self._python_service_resource) + + if self._enable_model_manager: + application.add_route("/models", self._python_service_resource) + application.add_route("/models/{model_name}", self._python_service_resource) + application.add_route("/models/{model_name}/invoke", self._python_service_resource) + + +app = falcon.API() +resources = ServiceResources() +resources.add_routes(app) + +if __name__ == "__main__": + # Define the command-line arguments + parser = argparse.ArgumentParser() + parser.add_argument( + "-b", "--bind", type=str, required=True, help="Specify a server socket to bind." + ) + parser.add_argument( + "-k", + "--worker-class", + type=str, + required=True, + choices=["sync", "eventlet", "gevent", "tornado", "gthread", "sync"], + help="The type of worker process to run", + ) + parser.add_argument("-c", "--chdir", type=str, required=True, help="Change root dir") + parser.add_argument( + "-w", + "--workers", + type=int, + required=True, + help="The number of worker processes. This number should generally be between 2-4 workers per core in the server.", + ) + parser.add_argument("-t", "--threads", type=int, required=True, help="The number of threads") + parser.add_argument("-l", "--log-level", type=str, required=True) + parser.add_argument("-o", "--timeout", type=int, required=True, help="Gunicorn timeout") + + # Parse the command-line arguments + args = parser.parse_args() + + # Create gunicorn options + options = { + "bind": args.bind, + "worker_class": args.worker_class, + "chdir": args.chdir, + "workers": args.workers, + "threads": args.threads, + "loglevel": args.log_level, + "timeout": args.timeout, + "raw_env": [ + f"TFS_GRPC_PORTS={TFS_GRPC_PORTS}", + f"TFS_REST_PORTS={TFS_REST_PORTS}", + f'SAGEMAKER_MULTI_MODEL={os.environ.get("SAGEMAKER_MULTI_MODEL")}', + f"SAGEMAKER_SAFE_PORT_RANGE={SAGEMAKER_TFS_PORT_RANGE}", + f'SAGEMAKER_TFS_WAIT_TIME_SECONDS={os.environ.get("SAGEMAKER_TFS_WAIT_TIME_SECONDS")}', + f'SAGEMAKER_TFS_INTER_OP_PARALLELISM={os.environ.get("SAGEMAKER_TFS_INTER_OP_PARALLELISM", 0)}', + f'SAGEMAKER_TFS_INTRA_OP_PARALLELISM={os.environ.get("SAGEMAKER_TFS_INTRA_OP_PARALLELISM", 0)}', + f'SAGEMAKER_TFS_INSTANCE_COUNT={os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", "1")}', + f'SAGEMAKER_GUNICORN_WORKERS={os.environ.get("SAGEMAKER_GUNICORN_WORKERS", "1")}', + ], + } + + from gunicorn.app.base import BaseApplication + + class StandaloneApplication(BaseApplication): + def __init__(self, app, options=None): + self.options = options or {} + self.application = app + super().__init__() + + def load_config(self): + config = { + key: value + for key, value in self.options.items() + if key in self.cfg.settings and value is not None + } + for key, value in config.items(): + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + + StandaloneApplication(app, options).run() diff --git a/scripts/tensorflow/inference/sagemaker/serve b/scripts/tensorflow/inference/sagemaker/serve new file mode 100755 index 000000000000..9fac6a93ab41 --- /dev/null +++ b/scripts/tensorflow/inference/sagemaker/serve @@ -0,0 +1,3 @@ +#!/bin/bash + +python3 /sagemaker/serve.py diff --git a/scripts/tensorflow/inference/sagemaker/serve.py b/scripts/tensorflow/inference/sagemaker/serve.py new file mode 100644 index 000000000000..33ff260bc80c --- /dev/null +++ b/scripts/tensorflow/inference/sagemaker/serve.py @@ -0,0 +1,522 @@ +# Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import boto3 +import logging +import os +import re +import signal +import subprocess +import tfs_utils + +from contextlib import contextmanager + +logging.basicConfig( + format="%(process)d %(asctime)s %(levelname)-8s %(message)s", force=True, level=logging.INFO +) +log = logging.getLogger(__name__) + +JS_PING = "js_content tensorflowServing.ping" +JS_INVOCATIONS = "js_content tensorflowServing.invocations" +GUNICORN_PING = "proxy_pass http://gunicorn_upstream/ping" +GUNICORN_INVOCATIONS = "proxy_pass http://gunicorn_upstream/invocations" +CODE_DIR = ( + "/opt/ml/code" + if os.environ.get("SAGEMAKER_MULTI_MODEL", "False").lower() == "true" + else "/opt/ml/model/code" +) +PYTHON_LIB_PATH = os.path.join(CODE_DIR, "lib") +REQUIREMENTS_PATH = os.path.join(CODE_DIR, "requirements.txt") +INFERENCE_PATH = os.path.join(CODE_DIR, "inference.py") + + +class ServiceManager(object): + def __init__(self): + self._state = "initializing" + self._nginx = None + self._tfs = [] + self._gunicorn = None + self._gunicorn_command = None + self._gunicorn_env = None + self._enable_python_service = False + self._tfs_version = os.environ.get("SAGEMAKER_TFS_VERSION", "1.13") + self._nginx_http_port = os.environ.get("SAGEMAKER_BIND_TO_PORT", "8080") + self._nginx_loglevel = os.environ.get("SAGEMAKER_TFS_NGINX_LOGLEVEL", "error") + self._tfs_default_model_name = os.environ.get("SAGEMAKER_TFS_DEFAULT_MODEL_NAME", "None") + self._sagemaker_port_range = os.environ.get("SAGEMAKER_SAFE_PORT_RANGE", None) + self._gunicorn_workers = os.environ.get("SAGEMAKER_GUNICORN_WORKERS", 1) + self._gunicorn_threads = os.environ.get("SAGEMAKER_GUNICORN_THREADS", 1) + self._gunicorn_loglevel = os.environ.get("SAGEMAKER_GUNICORN_LOGLEVEL", "info") + self._tfs_config_path = "/sagemaker/model-config.cfg" + self._tfs_batching_config_path = "/sagemaker/batching-config.cfg" + + _enable_batching = os.environ.get("SAGEMAKER_TFS_ENABLE_BATCHING", "false").lower() + _enable_multi_model_endpoint = os.environ.get("SAGEMAKER_MULTI_MODEL", "false").lower() + # Use this to specify memory that is needed to initialize CUDA/cuDNN and other GPU libraries + self._tfs_gpu_margin = float(os.environ.get("SAGEMAKER_TFS_FRACTIONAL_GPU_MEM_MARGIN", 0.2)) + self._tfs_instance_count = int(os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", 1)) + self._tfs_wait_time_seconds = int( + os.environ.get("SAGEMAKER_TFS_WAIT_TIME_SECONDS", 55 // self._tfs_instance_count) + ) + self._tfs_inter_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTER_OP_PARALLELISM", 0) + self._tfs_intra_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTRA_OP_PARALLELISM", 0) + self._gunicorn_worker_class = os.environ.get("SAGEMAKER_GUNICORN_WORKER_CLASS", "gevent") + self._gunicorn_timeout_seconds = int( + os.environ.get("SAGEMAKER_GUNICORN_TIMEOUT_SECONDS", 30) + ) + self._nginx_proxy_read_timeout_seconds = int( + os.environ.get("SAGEMAKER_NGINX_PROXY_READ_TIMEOUT_SECONDS", 60) + ) + + # Nginx proxy read timeout should not be less than the GUnicorn timeout. If it is, this + # can result in upstream time out errors. + if self._gunicorn_timeout_seconds > self._nginx_proxy_read_timeout_seconds: + log.info( + "GUnicorn timeout was higher than Nginx proxy read timeout." + " Setting Nginx proxy read timeout from {} seconds to {} seconds" + " to match GUnicorn timeout.".format( + self._nginx_proxy_read_timeout_seconds, self._gunicorn_timeout_seconds + ) + ) + self._nginx_proxy_read_timeout_seconds = self._gunicorn_timeout_seconds + + if os.environ.get("OMP_NUM_THREADS") is None: + os.environ["OMP_NUM_THREADS"] = "1" + + if _enable_multi_model_endpoint not in ["true", "false"]: + raise ValueError("SAGEMAKER_MULTI_MODEL must be 'true' or 'false'") + self._tfs_enable_multi_model_endpoint = _enable_multi_model_endpoint == "true" + + self._need_python_service() + log.info("PYTHON SERVICE: {}".format(str(self._enable_python_service))) + + if _enable_batching not in ["true", "false"]: + raise ValueError("SAGEMAKER_TFS_ENABLE_BATCHING must be 'true' or 'false'") + self._tfs_enable_batching = _enable_batching == "true" + + if _enable_multi_model_endpoint not in ["true", "false"]: + raise ValueError("SAGEMAKER_MULTI_MODEL must be 'true' or 'false'") + self._tfs_enable_multi_model_endpoint = _enable_multi_model_endpoint == "true" + + self._use_gunicorn = self._enable_python_service or self._tfs_enable_multi_model_endpoint + + if self._sagemaker_port_range is not None: + parts = self._sagemaker_port_range.split("-") + low = int(parts[0]) + hi = int(parts[1]) + self._tfs_grpc_ports = [] + self._tfs_rest_ports = [] + if low + 2 * self._tfs_instance_count > hi: + raise ValueError( + "not enough ports available in SAGEMAKER_SAFE_PORT_RANGE ({})".format( + self._sagemaker_port_range + ) + ) + # select non-overlapping grpc and rest ports based on tfs instance count + for i in range(self._tfs_instance_count): + self._tfs_grpc_ports.append(str(low + 2 * i)) + self._tfs_rest_ports.append(str(low + 2 * i + 1)) + # concat selected ports respectively in order to pass them to python service + self._tfs_grpc_concat_ports = self._concat_ports(self._tfs_grpc_ports) + self._tfs_rest_concat_ports = self._concat_ports(self._tfs_rest_ports) + else: + # just use the standard default ports + self._tfs_grpc_ports = ["9000"] + self._tfs_rest_ports = ["8501"] + # provide single concat port here for default case + self._tfs_grpc_concat_ports = "9000" + self._tfs_rest_concat_ports = "8501" + + # set environment variable for python service + os.environ["TFS_GRPC_PORTS"] = self._tfs_grpc_concat_ports + os.environ["TFS_REST_PORTS"] = self._tfs_rest_concat_ports + + def _need_python_service(self): + if ( + os.path.exists(INFERENCE_PATH) + or os.path.exists(REQUIREMENTS_PATH) + or os.path.exists(PYTHON_LIB_PATH) + ): + self._enable_python_service = True + if os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_BUCKET") and os.environ.get( + "SAGEMAKER_MULTI_MODEL_UNIVERSAL_PREFIX" + ): + self._enable_python_service = True + + def _concat_ports(self, ports): + str_ports = [str(port) for port in ports] + concat_str_ports = ",".join(str_ports) + return concat_str_ports + + def _create_tfs_config(self): + models = tfs_utils.find_models() + + if not models: + raise ValueError("no SavedModel bundles found!") + + if self._tfs_default_model_name == "None": + default_model = os.path.basename(models[0]) + if default_model: + self._tfs_default_model_name = default_model + log.info("using default model name: {}".format(self._tfs_default_model_name)) + else: + log.info("no default model detected") + + # config (may) include duplicate 'config' keys, so we can't just dump a dict + config = "model_config_list: {\n" + for m in models: + config += " config: {\n" + config += " name: '{}'\n".format(os.path.basename(m)) + config += " base_path: '{}'\n".format(m) + config += " model_platform: 'tensorflow'\n" + + config += " model_version_policy: {\n" + config += " specific: {\n" + for version in tfs_utils.find_model_versions(m): + config += " versions: {}\n".format(version) + config += " }\n" + config += " }\n" + + config += " }\n" + config += "}\n" + + log.info("tensorflow serving model config: \n%s\n", config) + + with open(self._tfs_config_path, "w", encoding="utf8") as f: + f.write(config) + + def _setup_gunicorn(self): + python_path_content = [] + python_path_option = "" + + bucket = os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_BUCKET", None) + prefix = os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_PREFIX", None) + + if not os.path.exists(CODE_DIR) and bucket and prefix: + self._download_scripts(bucket, prefix) + + if self._enable_python_service: + lib_path_exists = os.path.exists(PYTHON_LIB_PATH) + requirements_exists = os.path.exists(REQUIREMENTS_PATH) + python_path_content = ["/opt/ml/model/code"] + python_path_option = "--pythonpath " + + if lib_path_exists: + python_path_content.append(PYTHON_LIB_PATH) + + if requirements_exists: + if lib_path_exists: + log.warning( + "loading modules in '{}', ignoring requirements.txt".format(PYTHON_LIB_PATH) + ) + else: + log.info("installing packages from requirements.txt...") + pip_install_cmd = "pip3 install -r {}".format(REQUIREMENTS_PATH) + try: + subprocess.check_call(pip_install_cmd.split()) + except subprocess.CalledProcessError: + log.error("failed to install required packages, exiting.") + self._stop() + raise ChildProcessError("failed to install required packages.") + + gunicorn_command = ( + "python3 /sagemaker/python_service.py -b unix:/tmp/gunicorn.sock -k {} --chdir /sagemaker " + "--workers {} --threads {} --log-level {} --timeout {} " + ).format( + self._gunicorn_worker_class, + self._gunicorn_workers, + self._gunicorn_threads, + self._gunicorn_loglevel, + self._gunicorn_timeout_seconds, + ) + + log.info("gunicorn command: {}".format(gunicorn_command)) + self._gunicorn_command = gunicorn_command + gunicorn_env = { + "TFS_GRPC_PORTS": self._tfs_grpc_concat_ports, + "TFS_REST_PORTS": self._tfs_rest_concat_ports, + "SAGEMAKER_MULTI_MODEL": str(self._tfs_enable_multi_model_endpoint), + "SAGEMAKER_TFS_WAIT_TIME_SECONDS": str(self._tfs_wait_time_seconds), + "SAGEMAKER_TFS_INTER_OP_PARALLELISM": str(self._tfs_inter_op_parallelism), + "SAGEMAKER_TFS_INTRA_OP_PARALLELISM": str(self._tfs_intra_op_parallelism), + "SAGEMAKER_TFS_INSTANCE_COUNT": str(self._tfs_instance_count), + "PYTHONPATH": ":".join(python_path_content), + "SAGEMAKER_GUNICORN_WORKERS": str(self._gunicorn_workers), + } + if self._sagemaker_port_range is not None: + gunicorn_env["SAGEMAKER_SAFE_PORT_RANGE"] = self._sagemaker_port_range + log.info(f"gunicorn env: {gunicorn_env}") + self._gunicorn_env = gunicorn_env + + def _download_scripts(self, bucket, prefix): + log.info("checking boto session region ...") + boto_session = boto3.session.Session() + boto_region = boto_session.region_name + if boto_region in ("us-iso-east-1", "us-gov-west-1"): + raise ValueError("Universal scripts is not supported in us-iso-east-1 or us-gov-west-1") + + log.info("downloading universal scripts ...") + client = boto3.client("s3") + resource = boto3.resource("s3") + # download files + paginator = client.get_paginator("list_objects") + for result in paginator.paginate(Bucket=bucket, Delimiter="/", Prefix=prefix): + for file in result.get("Contents", []): + destination = os.path.join(CODE_DIR, file.get("Key").split("/")[-1]) + if not os.path.exists(os.path.dirname(destination)): + os.makedirs(os.path.dirname(destination)) + resource.meta.client.download_file(bucket, file.get("Key"), destination) + + def _create_nginx_tfs_upstream(self): + indentation = " " + tfs_upstream = "" + for port in self._tfs_rest_ports: + tfs_upstream += "{}server localhost:{};\n".format(indentation, port) + tfs_upstream = tfs_upstream[len(indentation) : -2] + + return tfs_upstream + + def _create_nginx_config(self): + template = self._read_nginx_template() + pattern = re.compile(r"%(\w+)%") + + template_values = { + "TFS_VERSION": self._tfs_version, + "TFS_UPSTREAM": self._create_nginx_tfs_upstream(), + "TFS_DEFAULT_MODEL_NAME": self._tfs_default_model_name, + "NGINX_HTTP_PORT": self._nginx_http_port, + "NGINX_LOG_LEVEL": self._nginx_loglevel, + "FORWARD_PING_REQUESTS": GUNICORN_PING if self._use_gunicorn else JS_PING, + "FORWARD_INVOCATION_REQUESTS": ( + GUNICORN_INVOCATIONS if self._use_gunicorn else JS_INVOCATIONS + ), + "PROXY_READ_TIMEOUT": str(self._nginx_proxy_read_timeout_seconds), + } + + config = pattern.sub(lambda x: template_values[x.group(1)], template) + log.info("nginx config: \n%s\n", config) + + with open("/sagemaker/nginx.conf", "w", encoding="utf8") as f: + f.write(config) + + def _read_nginx_template(self): + with open("/sagemaker/nginx.conf.template", "r", encoding="utf8") as f: + template = f.read() + if not template: + raise ValueError("failed to read nginx.conf.template") + + return template + + def _enable_per_process_gpu_memory_fraction(self): + nvidia_smi_exist = os.path.exists("/usr/bin/nvidia-smi") + if self._tfs_instance_count > 1 and nvidia_smi_exist: + return True + + return False + + def _get_number_of_gpu_on_host(self): + nvidia_smi_exist = os.path.exists("/usr/bin/nvidia-smi") + if nvidia_smi_exist: + return len( + subprocess.check_output(["nvidia-smi", "-L"]).decode("utf-8").strip().split("\n") + ) + return 0 + + def _calculate_per_process_gpu_memory_fraction(self): + return round((1 - self._tfs_gpu_margin) / float(self._tfs_instance_count), 4) + + def _start_tfs(self): + self._log_version("tensorflow_model_server --version", "tensorflow version info:") + + for i in range(self._tfs_instance_count): + p = self._start_single_tfs(i) + self._tfs.append(p) + + def _start_gunicorn(self): + self._log_version("gunicorn --version", "gunicorn version info:") + env = os.environ.copy() + env["TFS_DEFAULT_MODEL_NAME"] = self._tfs_default_model_name + env.update(self._gunicorn_env) + p = subprocess.Popen(self._gunicorn_command.split(), env=env) + log.info("started gunicorn (pid: %d)", p.pid) + self._gunicorn = p + + def _start_nginx(self): + self._log_version("/usr/sbin/nginx -V", "nginx version info:") + p = subprocess.Popen("/usr/sbin/nginx -c /sagemaker/nginx.conf".split()) + log.info("started nginx (pid: %d)", p.pid) + self._nginx = p + + def _log_version(self, command, message): + try: + output = ( + subprocess.check_output(command.split(), stderr=subprocess.STDOUT) + .decode("utf-8", "backslashreplace") + .strip() + ) + log.info("{}\n{}".format(message, output)) + except subprocess.CalledProcessError: + log.warning("failed to run command: %s", command) + + def _stop(self, *args): # pylint: disable=W0613 + self._state = "stopping" + log.info("stopping services") + try: + os.kill(self._nginx.pid, signal.SIGQUIT) + except OSError: + pass + try: + if self._gunicorn: + os.kill(self._gunicorn.pid, signal.SIGTERM) + except OSError: + pass + try: + for tfs in self._tfs: + os.kill(tfs.pid, signal.SIGTERM) + except OSError: + pass + + self._state = "stopped" + log.info("stopped") + + def _wait_for_gunicorn(self): + while True: + if os.path.exists("/tmp/gunicorn.sock"): + log.info("gunicorn server is ready!") + return + + def _wait_for_tfs(self): + for i in range(self._tfs_instance_count): + tfs_utils.wait_for_model( + self._tfs_rest_ports[i], self._tfs_default_model_name, self._tfs_wait_time_seconds + ) + + @contextmanager + def _timeout(self, seconds): + def _raise_timeout_error(signum, frame): + raise TimeoutError("time out after {} seconds".format(seconds)) + + try: + signal.signal(signal.SIGALRM, _raise_timeout_error) + signal.alarm(seconds) + yield + finally: + signal.alarm(0) + + def _is_tfs_process(self, pid): + for p in self._tfs: + if p.pid == pid: + return True + return False + + def _find_tfs_process(self, pid): + for index, p in enumerate(self._tfs): + if p.pid == pid: + return index + return None + + def _restart_single_tfs(self, pid): + instance_id = self._find_tfs_process(pid) + if instance_id is None: + raise ValueError("Cannot find tfs with pid: {};".format(pid)) + p = self._start_single_tfs(instance_id) + self._tfs[instance_id] = p + + def _start_single_tfs(self, instance_id): + cmd = tfs_utils.tfs_command( + self._tfs_grpc_ports[instance_id], + self._tfs_rest_ports[instance_id], + self._tfs_config_path, + self._tfs_enable_batching, + self._tfs_batching_config_path, + tfs_intra_op_parallelism=self._tfs_intra_op_parallelism, + tfs_inter_op_parallelism=self._tfs_inter_op_parallelism, + tfs_enable_gpu_memory_fraction=self._enable_per_process_gpu_memory_fraction(), + tfs_gpu_memory_fraction=self._calculate_per_process_gpu_memory_fraction(), + ) + log.info("tensorflow serving command: {}".format(cmd)) + + num_gpus = self._get_number_of_gpu_on_host() + if num_gpus > 1: + # utilizing multi-gpu + worker_env = os.environ.copy() + worker_env["CUDA_VISIBLE_DEVICES"] = str(instance_id % num_gpus) + p = subprocess.Popen(cmd.split(), env=worker_env) + log.info( + "started tensorflow serving (pid: {}) on GPU: {}".format( + p.pid, instance_id % num_gpus + ) + ) + else: + # cpu and single gpu + p = subprocess.Popen(cmd.split()) + log.info("started tensorflow serving (pid: {})".format(p.pid)) + + return p + + def _monitor(self): + while True: + pid, status = os.wait() + + if self._state != "started": + break + + if pid == self._nginx.pid: + log.warning("unexpected nginx exit (status: {}). restarting.".format(status)) + self._start_nginx() + + elif self._is_tfs_process(pid): + log.warning( + "unexpected tensorflow serving exit (status: {}). restarting.".format(status) + ) + try: + self._restart_single_tfs(pid) + except (ValueError, OSError) as error: + log.error("Failed to restart tensorflow serving. {}".format(error)) + + elif self._gunicorn and pid == self._gunicorn.pid: + log.warning("unexpected gunicorn exit (status: {}). restarting.".format(status)) + self._start_gunicorn() + + def start(self): + log.info("starting services") + self._state = "starting" + signal.signal(signal.SIGTERM, self._stop) + + if self._tfs_enable_batching: + log.info("batching is enabled") + tfs_utils.create_batching_config(self._tfs_batching_config_path) + + if self._tfs_enable_multi_model_endpoint: + log.info("multi-model endpoint is enabled, TFS model servers will be started later") + else: + self._create_tfs_config() + self._start_tfs() + self._wait_for_tfs() + + self._create_nginx_config() + + if self._use_gunicorn: + self._setup_gunicorn() + self._start_gunicorn() + # make sure gunicorn is up + with self._timeout(seconds=self._gunicorn_timeout_seconds): + self._wait_for_gunicorn() + + self._start_nginx() + self._state = "started" + self._monitor() + self._stop() + + +if __name__ == "__main__": + ServiceManager().start() diff --git a/scripts/tensorflow/inference/sagemaker/tensorflowServing.js b/scripts/tensorflow/inference/sagemaker/tensorflowServing.js new file mode 100644 index 000000000000..65732ac8b386 --- /dev/null +++ b/scripts/tensorflow/inference/sagemaker/tensorflowServing.js @@ -0,0 +1,239 @@ +var tfs_base_uri = '/tfs/v1/models/' +var custom_attributes_header = 'X-Amzn-SageMaker-Custom-Attributes' + +function invocations(r) { + var ct = r.headersIn['Content-Type'] + + if ('application/json' == ct || 'application/jsonlines' == ct || 'application/jsons' == ct) { + json_request(r) + } else if ('text/csv' == ct) { + csv_request(r) + } else { + return_error(r, 415, 'Unsupported Media Type: ' + (ct || 'Unknown')) + } +} + +function ping(r) { + var uri = make_tfs_uri(r, false) + + function callback (reply) { + if (reply.status == 200 && reply.responseText.includes('"AVAILABLE"')) { + r.return(200) + } else { + r.error('failed ping' + reply.responseText) + r.return(502) + } + } + + r.subrequest(uri, callback) +} + +function ping_without_model(r) { + // hack for TF 1.11 and MME + // for TF 1.11, send an arbitrary fixed request to the default model. + // if response is 400, the model is ok (but input was bad), so return 200 + // for MME, the default model name is None and does not exist + // also return 200 in unlikely case our request was really valid + + var uri = make_tfs_uri(r, true) + var options = { + method: 'POST', + body: '{"instances": "invalid"}' + } + + function callback (reply) { + if (reply.status == 200 || reply.status == 400 || + reply.responseText.includes('Servable not found for request: Latest(None)')) { + r.return(200) + } else { + r.error('failed ping' + reply.responseText) + r.return(502) + } + } + + r.subrequest(uri, options, callback) +} + +function return_error(r, code, message) { + if (message) { + r.return(code, '{"error": "' + message + '"}') + } else { + r.return(code) + } +} + +function tfs_json_request(r, json) { + var uri = make_tfs_uri(r, true) + var options = { + method: 'POST', + body: json + } + + var accept = r.headersIn.Accept + function callback (reply) { + var body = reply.responseText + if (reply.status == 400) { + // "fix" broken json escaping in \'instances\' message + body = body.replace("\\'instances\\'", "'instances'") + } + + if (accept != undefined) { + var content_types = accept.trim().replace(" ", "").split(",") + if (content_types.includes('application/jsonlines') || content_types.includes('application/json')) { + body = body.replace(/\n/g, '') + r.headersOut['Content-Type'] = content_types[0] + } + } + r.return(reply.status, body) + } + + r.subrequest(uri, options, callback) + +} + +function make_tfs_uri(r, with_method) { + var attributes = parse_custom_attributes(r) + + var uri = tfs_base_uri + attributes['tfs-model-name'] + if ('tfs-model-version' in attributes) { + uri += '/versions/' + attributes['tfs-model-version'] + } + + if (with_method) { + uri += ':' + (attributes['tfs-method'] || 'predict') + } + + return uri +} + +function parse_custom_attributes(r) { + var attributes = {} + var kv_pattern = /tfs-[a-z\-]+=[^,]+/g + var header = r.headersIn[custom_attributes_header] + if (header) { + var matches = header.match(kv_pattern) + if (matches) { + for (var i = 0; i < matches.length; i++) { + var kv = matches[i].split('=') + if (kv.length === 2) { + attributes[kv[0]] = kv[1] + } + } + } + } + + // for MME invocations, tfs-model-name is in the uri, or use default_tfs_model + if (!attributes['tfs-model-name']) { + var uri_pattern = /\/models\/[^,]+\/invoke/g + var model_name = r.uri.match(uri_pattern) + if (model_name[0]) { + model_name = r.uri.replace('/models/', '').replace('/invoke', '') + attributes['tfs-model-name'] = model_name + } else { + attributes['tfs-model-name'] = r.variables.default_tfs_model + } + } + + return attributes +} + +function json_request(r) { + var data = r.requestText + + if (is_tfs_json(data)) { + tfs_json_request(r, data) + } else if (is_json_lines(data)) { + json_lines_request(r, data) + } else { + generic_json_request(r, data) + } +} + +function is_tfs_json(data) { + return /"(instances|inputs|examples)"\s*:/.test(data) +} + +function is_json_lines(data) { + // objects separated only by (optional) whitespace means jsons/json-lines + return /[}\]]\s*[\[{]/.test(data) +} + +function generic_json_request(r, data) { + if (! /^\s*\[\s*\[/.test(data)) { + data = '[' + data + ']' + } + + var json = '{"instances":' + data + '}' + tfs_json_request(r, json) +} + +function json_lines_request(r, data) { + var lines = data.trim().split(/\r?\n/) + var builder = [] + builder.push('{"instances":') + if (lines.length != 1) { + builder.push('[') + } + + for (var i = 0; i < lines.length; i++) { + var line = lines[i].trim() + if (line) { + var instance = (i == 0) ? '' : ',' + instance += line + builder.push(instance) + } + } + + builder.push(lines.length == 1 ? '}' : ']}') + tfs_json_request(r, builder.join('')) +} + +function csv_request(r) { + var data = r.requestText + // look for initial quote or numeric-only data in 1st field + var needs_quotes = data.search(/^\s*("|[\d.Ee+\-]+.*)/) != 0 + var lines = data.trim().split(/\r?\n/) + var builder = [] + builder.push('{"instances":[') + + for (var i = 0; i < lines.length; i++) { + var line = lines[i].trim() + if (line) { + var line_builder = [] + // Only wrap line in brackets if there are multiple columns. + // If there's only one column and it has a string with a comma, + // the input will be wrapped in an extra set of brackets. + var has_multiple_columns = line.search(',') != -1 + + if (has_multiple_columns) { + line_builder.push('[') + } + + if (needs_quotes) { + line_builder.push('"') + line_builder.push(line.replace('"', '\\"').replace(',', '","')) + line_builder.push('"') + } else { + line_builder.push(line) + } + + if (has_multiple_columns) { + line_builder.push(']') + } + + var json_line = line_builder.join('') + builder.push(json_line) + + if (i != lines.length - 1) + builder.push(',') + } + } + + builder.push(']}') + tfs_json_request(r, builder.join('')) +} + +export default {invocations, ping, ping_without_model, return_error, + tfs_json_request, make_tfs_uri, parse_custom_attributes, + json_request, is_tfs_json, is_json_lines, generic_json_request, + json_lines_request, csv_request}; diff --git a/scripts/tensorflow/inference/sagemaker/tfs_utils.py b/scripts/tensorflow/inference/sagemaker/tfs_utils.py new file mode 100644 index 000000000000..566f15d23508 --- /dev/null +++ b/scripts/tensorflow/inference/sagemaker/tfs_utils.py @@ -0,0 +1,337 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging +import multiprocessing +import os +import re +import requests +import json +import time + +from multi_model_utils import timeout +from urllib3.util.retry import Retry +from urllib3.exceptions import NewConnectionError, MaxRetryError +from collections import namedtuple +from multi_model_utils import MultiModelException + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + +DEFAULT_CONTENT_TYPE = "application/json" +DEFAULT_ACCEPT_HEADER = "application/json" +CUSTOM_ATTRIBUTES_HEADER = "X-Amzn-SageMaker-Custom-Attributes" + +Context = namedtuple( + "Context", + "model_name, model_version, method, rest_uri, grpc_port, channel, " + "custom_attributes, request_content_type, accept_header, content_length", +) + + +def parse_request(req, rest_port, grpc_port, default_model_name, model_name=None, channel=None): + tfs_attributes = parse_tfs_custom_attributes(req) + tfs_uri = make_tfs_uri(rest_port, tfs_attributes, default_model_name, model_name) + + if not model_name: + model_name = tfs_attributes.get("tfs-model-name") + + context = Context( + model_name, + tfs_attributes.get("tfs-model-version"), + tfs_attributes.get("tfs-method"), + tfs_uri, + grpc_port, + channel, + req.get_header(CUSTOM_ATTRIBUTES_HEADER), + req.get_header("Content-Type") or DEFAULT_CONTENT_TYPE, + req.get_header("Accept") or DEFAULT_ACCEPT_HEADER, + req.content_length, + ) + + data = req.stream + return data, context + + +def make_tfs_uri(port, attributes, default_model_name, model_name=None): + log.info("sagemaker tfs attributes: \n{}".format(attributes)) + + tfs_model_name = model_name or attributes.get("tfs-model-name", default_model_name) + tfs_model_version = attributes.get("tfs-model-version") + tfs_method = attributes.get("tfs-method", "predict") + + uri = "http://localhost:{}/v1/models/{}".format(port, tfs_model_name) + if tfs_model_version: + uri += "/versions/" + tfs_model_version + uri += ":" + tfs_method + return uri + + +def parse_tfs_custom_attributes(req): + attributes = {} + header = req.get_header(CUSTOM_ATTRIBUTES_HEADER) + if header: + matches = re.findall(r"(tfs-[a-z\-]+=[^,]+)", header) + attributes = dict(attribute.split("=") for attribute in matches) + return attributes + + +def create_tfs_config_individual_model(model_name, base_path): + config = "model_config_list: {\n" + config += " config: {\n" + config += " name: '{}'\n".format(model_name) + config += " base_path: '{}'\n".format(base_path) + config += " model_platform: 'tensorflow'\n" + + config += " model_version_policy: {\n" + config += " specific: {\n" + for version in find_model_versions(base_path): + config += " versions: {}\n".format(version) + config += " }\n" + config += " }\n" + + config += " }\n" + config += "}\n" + return config + + +def tfs_command( + tfs_grpc_port, + tfs_rest_port, + tfs_config_path, + tfs_enable_batching, + tfs_batching_config_file, + tfs_intra_op_parallelism=None, + tfs_inter_op_parallelism=None, + tfs_enable_gpu_memory_fraction=False, + tfs_gpu_memory_fraction=None, +): + cmd = ( + "tensorflow_model_server " + "--port={} " + "--rest_api_port={} " + "--model_config_file={} " + "--max_num_load_retries=0 {} {} {} {}".format( + tfs_grpc_port, + tfs_rest_port, + tfs_config_path, + get_tfs_batching_args(tfs_enable_batching, tfs_batching_config_file), + get_tensorflow_intra_op_parallelism_args(tfs_intra_op_parallelism), + get_tensorflow_inter_op_parallelism_args(tfs_inter_op_parallelism), + get_tfs_gpu_mem_args(tfs_enable_gpu_memory_fraction, tfs_gpu_memory_fraction), + ) + ) + return cmd + + +def find_models(): + base_path = "/opt/ml/model" + models = [] + for f in _find_saved_model_files(base_path): + parts = f.split("/") + if len(parts) >= 6 and re.match(r"^\d+$", parts[-2]): + model_path = "/".join(parts[0:-2]) + if model_path not in models: + models.append(model_path) + return models + + +def find_model_versions(model_path): + """Remove leading zeros from the version number, returns list of versions""" + return [ + version[:-1].lstrip("0") + version[-1] + for version in os.listdir(model_path) + if version.isnumeric() + ] + + +def _find_saved_model_files(path): + for e in os.scandir(path): + if e.is_dir(): + yield from _find_saved_model_files(os.path.join(path, e.name)) + else: + if e.name == "saved_model.pb": + yield os.path.join(path, e.name) + + +def get_tfs_batching_args(enable_batching, tfs_batching_config): + if enable_batching: + return "--enable_batching=true " "--batching_parameters_file={}".format(tfs_batching_config) + else: + return "" + + +def get_tensorflow_intra_op_parallelism_args(tfs_intra_op_parallelism): + if tfs_intra_op_parallelism: + return "--tensorflow_intra_op_parallelism={}".format(tfs_intra_op_parallelism) + else: + return "" + + +def get_tensorflow_inter_op_parallelism_args(tfs_inter_op_parallelism): + if tfs_inter_op_parallelism: + return "--tensorflow_inter_op_parallelism={}".format(tfs_inter_op_parallelism) + else: + return "" + + +def get_tfs_gpu_mem_args(enable_gpu_memory_fraction, gpu_memory_fraction): + if enable_gpu_memory_fraction and gpu_memory_fraction: + return "--per_process_gpu_memory_fraction={}".format(gpu_memory_fraction) + else: + return "" + + +def create_batching_config(batching_config_file): + class _BatchingParameter: + def __init__(self, key, env_var, value, defaulted_message): + self.key = key + self.env_var = env_var + self.value = value + self.defaulted_message = defaulted_message + + cpu_count = multiprocessing.cpu_count() + batching_parameters = [ + _BatchingParameter( + "max_batch_size", + "SAGEMAKER_TFS_MAX_BATCH_SIZE", + 8, + "max_batch_size defaulted to {}. Set {} to override default. " + "Tuning this parameter may yield better performance.", + ), + _BatchingParameter( + "batch_timeout_micros", + "SAGEMAKER_TFS_BATCH_TIMEOUT_MICROS", + 1000, + "batch_timeout_micros defaulted to {}. Set {} to override " + "default. Tuning this parameter may yield better performance.", + ), + _BatchingParameter( + "num_batch_threads", + "SAGEMAKER_TFS_NUM_BATCH_THREADS", + cpu_count, + "num_batch_threads defaulted to {}," "the number of CPUs. Set {} to override default.", + ), + _BatchingParameter( + "max_enqueued_batches", + "SAGEMAKER_TFS_MAX_ENQUEUED_BATCHES", + # Batch limits number of concurrent requests, which limits number + # of enqueued batches, so this can be set high for Batch + 100000000 if "SAGEMAKER_BATCH" in os.environ else cpu_count, + "max_enqueued_batches defaulted to {}. Set {} to override default. " + "Tuning this parameter may be necessary to tune out-of-memory " + "errors occur.", + ), + ] + + warning_message = "" + for batching_parameter in batching_parameters: + if batching_parameter.env_var in os.environ: + batching_parameter.value = os.environ[batching_parameter.env_var] + else: + warning_message += batching_parameter.defaulted_message.format( + batching_parameter.value, batching_parameter.env_var + ) + warning_message += "\n" + if warning_message: + log.warning(warning_message) + + config = "" + for batching_parameter in batching_parameters: + config += "%s { value: %s }\n" % (batching_parameter.key, batching_parameter.value) + + log.info("batching config: \n%s\n", config) + with open(batching_config_file, "w", encoding="utf8") as f: + f.write(config) + + +def wait_for_model(rest_port, model_name, timeout_seconds, pid=None): + """ + Notice: + The calculation for retry count based on timeout_seconds might introduce a small delta (0.1s) for each retry + which might cause total timeout longer than timeout_seconds + """ + tfs_url = "http://localhost:{}/v1/models/{}".format(rest_port, model_name) + start = time.time() + try: + session = requests.Session() + backoff_factor = 0.1 + # sleep = {backoff factor} * (2 ^ ({number of retries so far} - 1)) + retry_count = retry_from_timeout(timeout_seconds, backoff_factor) + retries = Retry(total=retry_count, backoff_factor=backoff_factor) + session.mount("http://", requests.adapters.HTTPAdapter(max_retries=retries)) + log.info( + "Trying to connect with model server: {} with timeout : {} and retry : {}".format( + tfs_url, timeout_seconds, retry_count + ) + ) + response = session.get(tfs_url, timeout=0.1) + log.info( + f"tfs response status_code: {response.status_code} with content : {json.loads(response.content)}" + ) + end = time.time() + if response.status_code == 200: + if is_model_ready(response): + return + elif wait_for_model_ready(tfs_url, timeout_seconds - int(end - start)): + return + + raise MultiModelException(408, "Timed out after {} seconds".format(timeout_seconds), pid) + except ( + ConnectionRefusedError, + NewConnectionError, + MaxRetryError, + requests.exceptions.ConnectionError, + ): + raise MultiModelException(408, "Timed out after {} seconds".format(timeout_seconds), pid) + + +def is_model_ready(response): + versions = json.loads(response.content)["model_version_status"] + if all(version["state"] == "AVAILABLE" for version in versions): + return True + return False + + +def wait_for_model_ready(url, timeout_seconds): + try: + while timeout_seconds > 0: + response = requests.get(url, timeout=0.1) + log.info( + f"wait_for_model_ready response status_code : {response.status_code} " + f"response : {json.loads(response.content)} timeout in : {timeout_seconds}s" + ) + if response.status_code != 200: + return False + if is_model_ready(response): + return True + timeout_seconds -= 1 + return False + except requests.exceptions.RequestException: + return False + + +def retry_from_timeout(timeout_seconds, backoff_factor): + retry_count = 1 + retry_time = 0 + while retry_time < timeout_seconds: + retry_count += 1 + retry_time += backoff_factor * (2 ** (retry_count + 1)) + return retry_count + + +def get_cpu_memory_util(): + total_memory, used_memory, free_memory = map( + int, os.popen("free -t -m").readlines()[-1].split()[1:] + ) + return round((used_memory / total_memory) * 100, 2) diff --git a/scripts/tensorflow/inference/tf_serving_entrypoint.sh b/scripts/tensorflow/inference/tf_serving_entrypoint.sh new file mode 100755 index 000000000000..60d67f9a7515 --- /dev/null +++ b/scripts/tensorflow/inference/tf_serving_entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Execute telemetry script if it exists, suppress errors +bash /usr/local/bin/bash_telemetry.sh >/dev/null 2>&1 || true + +/usr/local/bin/tensorflow_model_server --port=8500 --rest_api_port=8501 --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME} "$@" diff --git a/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json b/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json new file mode 100644 index 000000000000..fe51488c7066 --- /dev/null +++ b/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json @@ -0,0 +1 @@ +[] diff --git a/test/tensorflow/integration/inference/.gitkeep b/test/tensorflow/integration/inference/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/tensorflow/integration/inference/__init__.py b/test/tensorflow/integration/inference/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/tensorflow/integration/inference/conftest.py b/test/tensorflow/integration/inference/conftest.py new file mode 100644 index 000000000000..0a30ef0692e3 --- /dev/null +++ b/test/tensorflow/integration/inference/conftest.py @@ -0,0 +1,107 @@ +"""Pytest fixtures for TF 2.20 inference integration tests on SageMaker. + +Fixtures intentionally defer all AWS calls until test-execution time so that +``pytest --collect-only`` works in environments without AWS credentials. +""" + +from __future__ import annotations + +import os +import time +from uuid import uuid4 + +import pytest + + +@pytest.fixture(scope="session") +def aws_region() -> str: + """AWS region for SageMaker operations. Defaults to us-west-2.""" + return os.environ.get("AWS_REGION", "us-west-2") + + +@pytest.fixture(scope="session") +def sagemaker_role_arn() -> str: + """SageMaker execution role ARN. Skips the test if not set.""" + arn = os.environ.get("SAGEMAKER_ROLE_ARN") + if not arn: + pytest.skip("SAGEMAKER_ROLE_ARN not set") + return arn + + +@pytest.fixture(scope="session") +def inference_image_uri() -> str: + """ECR URI for the TF 2.20 inference image under test. Skips if not set.""" + uri = os.environ.get("INFERENCE_IMAGE_URI") + if not uri: + pytest.skip("INFERENCE_IMAGE_URI not set") + return uri + + +@pytest.fixture(scope="session") +def boto_session(aws_region: str): + """A boto3 session bound to the configured region.""" + import boto3 + + return boto3.Session(region_name=aws_region) + + +@pytest.fixture(scope="session") +def sagemaker_session(boto_session): + """A SageMaker SDK session for high-level deploy/predict calls.""" + import sagemaker + + return sagemaker.Session(boto_session=boto_session) + + +@pytest.fixture +def unique_name(): + """Returns a callable producing collision-resistant resource names. + + Usage: + name = unique_name("tf220-single") + """ + + def _make(prefix: str) -> str: + return f"{prefix}-{int(time.time())}-{uuid4().hex[:6]}" + + return _make + + +@pytest.fixture +def cleanup_endpoint(sagemaker_session): + """Yield-style fixture that tears down endpoint, endpoint config, and model. + + Usage: + def test_x(cleanup_endpoint, ...): + cleanup_endpoint(endpoint_name, model_name=model_name) + # ... deploy + predict ... + """ + registered: list[dict] = [] + + def _register(endpoint_name: str, model_name: str | None = None) -> None: + registered.append({"endpoint_name": endpoint_name, "model_name": model_name}) + + yield _register + + sm_client = sagemaker_session.boto_session.client("sagemaker") + for item in registered: + endpoint_name = item["endpoint_name"] + model_name = item["model_name"] + + for delete_call, kwargs in ( + (sm_client.delete_endpoint, {"EndpointName": endpoint_name}), + (sm_client.delete_endpoint_config, {"EndpointConfigName": endpoint_name}), + ): + try: + delete_call(**kwargs) + except sm_client.exceptions.ClientError: + # Swallow NotFound / already-deleted; teardown should be best-effort. + pass + except Exception: + pass + + if model_name: + try: + sm_client.delete_model(ModelName=model_name) + except Exception: + pass diff --git a/test/tensorflow/integration/inference/requirements.txt b/test/tensorflow/integration/inference/requirements.txt new file mode 100644 index 000000000000..61a934efe5b2 --- /dev/null +++ b/test/tensorflow/integration/inference/requirements.txt @@ -0,0 +1,5 @@ +pytest>=8.0.0 +sagemaker>=3.4.0 +boto3>=1.34.0 +numpy>=1.26.4 +tensorflow-serving-api==2.20.0 diff --git a/test/tensorflow/integration/inference/resources/__init__.py b/test/tensorflow/integration/inference/resources/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/tensorflow/integration/inference/resources/build_sample_model.py b/test/tensorflow/integration/inference/resources/build_sample_model.py new file mode 100644 index 000000000000..9d6020160ad9 --- /dev/null +++ b/test/tensorflow/integration/inference/resources/build_sample_model.py @@ -0,0 +1,60 @@ +"""Build a tiny TensorFlow SavedModel and tar it for SageMaker deployment. + +The model performs ``y = x * multiplier`` for a runtime-supplied multiplier. +Tarballs are written to a caller-provided directory; nothing is checked in. +""" + +from __future__ import annotations + +import os +import tarfile +import tempfile +from pathlib import Path + + +def build_sample_model( + output_dir: str | os.PathLike | None = None, + multiplier: float = 2.0, + model_name: str = "model", + tar_filename: str = "model.tar.gz", +) -> str: + """Build a SavedModel that multiplies its input by ``multiplier``. + + The SavedModel is laid out under ``/export/Servo/1`` so that + SageMaker's TensorFlow Serving handler can discover the version directory. + Returns the absolute path to the produced ``model.tar.gz``. + """ + import tensorflow as tf + + output_dir = Path(output_dir) if output_dir else Path(tempfile.mkdtemp(prefix="tf220-sample-")) + output_dir.mkdir(parents=True, exist_ok=True) + + # SageMaker TFS expects: model.tar.gz -> //saved_model.pb + saved_model_dir = output_dir / model_name / "1" + saved_model_dir.mkdir(parents=True, exist_ok=True) + + multiplier_const = tf.constant(multiplier, dtype=tf.float32) + + class MultiplierModel(tf.Module): + @tf.function(input_signature=[tf.TensorSpec(shape=[None, None], dtype=tf.float32)]) + def __call__(self, x): + return {"output": x * multiplier_const} + + model = MultiplierModel() + tf.saved_model.save( + model, + str(saved_model_dir), + signatures={"serving_default": model.__call__}, + ) + + tar_path = output_dir / tar_filename + with tarfile.open(tar_path, "w:gz") as tar: + # Archive the model_name dir as the top-level entry inside the tarball. + tar.add(str(output_dir / model_name), arcname=model_name) + + return str(tar_path) + + +if __name__ == "__main__": + path = build_sample_model() + print(path) diff --git a/test/tensorflow/integration/inference/test_multi_model_endpoint.py b/test/tensorflow/integration/inference/test_multi_model_endpoint.py new file mode 100644 index 000000000000..2d7a03d0f6fd --- /dev/null +++ b/test/tensorflow/integration/inference/test_multi_model_endpoint.py @@ -0,0 +1,129 @@ +"""Multi-model endpoint (MME) integration test for TF 2.20 inference DLC. + +Builds two tiny SavedModels (``y = 2x`` and ``y = 3x``), uploads both to a +shared S3 prefix, deploys a SageMaker MME backed by the v2 inference image, +and asserts that ``TargetModel`` routes invocations correctly. +""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +import pytest + +from .resources.build_sample_model import build_sample_model + + +INSTANCE_TYPE = "ml.c5.xlarge" + + +def _decode(response) -> dict: + """Normalize an MME runtime invoke response Body into a Python dict.""" + body = response["Body"].read() if hasattr(response, "get") else response + if isinstance(body, (bytes, bytearray)): + body = body.decode("utf-8") + return json.loads(body) + + +def _values_from_predictions(predictions) -> list: + """Pull the numeric output list out of either signature-keyed or raw rows.""" + assert predictions and isinstance(predictions, list) + first = predictions[0] + if isinstance(first, dict) and "output" in first: + return first["output"] + return first + + +def test_mme_two_models( + sagemaker_session, + sagemaker_role_arn, + inference_image_uri, + unique_name, + cleanup_endpoint, +): + from sagemaker.multidatamodel import MultiDataModel + from sagemaker.tensorflow.serving import TensorFlowModel + + with tempfile.TemporaryDirectory(prefix="tf220-mme-") as workdir: + workdir_path = Path(workdir) + + # Build two models with different multipliers, each in its own subdir + # so build_sample_model doesn't collide on the SavedModel layout. + model1_dir = workdir_path / "m1" + model2_dir = workdir_path / "m2" + model1_tar = build_sample_model( + output_dir=model1_dir, multiplier=2.0, model_name="model", tar_filename="model1.tar.gz" + ) + model2_tar = build_sample_model( + output_dir=model2_dir, multiplier=3.0, model_name="model", tar_filename="model2.tar.gz" + ) + + bucket = sagemaker_session.default_bucket() + run_id = unique_name("mme") + s3_key_prefix = f"tf220-inference-tests/mme-models/{run_id}" + + # Upload each tarball under the shared MME prefix so the runtime can + # resolve TargetModel relative to the same S3 location. + sagemaker_session.upload_data( + path=model1_tar, bucket=bucket, key_prefix=s3_key_prefix + ) + sagemaker_session.upload_data( + path=model2_tar, bucket=bucket, key_prefix=s3_key_prefix + ) + s3_model_prefix = f"s3://{bucket}/{s3_key_prefix}/" + + endpoint_name = unique_name("tf220-mme") + base_model_name = unique_name("tf220-mme-model") + cleanup_endpoint(endpoint_name, model_name=base_model_name) + + # The TensorFlowModel acts as the container template; MultiDataModel + # reuses its image_uri + execution role for the endpoint config. + base_model = TensorFlowModel( + model_data=model1_tar, # placeholder; MME ignores model_data at runtime + role=sagemaker_role_arn, + image_uri=inference_image_uri, + sagemaker_session=sagemaker_session, + name=base_model_name, + ) + + mme = MultiDataModel( + name=base_model_name, + model_data_prefix=s3_model_prefix, + model=base_model, + sagemaker_session=sagemaker_session, + ) + + mme.deploy( + initial_instance_count=1, + instance_type=INSTANCE_TYPE, + endpoint_name=endpoint_name, + ) + + runtime = sagemaker_session.boto_session.client("sagemaker-runtime") + payload = json.dumps({"instances": [[1.0, 2.0, 3.0]]}) + + # Invoke model1 (x * 2.0) + resp1 = runtime.invoke_endpoint( + EndpointName=endpoint_name, + ContentType="application/json", + TargetModel="model1.tar.gz", + Body=payload, + ) + body1 = _decode(resp1) + assert "predictions" in body1, f"model1 response missing predictions: {body1!r}" + values1 = _values_from_predictions(body1["predictions"]) + assert values1 == pytest.approx([2.0, 4.0, 6.0]), f"model1 got {values1!r}" + + # Invoke model2 (x * 3.0) + resp2 = runtime.invoke_endpoint( + EndpointName=endpoint_name, + ContentType="application/json", + TargetModel="model2.tar.gz", + Body=payload, + ) + body2 = _decode(resp2) + assert "predictions" in body2, f"model2 response missing predictions: {body2!r}" + values2 = _values_from_predictions(body2["predictions"]) + assert values2 == pytest.approx([3.0, 6.0, 9.0]), f"model2 got {values2!r}" diff --git a/test/tensorflow/integration/inference/test_single_model_endpoint.py b/test/tensorflow/integration/inference/test_single_model_endpoint.py new file mode 100644 index 000000000000..8826b61aa0e0 --- /dev/null +++ b/test/tensorflow/integration/inference/test_single_model_endpoint.py @@ -0,0 +1,89 @@ +"""Single-model endpoint integration test for TF 2.20 inference DLC. + +Builds a tiny ``y = 2x`` SavedModel, deploys it to a single-instance SageMaker +endpoint backed by the v2 inference image under test, and asserts the +predicted values. +""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +import pytest + +from .resources.build_sample_model import build_sample_model + + +INSTANCE_TYPE = "ml.c5.xlarge" + + +def test_single_model_predict( + sagemaker_session, + sagemaker_role_arn, + inference_image_uri, + unique_name, + cleanup_endpoint, +): + from sagemaker.tensorflow.serving import TensorFlowModel + + with tempfile.TemporaryDirectory(prefix="tf220-single-") as workdir: + tar_path = build_sample_model( + output_dir=workdir, + multiplier=2.0, + model_name="model", + ) + + bucket = sagemaker_session.default_bucket() + key_prefix = f"tf220-inference-tests/{Path(tar_path).stem}-{unique_name('single')}" + model_data = sagemaker_session.upload_data( + path=tar_path, + bucket=bucket, + key_prefix=key_prefix, + ) + + endpoint_name = unique_name("tf220-single") + model_name = unique_name("tf220-single-model") + cleanup_endpoint(endpoint_name, model_name=model_name) + + tf_model = TensorFlowModel( + model_data=model_data, + role=sagemaker_role_arn, + image_uri=inference_image_uri, + sagemaker_session=sagemaker_session, + name=model_name, + ) + + predictor = tf_model.deploy( + initial_instance_count=1, + instance_type=INSTANCE_TYPE, + endpoint_name=endpoint_name, + ) + + try: + payload = {"instances": [[1.0, 2.0, 3.0]]} + response = predictor.predict(payload) + + # The TFS predictor may return a dict already, or a JSON string — + # normalize both shapes. + if isinstance(response, (bytes, str)): + response = json.loads(response) + + assert "predictions" in response, f"missing predictions key in {response!r}" + predictions = response["predictions"] + assert predictions and isinstance(predictions, list) + + # Output signature is {"output": x * 2.0} -> TFS surfaces the tensor + # under the signature output key when there is a single named tensor; + # some TFS versions instead return the raw list. Handle both. + first = predictions[0] + if isinstance(first, dict) and "output" in first: + values = first["output"] + else: + values = first + + assert values == pytest.approx([2.0, 4.0, 6.0]), f"got {values!r}" + finally: + # cleanup_endpoint teardown handles resources; no manual delete needed. + pass From 8ad193cb197952f59bedea4dba0bc58cda871f8a Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 11:21:39 -0700 Subject: [PATCH 02/11] chore: apply pre-commit fixes Run pre-commit hooks (ruff-format, ruff-check, requirements-txt-fixer, trailing-whitespace) on Phase 1-6 files. Mechanical reformatting plus noqa annotations for intentional E402 imports after gevent monkey-patch in python_service.py and one F841 unused variable in serve.py (preserved as-is from master TF 2.19 vendored handler). --- .../inference/sagemaker/nginx.conf.template | 3 +- .../inference/sagemaker/python_service.py | 50 +++++++++---------- .../tensorflow/inference/sagemaker/serve.py | 8 +-- .../inference/sagemaker/tfs_utils.py | 15 +++--- .../integration/inference/requirements.txt | 4 +- .../inference/test_multi_model_endpoint.py | 9 +--- .../inference/test_single_model_endpoint.py | 1 - 7 files changed, 40 insertions(+), 50 deletions(-) diff --git a/scripts/tensorflow/inference/sagemaker/nginx.conf.template b/scripts/tensorflow/inference/sagemaker/nginx.conf.template index 3951fe96b939..a975069e57d2 100644 --- a/scripts/tensorflow/inference/sagemaker/nginx.conf.template +++ b/scripts/tensorflow/inference/sagemaker/nginx.conf.template @@ -17,7 +17,7 @@ http { access_log /dev/stdout combined; js_import tensorflowServing.js; - proxy_read_timeout %PROXY_READ_TIMEOUT%; + proxy_read_timeout %PROXY_READ_TIMEOUT%; upstream tfs_upstream { %TFS_UPSTREAM%; @@ -64,4 +64,3 @@ http { keepalive_timeout 3; } } - \ No newline at end of file diff --git a/scripts/tensorflow/inference/sagemaker/python_service.py b/scripts/tensorflow/inference/sagemaker/python_service.py index 0cb2dc9ecabf..fb7fd708da11 100644 --- a/scripts/tensorflow/inference/sagemaker/python_service.py +++ b/scripts/tensorflow/inference/sagemaker/python_service.py @@ -16,26 +16,24 @@ gevent.monkey.patch_all() -import bisect -import argparse -import importlib.util -import json -import logging -import os -import signal -import subprocess -import grpc -import sys -import shutil -import copy -import pickle - -import falcon -import requests -import random - -from multi_model_utils import MultiModelException, lock -import tfs_utils +import argparse # noqa: E402 +import copy # noqa: E402 +import importlib.util # noqa: E402 +import json # noqa: E402 +import logging # noqa: E402 +import os # noqa: E402 +import pickle # noqa: E402 +import random # noqa: E402 +import shutil # noqa: E402 +import signal # noqa: E402 +import subprocess # noqa: E402 +import sys # noqa: E402 + +import falcon # noqa: E402 +import grpc # noqa: E402 +import requests # noqa: E402 +import tfs_utils # noqa: E402 +from multi_model_utils import MultiModelException, lock # noqa: E402 SAGEMAKER_MULTI_MODEL_ENABLED = os.environ.get("SAGEMAKER_MULTI_MODEL", "false").lower() == "true" INFERENCE_SCRIPT_PATH = ( @@ -656,13 +654,13 @@ def add_routes(self, application): "raw_env": [ f"TFS_GRPC_PORTS={TFS_GRPC_PORTS}", f"TFS_REST_PORTS={TFS_REST_PORTS}", - f'SAGEMAKER_MULTI_MODEL={os.environ.get("SAGEMAKER_MULTI_MODEL")}', + f"SAGEMAKER_MULTI_MODEL={os.environ.get('SAGEMAKER_MULTI_MODEL')}", f"SAGEMAKER_SAFE_PORT_RANGE={SAGEMAKER_TFS_PORT_RANGE}", - f'SAGEMAKER_TFS_WAIT_TIME_SECONDS={os.environ.get("SAGEMAKER_TFS_WAIT_TIME_SECONDS")}', - f'SAGEMAKER_TFS_INTER_OP_PARALLELISM={os.environ.get("SAGEMAKER_TFS_INTER_OP_PARALLELISM", 0)}', - f'SAGEMAKER_TFS_INTRA_OP_PARALLELISM={os.environ.get("SAGEMAKER_TFS_INTRA_OP_PARALLELISM", 0)}', - f'SAGEMAKER_TFS_INSTANCE_COUNT={os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", "1")}', - f'SAGEMAKER_GUNICORN_WORKERS={os.environ.get("SAGEMAKER_GUNICORN_WORKERS", "1")}', + f"SAGEMAKER_TFS_WAIT_TIME_SECONDS={os.environ.get('SAGEMAKER_TFS_WAIT_TIME_SECONDS')}", + f"SAGEMAKER_TFS_INTER_OP_PARALLELISM={os.environ.get('SAGEMAKER_TFS_INTER_OP_PARALLELISM', 0)}", + f"SAGEMAKER_TFS_INTRA_OP_PARALLELISM={os.environ.get('SAGEMAKER_TFS_INTRA_OP_PARALLELISM', 0)}", + f"SAGEMAKER_TFS_INSTANCE_COUNT={os.environ.get('SAGEMAKER_TFS_INSTANCE_COUNT', '1')}", + f"SAGEMAKER_GUNICORN_WORKERS={os.environ.get('SAGEMAKER_GUNICORN_WORKERS', '1')}", ], } diff --git a/scripts/tensorflow/inference/sagemaker/serve.py b/scripts/tensorflow/inference/sagemaker/serve.py index 33ff260bc80c..577095c8a8b4 100644 --- a/scripts/tensorflow/inference/sagemaker/serve.py +++ b/scripts/tensorflow/inference/sagemaker/serve.py @@ -10,16 +10,16 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import boto3 import logging import os import re import signal import subprocess -import tfs_utils - from contextlib import contextmanager +import boto3 +import tfs_utils + logging.basicConfig( format="%(process)d %(asctime)s %(levelname)-8s %(message)s", force=True, level=logging.INFO ) @@ -208,7 +208,7 @@ def _setup_gunicorn(self): lib_path_exists = os.path.exists(PYTHON_LIB_PATH) requirements_exists = os.path.exists(REQUIREMENTS_PATH) python_path_content = ["/opt/ml/model/code"] - python_path_option = "--pythonpath " + python_path_option = "--pythonpath " # noqa: F841 if lib_path_exists: python_path_content.append(PYTHON_LIB_PATH) diff --git a/scripts/tensorflow/inference/sagemaker/tfs_utils.py b/scripts/tensorflow/inference/sagemaker/tfs_utils.py index 566f15d23508..22493fd2b1a7 100644 --- a/scripts/tensorflow/inference/sagemaker/tfs_utils.py +++ b/scripts/tensorflow/inference/sagemaker/tfs_utils.py @@ -11,19 +11,18 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import json import logging import multiprocessing import os import re -import requests -import json import time - -from multi_model_utils import timeout -from urllib3.util.retry import Retry -from urllib3.exceptions import NewConnectionError, MaxRetryError from collections import namedtuple + +import requests from multi_model_utils import MultiModelException +from urllib3.exceptions import MaxRetryError, NewConnectionError +from urllib3.util.retry import Retry logging.basicConfig(level=logging.INFO) log = logging.getLogger(__name__) @@ -166,7 +165,7 @@ def _find_saved_model_files(path): def get_tfs_batching_args(enable_batching, tfs_batching_config): if enable_batching: - return "--enable_batching=true " "--batching_parameters_file={}".format(tfs_batching_config) + return "--enable_batching=true --batching_parameters_file={}".format(tfs_batching_config) else: return "" @@ -220,7 +219,7 @@ def __init__(self, key, env_var, value, defaulted_message): "num_batch_threads", "SAGEMAKER_TFS_NUM_BATCH_THREADS", cpu_count, - "num_batch_threads defaulted to {}," "the number of CPUs. Set {} to override default.", + "num_batch_threads defaulted to {},the number of CPUs. Set {} to override default.", ), _BatchingParameter( "max_enqueued_batches", diff --git a/test/tensorflow/integration/inference/requirements.txt b/test/tensorflow/integration/inference/requirements.txt index 61a934efe5b2..370334526e6f 100644 --- a/test/tensorflow/integration/inference/requirements.txt +++ b/test/tensorflow/integration/inference/requirements.txt @@ -1,5 +1,5 @@ -pytest>=8.0.0 -sagemaker>=3.4.0 boto3>=1.34.0 numpy>=1.26.4 +pytest>=8.0.0 +sagemaker>=3.4.0 tensorflow-serving-api==2.20.0 diff --git a/test/tensorflow/integration/inference/test_multi_model_endpoint.py b/test/tensorflow/integration/inference/test_multi_model_endpoint.py index 2d7a03d0f6fd..6be7a454a305 100644 --- a/test/tensorflow/integration/inference/test_multi_model_endpoint.py +++ b/test/tensorflow/integration/inference/test_multi_model_endpoint.py @@ -15,7 +15,6 @@ from .resources.build_sample_model import build_sample_model - INSTANCE_TYPE = "ml.c5.xlarge" @@ -66,12 +65,8 @@ def test_mme_two_models( # Upload each tarball under the shared MME prefix so the runtime can # resolve TargetModel relative to the same S3 location. - sagemaker_session.upload_data( - path=model1_tar, bucket=bucket, key_prefix=s3_key_prefix - ) - sagemaker_session.upload_data( - path=model2_tar, bucket=bucket, key_prefix=s3_key_prefix - ) + sagemaker_session.upload_data(path=model1_tar, bucket=bucket, key_prefix=s3_key_prefix) + sagemaker_session.upload_data(path=model2_tar, bucket=bucket, key_prefix=s3_key_prefix) s3_model_prefix = f"s3://{bucket}/{s3_key_prefix}/" endpoint_name = unique_name("tf220-mme") diff --git a/test/tensorflow/integration/inference/test_single_model_endpoint.py b/test/tensorflow/integration/inference/test_single_model_endpoint.py index 8826b61aa0e0..be480c43f11a 100644 --- a/test/tensorflow/integration/inference/test_single_model_endpoint.py +++ b/test/tensorflow/integration/inference/test_single_model_endpoint.py @@ -15,7 +15,6 @@ from .resources.build_sample_model import build_sample_model - INSTANCE_TYPE = "ml.c5.xlarge" From a48a279a0b541190212adacc5454d5886bb3285c Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 11:48:17 -0700 Subject: [PATCH 03/11] chore: apply dockerfmt formatting to TF 2.20 inference Dockerfiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-indent RUN/ENV continuation lines from 4-space to 2-space per dockerfmt's normalization. Mechanical reformatting only — no logic changes. Closes the dockerfmt CI failure on PR #6243. --- .../tensorflow/inference/2.20/Dockerfile.cpu | 90 +++++++++--------- .../tensorflow/inference/2.20/Dockerfile.cuda | 94 +++++++++---------- 2 files changed, 92 insertions(+), 92 deletions(-) diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cpu b/docker/tensorflow/inference/2.20/Dockerfile.cpu index 27566aecc723..6ce6f365aba5 100644 --- a/docker/tensorflow/inference/2.20/Dockerfile.cpu +++ b/docker/tensorflow/inference/2.20/Dockerfile.cpu @@ -34,10 +34,10 @@ FROM amazonlinux:2023 AS builder-base ARG PYTHON_VERSION RUN dnf install -y --allowerasing \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \ - gcc gcc-c++ make cmake git openssl-devel ninja-build \ - tar xz curl wget \ - && dnf clean all + python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \ + gcc gcc-c++ make cmake git openssl-devel ninja-build \ + tar xz curl wget \ + && dnf clean all COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv @@ -57,7 +57,7 @@ RUN dnf install -y --allowerasing python${PYTHON_VERSION} curl && dnf clean all COPY --from=builder-base /opt/venv /opt/venv COPY scripts/common/setup_oss_compliance.sh /tmp/setup_oss_compliance.sh RUN PATH="/opt/venv/bin:${PATH}" bash /tmp/setup_oss_compliance.sh python${PYTHON_VERSION} \ - && touch /root/THIRD_PARTY_SOURCE_CODE_URLS + && touch /root/THIRD_PARTY_SOURCE_CODE_URLS # ── Stage: builder-njs (compile nginx + njs dynamic module from source) ───── @@ -69,26 +69,26 @@ ARG NGINX_VERSION ARG NJS_VERSION RUN dnf install -y --allowerasing \ - gcc gcc-c++ make \ - pcre-devel pcre2-devel zlib-devel openssl-devel \ - curl tar xz \ - && dnf clean all + gcc gcc-c++ make \ + pcre-devel pcre2-devel zlib-devel openssl-devel \ + curl tar xz \ + && dnf clean all WORKDIR /tmp/njs-build RUN curl -fsSLO "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ - && tar xzf nginx-${NGINX_VERSION}.tar.gz \ - && curl -fsSL "https://github.com/nginx/njs/archive/refs/tags/${NJS_VERSION}.tar.gz" -o njs-${NJS_VERSION}.tar.gz \ - && tar xzf njs-${NJS_VERSION}.tar.gz + && tar xzf nginx-${NGINX_VERSION}.tar.gz \ + && curl -fsSL "https://github.com/nginx/njs/archive/refs/tags/${NJS_VERSION}.tar.gz" -o njs-${NJS_VERSION}.tar.gz \ + && tar xzf njs-${NJS_VERSION}.tar.gz RUN cd nginx-${NGINX_VERSION} \ - && ./configure \ + && ./configure \ --with-compat \ --add-dynamic-module=/tmp/njs-build/njs-${NJS_VERSION}/nginx \ - && make -j$(nproc) modules \ - && mkdir -p /out/modules \ - && cp objs/ngx_http_js_module.so /out/modules/ \ - && cp objs/ngx_stream_js_module.so /out/modules/ + && make -j$(nproc) modules \ + && mkdir -p /out/modules \ + && cp objs/ngx_http_js_module.so /out/modules/ \ + && cp objs/ngx_stream_js_module.so /out/modules/ # ── Stage: runtime-base (shared base for output stages) ───────────────────── @@ -107,23 +107,23 @@ LABEL framework="tensorflow" LABEL framework_version="${TF_SERVING_VERSION}" ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONIOENCODING=UTF-8 \ - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - DLC_CONTAINER_TYPE=inference \ - MODEL_BASE_PATH=/models \ - MODEL_NAME=model + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + DLC_CONTAINER_TYPE=inference \ + MODEL_BASE_PATH=/models \ + MODEL_NAME=model # Runtime system deps via dnf — equivalent of master TF 2.19 CPU apt-get block. RUN dnf install -y --allowerasing \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel \ - nginx \ - gcc gcc-c++ make git \ - tar gzip xz which findutils util-linux \ - libpng-devel freetype-devel zlib-devel \ - openssl unzip jq curl wget \ - && dnf clean all + python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel \ + nginx \ + gcc gcc-c++ make git \ + tar gzip xz which findutils util-linux \ + libpng-devel freetype-devel zlib-devel \ + openssl unzip jq curl wget \ + && dnf clean all # Copy venv from builder-base COPY --from=builder-base /opt/venv /opt/venv @@ -137,7 +137,7 @@ COPY --from=build_image /usr/local/bin/tensorflow_model_server /usr/local/bin/te # python symlink — some TF tooling expects /usr/local/bin/python. RUN ln -sf $(which python${PYTHON_VERSION}) /usr/local/bin/python \ - && ln -sf $(which pip3) /usr/local/bin/pip + && ln -sf $(which pip3) /usr/local/bin/pip ENV PATH="/opt/venv/bin:${PATH}" ENV LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" @@ -172,7 +172,7 @@ RUN mkdir -p /opt/ml/input/data /opt/ml/model /opt/ml/output /opt/ml/code # tensorflow-serving-api — installed inline with --no-deps; see locked # decision Q1 in cpu/pyproject.toml header. RUN /opt/venv/bin/uv pip install --no-deps --no-cache "tensorflow-serving-api==${TF_SERVING_VERSION}" 2>/dev/null \ - || /opt/venv/bin/pip install --no-deps --no-cache-dir "tensorflow-serving-api==${TF_SERVING_VERSION}" + || /opt/venv/bin/pip install --no-deps --no-cache-dir "tensorflow-serving-api==${TF_SERVING_VERSION}" # SageMaker handler artifacts (TFS toolkit ported in Phase 3 from master TF 2.19 # build_artifacts/sagemaker/: serve/serve.py, python_service.py, tfs_utils.py, @@ -185,21 +185,21 @@ COPY scripts/telemetry/bash_telemetry.sh.template /tmp/bash_telemetry.sh.templat ARG FRAMEWORK="tensorflow" ARG CONTAINER_TYPE="inference" RUN chmod +x /usr/local/bin/deep_learning_container.py \ - && sed -e "s/{{FRAMEWORK}}/${FRAMEWORK}/g" \ - -e "s/{{FRAMEWORK_VERSION}}/${TF_SERVING_VERSION}/g" \ - -e "s/{{CONTAINER_TYPE}}/${CONTAINER_TYPE}/g" \ - /tmp/bash_telemetry.sh.template >/usr/local/bin/bash_telemetry.sh \ - && chmod +x /usr/local/bin/bash_telemetry.sh \ - && rm /tmp/bash_telemetry.sh.template + && sed -e "s/{{FRAMEWORK}}/${FRAMEWORK}/g" \ + -e "s/{{FRAMEWORK_VERSION}}/${TF_SERVING_VERSION}/g" \ + -e "s/{{CONTAINER_TYPE}}/${CONTAINER_TYPE}/g" \ + /tmp/bash_telemetry.sh.template >/usr/local/bin/bash_telemetry.sh \ + && chmod +x /usr/local/bin/bash_telemetry.sh \ + && rm /tmp/bash_telemetry.sh.template # Security patch — run after all installers so every OS package is covered. RUN dnf upgrade -y --security --releasever latest \ - && dnf upgrade -y libcurl libcurl-minimal --refresh \ - && dnf clean all + && dnf upgrade -y libcurl libcurl-minimal --refresh \ + && dnf clean all # Telemetry bashrc hook — MUST be after `dnf upgrade --security`. RUN echo 'source /usr/local/bin/bash_telemetry.sh' >>/etc/bashrc \ - && echo 'source /usr/local/bin/bash_telemetry.sh' >>/root/.bashrc + && echo 'source /usr/local/bin/bash_telemetry.sh' >>/root/.bashrc # OSS compliance COPY --from=builder-oss /root/THIRD_PARTY_SOURCE_CODE_URLS /root/THIRD_PARTY_SOURCE_CODE_URLS @@ -215,8 +215,8 @@ COPY scripts/tensorflow/inference/dockerd_entrypoint.sh /usr/local/bin/dockerd_e COPY scripts/tensorflow/inference/tf_serving_entrypoint.sh /usr/local/bin/tf_serving_entrypoint.sh COPY scripts/common/start_cuda_compat.sh /usr/local/bin/start_cuda_compat.sh RUN chmod +x /usr/local/bin/dockerd_entrypoint.sh \ - /usr/local/bin/tf_serving_entrypoint.sh \ - /usr/local/bin/start_cuda_compat.sh + /usr/local/bin/tf_serving_entrypoint.sh \ + /usr/local/bin/start_cuda_compat.sh RUN rm -rf /tmp/* /root/.cache @@ -224,4 +224,4 @@ RUN rm -rf /tmp/* /root/.cache EXPOSE 8500 8501 ENTRYPOINT ["bash", "-m", "/usr/local/bin/dockerd_entrypoint.sh"] -CMD ["/usr/local/bin/tf_serving_entrypoint.sh"] +CMD ["/usr/local/bin/tf_serving_entrypoint.sh"] \ No newline at end of file diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cuda b/docker/tensorflow/inference/2.20/Dockerfile.cuda index c24e9d916f1b..36d1c5708953 100644 --- a/docker/tensorflow/inference/2.20/Dockerfile.cuda +++ b/docker/tensorflow/inference/2.20/Dockerfile.cuda @@ -40,10 +40,10 @@ FROM nvidia/cuda:${CUDA_VERSION}-devel-amzn2023 AS builder-base ARG PYTHON_VERSION RUN dnf install -y --allowerasing \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \ - gcc gcc-c++ make cmake git openssl-devel ninja-build \ - tar xz curl \ - && dnf clean all + python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \ + gcc gcc-c++ make cmake git openssl-devel ninja-build \ + tar xz curl \ + && dnf clean all COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv @@ -63,7 +63,7 @@ RUN dnf install -y --allowerasing python${PYTHON_VERSION} curl && dnf clean all COPY --from=builder-base /opt/venv /opt/venv COPY scripts/common/setup_oss_compliance.sh /tmp/setup_oss_compliance.sh RUN PATH="/opt/venv/bin:${PATH}" bash /tmp/setup_oss_compliance.sh python${PYTHON_VERSION} \ - && touch /root/THIRD_PARTY_SOURCE_CODE_URLS + && touch /root/THIRD_PARTY_SOURCE_CODE_URLS # ── Stage: builder-njs (compile nginx + njs dynamic module from source) ───── @@ -77,31 +77,31 @@ ARG NGINX_VERSION ARG NJS_VERSION RUN dnf install -y --allowerasing \ - gcc gcc-c++ make \ - pcre-devel pcre2-devel zlib-devel openssl-devel \ - curl tar xz \ - && dnf clean all + gcc gcc-c++ make \ + pcre-devel pcre2-devel zlib-devel openssl-devel \ + curl tar xz \ + && dnf clean all WORKDIR /tmp/njs-build # Fetch nginx + njs sources, both with checksum-validated release tarballs # from the upstream maintainer. RUN curl -fsSLO "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ - && tar xzf nginx-${NGINX_VERSION}.tar.gz \ - && curl -fsSL "https://github.com/nginx/njs/archive/refs/tags/${NJS_VERSION}.tar.gz" -o njs-${NJS_VERSION}.tar.gz \ - && tar xzf njs-${NJS_VERSION}.tar.gz + && tar xzf nginx-${NGINX_VERSION}.tar.gz \ + && curl -fsSL "https://github.com/nginx/njs/archive/refs/tags/${NJS_VERSION}.tar.gz" -o njs-${NJS_VERSION}.tar.gz \ + && tar xzf njs-${NJS_VERSION}.tar.gz # Configure nginx with the njs dynamic module, then build only the modules. # --with-compat is REQUIRED so the resulting .so loads into a stock nginx # binary (we install nginx via dnf in runtime-base; the .so must be ABI-compat). RUN cd nginx-${NGINX_VERSION} \ - && ./configure \ + && ./configure \ --with-compat \ --add-dynamic-module=/tmp/njs-build/njs-${NJS_VERSION}/nginx \ - && make -j$(nproc) modules \ - && mkdir -p /out/modules \ - && cp objs/ngx_http_js_module.so /out/modules/ \ - && cp objs/ngx_stream_js_module.so /out/modules/ + && make -j$(nproc) modules \ + && mkdir -p /out/modules \ + && cp objs/ngx_http_js_module.so /out/modules/ \ + && cp objs/ngx_stream_js_module.so /out/modules/ # ── Stage: runtime-base (shared base for output stages) ───────────────────── @@ -121,14 +121,14 @@ LABEL framework="tensorflow" LABEL framework_version="${TF_SERVING_VERSION}" ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONIOENCODING=UTF-8 \ - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - DLC_CONTAINER_TYPE=inference \ - CUDA_HOME=/usr/local/cuda \ - MODEL_BASE_PATH=/models \ - MODEL_NAME=model + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + DLC_CONTAINER_TYPE=inference \ + CUDA_HOME=/usr/local/cuda \ + MODEL_BASE_PATH=/models \ + MODEL_NAME=model # Runtime system deps via dnf — equivalent of master TF 2.19 GPU apt-get block, # minus the training-specific packages (NCCL/EFA/OpenMPI). nginx is installed @@ -136,13 +136,13 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ # into /usr/lib64/nginx/modules. tar/gzip/which/findutils/util-linux are # baseline shell utilities the SM handler scripts and TFS launcher rely on. RUN dnf install -y --allowerasing \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel \ - nginx \ - gcc gcc-c++ make git \ - tar gzip xz which findutils util-linux \ - libpng-devel freetype-devel zlib-devel \ - openssl unzip jq curl wget \ - && dnf clean all + python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel \ + nginx \ + gcc gcc-c++ make git \ + tar gzip xz which findutils util-linux \ + libpng-devel freetype-devel zlib-devel \ + openssl unzip jq curl wget \ + && dnf clean all # Copy venv from builder-base COPY --from=builder-base /opt/venv /opt/venv @@ -158,7 +158,7 @@ COPY --from=build_image /usr/local/bin/tensorflow_model_server /usr/local/bin/te # python symlink — some TF tooling expects /usr/local/bin/python. RUN ln -sf $(which python${PYTHON_VERSION}) /usr/local/bin/python \ - && ln -sf $(which pip3) /usr/local/bin/pip + && ln -sf $(which pip3) /usr/local/bin/pip ENV PATH="/opt/venv/bin:/usr/local/cuda/bin:${PATH}" ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/lib:${LD_LIBRARY_PATH}" @@ -198,7 +198,7 @@ RUN mkdir -p /opt/ml/input/data /opt/ml/model /opt/ml/output /opt/ml/code # locked decision Q1 in cuda/pyproject.toml header. The transitive runtime # needs (numpy / protobuf / grpcio) are already in the venv from builder-base. RUN /opt/venv/bin/uv pip install --no-deps --no-cache "tensorflow-serving-api-gpu==${TF_SERVING_VERSION}" 2>/dev/null \ - || /opt/venv/bin/pip install --no-deps --no-cache-dir "tensorflow-serving-api-gpu==${TF_SERVING_VERSION}" + || /opt/venv/bin/pip install --no-deps --no-cache-dir "tensorflow-serving-api-gpu==${TF_SERVING_VERSION}" # SageMaker handler artifacts (TFS toolkit ported in Phase 3 from master TF 2.19 # build_artifacts/sagemaker/: serve/serve.py, python_service.py, tfs_utils.py, @@ -211,26 +211,26 @@ COPY scripts/telemetry/bash_telemetry.sh.template /tmp/bash_telemetry.sh.templat ARG FRAMEWORK="tensorflow" ARG CONTAINER_TYPE="inference" RUN chmod +x /usr/local/bin/deep_learning_container.py \ - && sed -e "s/{{FRAMEWORK}}/${FRAMEWORK}/g" \ - -e "s/{{FRAMEWORK_VERSION}}/${TF_SERVING_VERSION}/g" \ - -e "s/{{CONTAINER_TYPE}}/${CONTAINER_TYPE}/g" \ - /tmp/bash_telemetry.sh.template >/usr/local/bin/bash_telemetry.sh \ - && chmod +x /usr/local/bin/bash_telemetry.sh \ - && rm /tmp/bash_telemetry.sh.template + && sed -e "s/{{FRAMEWORK}}/${FRAMEWORK}/g" \ + -e "s/{{FRAMEWORK_VERSION}}/${TF_SERVING_VERSION}/g" \ + -e "s/{{CONTAINER_TYPE}}/${CONTAINER_TYPE}/g" \ + /tmp/bash_telemetry.sh.template >/usr/local/bin/bash_telemetry.sh \ + && chmod +x /usr/local/bin/bash_telemetry.sh \ + && rm /tmp/bash_telemetry.sh.template # Security patch — run after all installers so every OS package is covered. # Force libcurl refresh — AL2023's --security filter sometimes misses libcurl # CVE patches that ship as regular updates rather than security advisories. RUN dnf upgrade -y --security --releasever latest \ - && dnf upgrade -y cuda-compat-* \ - && dnf upgrade -y libcurl libcurl-minimal --refresh \ - && dnf clean all + && dnf upgrade -y cuda-compat-* \ + && dnf upgrade -y libcurl libcurl-minimal --refresh \ + && dnf clean all # Telemetry bashrc hook — MUST be after `dnf upgrade --security` because dnf # may replace /etc/bashrc during the upgrade, silently wiping out any `source` # line added earlier (PT main pattern). RUN echo 'source /usr/local/bin/bash_telemetry.sh' >>/etc/bashrc \ - && echo 'source /usr/local/bin/bash_telemetry.sh' >>/root/.bashrc + && echo 'source /usr/local/bin/bash_telemetry.sh' >>/root/.bashrc # OSS compliance (copy artifacts from builder-oss) COPY --from=builder-oss /root/THIRD_PARTY_SOURCE_CODE_URLS /root/THIRD_PARTY_SOURCE_CODE_URLS @@ -250,8 +250,8 @@ COPY scripts/tensorflow/inference/dockerd_entrypoint.sh /usr/local/bin/dockerd_e COPY scripts/tensorflow/inference/tf_serving_entrypoint.sh /usr/local/bin/tf_serving_entrypoint.sh COPY scripts/common/start_cuda_compat.sh /usr/local/bin/start_cuda_compat.sh RUN chmod +x /usr/local/bin/dockerd_entrypoint.sh \ - /usr/local/bin/tf_serving_entrypoint.sh \ - /usr/local/bin/start_cuda_compat.sh + /usr/local/bin/tf_serving_entrypoint.sh \ + /usr/local/bin/start_cuda_compat.sh RUN rm -rf /tmp/* /root/.cache @@ -262,4 +262,4 @@ RUN rm -rf /tmp/* /root/.cache EXPOSE 8500 8501 ENTRYPOINT ["bash", "-m", "/usr/local/bin/dockerd_entrypoint.sh"] -CMD ["/usr/local/bin/tf_serving_entrypoint.sh"] +CMD ["/usr/local/bin/tf_serving_entrypoint.sh"] \ No newline at end of file From 3c613338342ce719270cbf6dfda197be4798109b Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 11:55:45 -0700 Subject: [PATCH 04/11] fix: install gzip in builder-njs stage so tar xzf works The builder-njs stage's `tar xzf` invocation needs gzip, which AL2023's base image does not include by default. CI build failed at the nginx source extraction step on PR #6243. Add `gzip xz` to the existing dnf install line in both Dockerfile.cuda and Dockerfile.cpu builder-njs stages. --- docker/tensorflow/inference/2.20/Dockerfile.cpu | 2 +- docker/tensorflow/inference/2.20/Dockerfile.cuda | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cpu b/docker/tensorflow/inference/2.20/Dockerfile.cpu index 6ce6f365aba5..1704aefb679c 100644 --- a/docker/tensorflow/inference/2.20/Dockerfile.cpu +++ b/docker/tensorflow/inference/2.20/Dockerfile.cpu @@ -71,7 +71,7 @@ ARG NJS_VERSION RUN dnf install -y --allowerasing \ gcc gcc-c++ make \ pcre-devel pcre2-devel zlib-devel openssl-devel \ - curl tar xz \ + curl tar gzip xz \ && dnf clean all WORKDIR /tmp/njs-build diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cuda b/docker/tensorflow/inference/2.20/Dockerfile.cuda index 36d1c5708953..f2da82db10f0 100644 --- a/docker/tensorflow/inference/2.20/Dockerfile.cuda +++ b/docker/tensorflow/inference/2.20/Dockerfile.cuda @@ -79,7 +79,7 @@ ARG NJS_VERSION RUN dnf install -y --allowerasing \ gcc gcc-c++ make \ pcre-devel pcre2-devel zlib-devel openssl-devel \ - curl tar xz \ + curl tar gzip xz \ && dnf clean all WORKDIR /tmp/njs-build From 21de4f295bc891bea1ccb1e4f2be8f552c03ac03 Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 12:03:58 -0700 Subject: [PATCH 05/11] fix: install libxml2-devel/libxslt-devel in builder-njs stage nginx ./configure --with-compat with the njs dynamic module auto-enables the HTTP XSLT module, which requires libxml2 and libxslt development headers. AL2023 base lacks these. Adding to the existing dnf install line in both Dockerfile.cpu and Dockerfile.cuda builder-njs stages. --- docker/tensorflow/inference/2.20/Dockerfile.cpu | 2 +- docker/tensorflow/inference/2.20/Dockerfile.cuda | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cpu b/docker/tensorflow/inference/2.20/Dockerfile.cpu index 1704aefb679c..ab20f0db6cf1 100644 --- a/docker/tensorflow/inference/2.20/Dockerfile.cpu +++ b/docker/tensorflow/inference/2.20/Dockerfile.cpu @@ -70,7 +70,7 @@ ARG NJS_VERSION RUN dnf install -y --allowerasing \ gcc gcc-c++ make \ - pcre-devel pcre2-devel zlib-devel openssl-devel \ + pcre-devel pcre2-devel zlib-devel openssl-devel libxml2-devel libxslt-devel \ curl tar gzip xz \ && dnf clean all diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cuda b/docker/tensorflow/inference/2.20/Dockerfile.cuda index f2da82db10f0..1a8d0ed8bec8 100644 --- a/docker/tensorflow/inference/2.20/Dockerfile.cuda +++ b/docker/tensorflow/inference/2.20/Dockerfile.cuda @@ -78,7 +78,7 @@ ARG NJS_VERSION RUN dnf install -y --allowerasing \ gcc gcc-c++ make \ - pcre-devel pcre2-devel zlib-devel openssl-devel \ + pcre-devel pcre2-devel zlib-devel openssl-devel libxml2-devel libxslt-devel \ curl tar gzip xz \ && dnf clean all From b541a7330660dc67951becf8efdae9203be9ef23 Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 12:12:01 -0700 Subject: [PATCH 06/11] fix: drop stream njs module end-to-end from builder-njs and runtime nginx + njs `make modules` with the http-only --add-dynamic-module produces ngx_http_js_module.so but not the stream variant. SageMaker HTTP-based model serving uses only the http njs module (tensorflowServing.js routes /invocations and /ping). Removing the unused cp in builder-njs, the orphan COPY in runtime-base, and the comment reference in both Dockerfile.cuda and Dockerfile.cpu. --- docker/tensorflow/inference/2.20/Dockerfile.cpu | 6 ++---- docker/tensorflow/inference/2.20/Dockerfile.cuda | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cpu b/docker/tensorflow/inference/2.20/Dockerfile.cpu index ab20f0db6cf1..356bf3e81e3f 100644 --- a/docker/tensorflow/inference/2.20/Dockerfile.cpu +++ b/docker/tensorflow/inference/2.20/Dockerfile.cpu @@ -61,7 +61,7 @@ RUN PATH="/opt/venv/bin:${PATH}" bash /tmp/setup_oss_compliance.sh python${PYTHO # ── Stage: builder-njs (compile nginx + njs dynamic module from source) ───── -# Output: /out/modules/ngx_http_js_module.so + /out/modules/ngx_stream_js_module.so +# Output: /out/modules/ngx_http_js_module.so # Identical to the cuda variant's builder-njs — njs build is CPU-only and # produces an architecture-matched .so that loads into the AL2023 nginx. FROM amazonlinux:2023 AS builder-njs @@ -87,8 +87,7 @@ RUN cd nginx-${NGINX_VERSION} \ --add-dynamic-module=/tmp/njs-build/njs-${NJS_VERSION}/nginx \ && make -j$(nproc) modules \ && mkdir -p /out/modules \ - && cp objs/ngx_http_js_module.so /out/modules/ \ - && cp objs/ngx_stream_js_module.so /out/modules/ + && cp objs/ngx_http_js_module.so /out/modules/ # ── Stage: runtime-base (shared base for output stages) ───────────────────── @@ -130,7 +129,6 @@ COPY --from=builder-base /opt/venv /opt/venv # Copy njs dynamic modules COPY --from=builder-njs /out/modules/ngx_http_js_module.so /usr/lib64/nginx/modules/ -COPY --from=builder-njs /out/modules/ngx_stream_js_module.so /usr/lib64/nginx/modules/ # TF Serving binary COPY --from=build_image /usr/local/bin/tensorflow_model_server /usr/local/bin/tensorflow_model_server diff --git a/docker/tensorflow/inference/2.20/Dockerfile.cuda b/docker/tensorflow/inference/2.20/Dockerfile.cuda index 1a8d0ed8bec8..ec6f732edd85 100644 --- a/docker/tensorflow/inference/2.20/Dockerfile.cuda +++ b/docker/tensorflow/inference/2.20/Dockerfile.cuda @@ -67,7 +67,7 @@ RUN PATH="/opt/venv/bin:${PATH}" bash /tmp/setup_oss_compliance.sh python${PYTHO # ── Stage: builder-njs (compile nginx + njs dynamic module from source) ───── -# Output: /out/modules/ngx_http_js_module.so + /out/modules/ngx_stream_js_module.so +# Output: /out/modules/ngx_http_js_module.so # We use the CUDA -devel base for build toolchain consistency with builder-base; # this stage produces a CPU-only .so artifact that is identical for cuda and cpu # variants (kept in this Dockerfile as well as Dockerfile.cpu for self-contained @@ -100,8 +100,7 @@ RUN cd nginx-${NGINX_VERSION} \ --add-dynamic-module=/tmp/njs-build/njs-${NJS_VERSION}/nginx \ && make -j$(nproc) modules \ && mkdir -p /out/modules \ - && cp objs/ngx_http_js_module.so /out/modules/ \ - && cp objs/ngx_stream_js_module.so /out/modules/ + && cp objs/ngx_http_js_module.so /out/modules/ # ── Stage: runtime-base (shared base for output stages) ───────────────────── @@ -151,7 +150,6 @@ COPY --from=builder-base /opt/venv /opt/venv # nginx (1.24.x) on AL2023 honors `load_module modules/ngx_http_js_module.so` # in its conf, since we built with --with-compat against matching nginx headers. COPY --from=builder-njs /out/modules/ngx_http_js_module.so /usr/lib64/nginx/modules/ -COPY --from=builder-njs /out/modules/ngx_stream_js_module.so /usr/lib64/nginx/modules/ # TF Serving binary — copied from the official upstream devel image. COPY --from=build_image /usr/local/bin/tensorflow_model_server /usr/local/bin/tensorflow_model_server From 6272f8b20710ac77f7a6d9fb4493bc9aa58d2e3d Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 14:35:57 -0700 Subject: [PATCH 07/11] fix: mirror training's pytest rootdir + requirements for sagemaker-test Training PR's sagemaker-test job avoids the test/conftest.py fabric import not by installing fabric, but by having test/tensorflow/pytest.ini cap pytest rootdir at test/tensorflow/. Mirror that mechanism here. - Add test/tensorflow/pytest.ini ([pytest] stanza) - Match training's requirements.txt content exactly (boto3, pytest, sagemaker>=3.0.0) at test/tensorflow/integration/inference/ - Both inference workflows now install via -r requirements.txt --- .../workflows/pr-tensorflow-inference-sagemaker-cpu.yml | 5 +---- .../workflows/pr-tensorflow-inference-sagemaker-cuda.yml | 5 +---- test/tensorflow/integration/inference/requirements.txt | 8 +++----- test/tensorflow/pytest.ini | 1 + 4 files changed, 6 insertions(+), 13 deletions(-) create mode 100644 test/tensorflow/pytest.ini diff --git a/.github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml b/.github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml index 8526b1057eb3..bdbc5ecb4fbf 100644 --- a/.github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml +++ b/.github/workflows/pr-tensorflow-inference-sagemaker-cpu.yml @@ -338,10 +338,7 @@ jobs: - name: Install test dependencies run: | - # TODO: Phase 5 — replace with `pip install -r - # test/tensorflow/integration/inference/requirements.txt` once - # Phase 5 supplies that file. - pip install pytest sagemaker boto3 + pip install -r test/tensorflow/integration/inference/requirements.txt - name: Run SageMaker inference tests env: diff --git a/.github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml b/.github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml index c94fd9876ab3..c387d8f1701d 100644 --- a/.github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml +++ b/.github/workflows/pr-tensorflow-inference-sagemaker-cuda.yml @@ -344,10 +344,7 @@ jobs: - name: Install test dependencies run: | - # TODO: Phase 5 — replace with `pip install -r - # test/tensorflow/integration/inference/requirements.txt` once - # Phase 5 supplies that file. - pip install pytest sagemaker boto3 + pip install -r test/tensorflow/integration/inference/requirements.txt - name: Run SageMaker inference tests env: diff --git a/test/tensorflow/integration/inference/requirements.txt b/test/tensorflow/integration/inference/requirements.txt index 370334526e6f..f60dca57bf56 100644 --- a/test/tensorflow/integration/inference/requirements.txt +++ b/test/tensorflow/integration/inference/requirements.txt @@ -1,5 +1,3 @@ -boto3>=1.34.0 -numpy>=1.26.4 -pytest>=8.0.0 -sagemaker>=3.4.0 -tensorflow-serving-api==2.20.0 +boto3 +pytest +sagemaker>=3.0.0 diff --git a/test/tensorflow/pytest.ini b/test/tensorflow/pytest.ini new file mode 100644 index 000000000000..eea2c180278f --- /dev/null +++ b/test/tensorflow/pytest.ini @@ -0,0 +1 @@ +[pytest] From cac7760ff4d5657e38eda4bc7da38fbf2c8578c4 Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 15:17:48 -0700 Subject: [PATCH 08/11] chore: allowlist 2 CUDA toolkit CVEs in tensorflow-2.20 inference CVE-2025-23339 and CVE-2025-23308 in cuda-toolkit-config-common 12.9.79. Fix requires CUDA 13 major version bump which is tracked as a separate currency update; CUDA 12.9 is pinned to match the AL2023 base image and parallel TF 2.21 training PR. --- .../tensorflow/tensorflow-2.20.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json b/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json index fe51488c7066..c179c6356f21 100644 --- a/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json +++ b/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json @@ -1 +1,12 @@ -[] +[ + { + "vulnerability_id": "CVE-2025-23339", + "reason": "cuda-toolkit-config-common 12.9.79 — fix requires CUDA 13 major bump (>=0:13.0.48-1) which cascades through TFS binary compatibility, cuDNN, and the parallel TF 2.21 training PR. CUDA 12.9 is pinned to match the AL2023 base image; CUDA 13 currency tracked separately.", + "review_by": "2026-09-13" + }, + { + "vulnerability_id": "CVE-2025-23308", + "reason": "cuda-toolkit-config-common 12.9.79 — fix requires CUDA 13 major bump (>=0:13.0.48-1) which cascades through TFS binary compatibility, cuDNN, and the parallel TF 2.21 training PR. CUDA 12.9 is pinned to match the AL2023 base image; CUDA 13 currency tracked separately.", + "review_by": "2026-09-13" + } +] From 9be9aac4cea404a0dc236fcc47e56c0ecadcb2cd Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 15:25:53 -0700 Subject: [PATCH 09/11] fix: rename allowlist overlay to match framework_version ecr_scan.py constructs the overlay path as /-.json. With framework_version=2.20.0 (full semver) the lookup is tensorflow-2.20.0.json, not the short tensorflow-2.20.json we previously committed. Rename to match so the two CUDA toolkit CVEs are actually picked up. --- .../tensorflow/{tensorflow-2.20.json => tensorflow-2.20.0.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/security/data/ecr_scan_allowlist/tensorflow/{tensorflow-2.20.json => tensorflow-2.20.0.json} (100%) diff --git a/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json b/test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.0.json similarity index 100% rename from test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.json rename to test/security/data/ecr_scan_allowlist/tensorflow/tensorflow-2.20.0.json From 4b99d0fbee309be2c3f85a4e7f8a8287fd15b5ca Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 15:47:42 -0700 Subject: [PATCH 10/11] fix: migrate inference integration tests to SageMaker SDK v3 API The PyPI sagemaker package v3.x removed the top-level sagemaker.Session class along with sagemaker.tensorflow.serving.TensorFlowModel and sagemaker.multidatamodel.MultiDataModel. Our conftest used the legacy v2 API. Rewrite to use boto3 directly (sagemaker, sagemaker-runtime, s3 clients) so tests work with the >=3.0.0 pin and are SDK-version independent. boto3 maps 1:1 to the create_model / create_endpoint_config / create_endpoint / invoke_endpoint flow these tests already needed, which is simpler than adopting v3's heavier ModelBuilder abstraction for integration tests. The sagemaker dep is dropped from requirements.txt since it is no longer imported. --- .../integration/inference/conftest.py | 116 ++++++++++++++++-- .../integration/inference/requirements.txt | 1 - .../inference/test_multi_model_endpoint.py | 76 +++++++----- .../inference/test_single_model_endpoint.py | 115 ++++++++++------- 4 files changed, 218 insertions(+), 90 deletions(-) diff --git a/test/tensorflow/integration/inference/conftest.py b/test/tensorflow/integration/inference/conftest.py index 0a30ef0692e3..a3d70acf59bb 100644 --- a/test/tensorflow/integration/inference/conftest.py +++ b/test/tensorflow/integration/inference/conftest.py @@ -1,5 +1,12 @@ """Pytest fixtures for TF 2.20 inference integration tests on SageMaker. +These fixtures use boto3 directly (sagemaker, sagemaker-runtime, s3 clients) +rather than the SageMaker Python SDK. The PyPI ``sagemaker`` package v3.x +removed the legacy v2 surfaces this suite relied on (``sagemaker.Session``, +``sagemaker.tensorflow.serving.TensorFlowModel``, ``sagemaker.multidatamodel``), +and the v3 ``ModelBuilder`` flow is heavier than what these integration tests +need. Going boto3-only keeps the tests transparent and SDK-version-independent. + Fixtures intentionally defer all AWS calls until test-execution time so that ``pytest --collect-only`` works in environments without AWS credentials. """ @@ -46,11 +53,51 @@ def boto_session(aws_region: str): @pytest.fixture(scope="session") -def sagemaker_session(boto_session): - """A SageMaker SDK session for high-level deploy/predict calls.""" - import sagemaker +def sagemaker_client(boto_session): + """Low-level SageMaker control-plane client (create/delete model, endpoint, ...).""" + return boto_session.client("sagemaker") + - return sagemaker.Session(boto_session=boto_session) +@pytest.fixture(scope="session") +def sagemaker_runtime_client(boto_session): + """SageMaker runtime client used to invoke endpoints.""" + return boto_session.client("sagemaker-runtime") + + +@pytest.fixture(scope="session") +def s3_client(boto_session): + """S3 client used to upload sample model tarballs.""" + return boto_session.client("s3") + + +@pytest.fixture(scope="session") +def default_bucket(boto_session, aws_region: str, s3_client) -> str: + """Resolve the ``sagemaker--`` default bucket, creating it if absent. + + Mirrors the behaviour of the v2 SDK's ``Session.default_bucket()`` so test + bodies can keep using a single, predictable bucket without callers having + to plumb one in. + """ + sts = boto_session.client("sts") + account_id = sts.get_caller_identity()["Account"] + bucket = f"sagemaker-{aws_region}-{account_id}" + + try: + s3_client.head_bucket(Bucket=bucket) + except Exception: + # Bucket missing or inaccessible — try to create it. us-east-1 must omit + # LocationConstraint; every other region requires it. + create_kwargs: dict = {"Bucket": bucket} + if aws_region != "us-east-1": + create_kwargs["CreateBucketConfiguration"] = {"LocationConstraint": aws_region} + try: + s3_client.create_bucket(**create_kwargs) + except Exception: + # Race or pre-existing-but-403 — leave the original error to surface + # at upload time rather than masking it here. + pass + + return bucket @pytest.fixture @@ -68,13 +115,61 @@ def _make(prefix: str) -> str: @pytest.fixture -def cleanup_endpoint(sagemaker_session): +def upload_to_s3(s3_client): + """Yield-style helper that uploads a local file to ``s3:///``. + + Returns the resulting ``s3://`` URI. Failures propagate; teardown is + intentionally not provided since SageMaker model artifacts are typically + left in the bucket for forensic inspection. + """ + + def _upload(local_path: str, bucket: str, key: str) -> str: + s3_client.upload_file(local_path, bucket, key) + return f"s3://{bucket}/{key}" + + return _upload + + +@pytest.fixture +def wait_for_endpoint(sagemaker_client): + """Poll ``describe_endpoint`` until the endpoint reaches ``InService``. + + Raises ``RuntimeError`` if the endpoint enters ``Failed`` / ``OutOfService`` + or if the wait exceeds ``timeout_seconds`` (default 1800s = 30 min, which + matches typical first-pull cold-start latency for inference DLCs). + """ + + def _wait(endpoint_name: str, timeout_seconds: int = 1800, poll_seconds: int = 30) -> None: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + resp = sagemaker_client.describe_endpoint(EndpointName=endpoint_name) + status = resp["EndpointStatus"] + if status == "InService": + return + if status in {"Failed", "OutOfService"}: + reason = resp.get("FailureReason", "") + raise RuntimeError( + f"endpoint {endpoint_name} entered terminal state {status}: {reason}" + ) + time.sleep(poll_seconds) + raise RuntimeError( + f"endpoint {endpoint_name} did not reach InService within {timeout_seconds}s" + ) + + return _wait + + +@pytest.fixture +def cleanup_endpoint(sagemaker_client): """Yield-style fixture that tears down endpoint, endpoint config, and model. Usage: def test_x(cleanup_endpoint, ...): cleanup_endpoint(endpoint_name, model_name=model_name) # ... deploy + predict ... + + Endpoint config is registered under the same name as the endpoint, matching + what the test bodies pass to ``create_endpoint_config``. """ registered: list[dict] = [] @@ -83,25 +178,22 @@ def _register(endpoint_name: str, model_name: str | None = None) -> None: yield _register - sm_client = sagemaker_session.boto_session.client("sagemaker") for item in registered: endpoint_name = item["endpoint_name"] model_name = item["model_name"] for delete_call, kwargs in ( - (sm_client.delete_endpoint, {"EndpointName": endpoint_name}), - (sm_client.delete_endpoint_config, {"EndpointConfigName": endpoint_name}), + (sagemaker_client.delete_endpoint, {"EndpointName": endpoint_name}), + (sagemaker_client.delete_endpoint_config, {"EndpointConfigName": endpoint_name}), ): try: delete_call(**kwargs) - except sm_client.exceptions.ClientError: - # Swallow NotFound / already-deleted; teardown should be best-effort. - pass except Exception: + # Swallow NotFound / already-deleted; teardown is best-effort. pass if model_name: try: - sm_client.delete_model(ModelName=model_name) + sagemaker_client.delete_model(ModelName=model_name) except Exception: pass diff --git a/test/tensorflow/integration/inference/requirements.txt b/test/tensorflow/integration/inference/requirements.txt index f60dca57bf56..7ae26195bc74 100644 --- a/test/tensorflow/integration/inference/requirements.txt +++ b/test/tensorflow/integration/inference/requirements.txt @@ -1,3 +1,2 @@ boto3 pytest -sagemaker>=3.0.0 diff --git a/test/tensorflow/integration/inference/test_multi_model_endpoint.py b/test/tensorflow/integration/inference/test_multi_model_endpoint.py index 6be7a454a305..559d14f5864c 100644 --- a/test/tensorflow/integration/inference/test_multi_model_endpoint.py +++ b/test/tensorflow/integration/inference/test_multi_model_endpoint.py @@ -3,6 +3,12 @@ Builds two tiny SavedModels (``y = 2x`` and ``y = 3x``), uploads both to a shared S3 prefix, deploys a SageMaker MME backed by the v2 inference image, and asserts that ``TargetModel`` routes invocations correctly. + +Uses boto3 directly (sagemaker, sagemaker-runtime, s3 clients) rather than +the SageMaker Python SDK, because the v3.x ``sagemaker`` package removed +the v2 ``MultiDataModel`` / ``TensorFlowModel`` classes this test originally +relied on. MME is a thin server-side construct anyway: ``Mode: MultiModel`` +on the PrimaryContainer plus a model_data S3 prefix. """ from __future__ import annotations @@ -18,9 +24,9 @@ INSTANCE_TYPE = "ml.c5.xlarge" -def _decode(response) -> dict: - """Normalize an MME runtime invoke response Body into a Python dict.""" - body = response["Body"].read() if hasattr(response, "get") else response +def _decode(invoke_response) -> dict: + """Decode an ``invoke_endpoint`` response Body into a Python dict.""" + body = invoke_response["Body"].read() if isinstance(body, (bytes, bytearray)): body = body.decode("utf-8") return json.loads(body) @@ -36,15 +42,16 @@ def _values_from_predictions(predictions) -> list: def test_mme_two_models( - sagemaker_session, + sagemaker_client, + sagemaker_runtime_client, sagemaker_role_arn, inference_image_uri, + default_bucket, + upload_to_s3, unique_name, cleanup_endpoint, + wait_for_endpoint, ): - from sagemaker.multidatamodel import MultiDataModel - from sagemaker.tensorflow.serving import TensorFlowModel - with tempfile.TemporaryDirectory(prefix="tf220-mme-") as workdir: workdir_path = Path(workdir) @@ -59,48 +66,55 @@ def test_mme_two_models( output_dir=model2_dir, multiplier=3.0, model_name="model", tar_filename="model2.tar.gz" ) - bucket = sagemaker_session.default_bucket() run_id = unique_name("mme") s3_key_prefix = f"tf220-inference-tests/mme-models/{run_id}" # Upload each tarball under the shared MME prefix so the runtime can # resolve TargetModel relative to the same S3 location. - sagemaker_session.upload_data(path=model1_tar, bucket=bucket, key_prefix=s3_key_prefix) - sagemaker_session.upload_data(path=model2_tar, bucket=bucket, key_prefix=s3_key_prefix) - s3_model_prefix = f"s3://{bucket}/{s3_key_prefix}/" + upload_to_s3(model1_tar, default_bucket, f"{s3_key_prefix}/model1.tar.gz") + upload_to_s3(model2_tar, default_bucket, f"{s3_key_prefix}/model2.tar.gz") + s3_model_prefix = f"s3://{default_bucket}/{s3_key_prefix}/" endpoint_name = unique_name("tf220-mme") base_model_name = unique_name("tf220-mme-model") cleanup_endpoint(endpoint_name, model_name=base_model_name) - # The TensorFlowModel acts as the container template; MultiDataModel - # reuses its image_uri + execution role for the endpoint config. - base_model = TensorFlowModel( - model_data=model1_tar, # placeholder; MME ignores model_data at runtime - role=sagemaker_role_arn, - image_uri=inference_image_uri, - sagemaker_session=sagemaker_session, - name=base_model_name, + # ``Mode: MultiModel`` plus an S3 prefix is the entire MME contract on + # the control plane; the runtime resolves ``TargetModel`` relative to + # ModelDataUrl. + sagemaker_client.create_model( + ModelName=base_model_name, + ExecutionRoleArn=sagemaker_role_arn, + PrimaryContainer={ + "Image": inference_image_uri, + "ModelDataUrl": s3_model_prefix, + "Mode": "MultiModel", + }, ) - mme = MultiDataModel( - name=base_model_name, - model_data_prefix=s3_model_prefix, - model=base_model, - sagemaker_session=sagemaker_session, + sagemaker_client.create_endpoint_config( + EndpointConfigName=endpoint_name, + ProductionVariants=[ + { + "VariantName": "AllTraffic", + "ModelName": base_model_name, + "InitialInstanceCount": 1, + "InstanceType": INSTANCE_TYPE, + "InitialVariantWeight": 1.0, + } + ], ) - mme.deploy( - initial_instance_count=1, - instance_type=INSTANCE_TYPE, - endpoint_name=endpoint_name, + sagemaker_client.create_endpoint( + EndpointName=endpoint_name, + EndpointConfigName=endpoint_name, ) + wait_for_endpoint(endpoint_name) - runtime = sagemaker_session.boto_session.client("sagemaker-runtime") payload = json.dumps({"instances": [[1.0, 2.0, 3.0]]}) # Invoke model1 (x * 2.0) - resp1 = runtime.invoke_endpoint( + resp1 = sagemaker_runtime_client.invoke_endpoint( EndpointName=endpoint_name, ContentType="application/json", TargetModel="model1.tar.gz", @@ -112,7 +126,7 @@ def test_mme_two_models( assert values1 == pytest.approx([2.0, 4.0, 6.0]), f"model1 got {values1!r}" # Invoke model2 (x * 3.0) - resp2 = runtime.invoke_endpoint( + resp2 = sagemaker_runtime_client.invoke_endpoint( EndpointName=endpoint_name, ContentType="application/json", TargetModel="model2.tar.gz", diff --git a/test/tensorflow/integration/inference/test_single_model_endpoint.py b/test/tensorflow/integration/inference/test_single_model_endpoint.py index be480c43f11a..4e77168851bc 100644 --- a/test/tensorflow/integration/inference/test_single_model_endpoint.py +++ b/test/tensorflow/integration/inference/test_single_model_endpoint.py @@ -3,6 +3,11 @@ Builds a tiny ``y = 2x`` SavedModel, deploys it to a single-instance SageMaker endpoint backed by the v2 inference image under test, and asserts the predicted values. + +Uses boto3 directly (sagemaker, sagemaker-runtime, s3 clients) rather than +the SageMaker Python SDK, because the v3.x ``sagemaker`` package removed the +v2 ``sagemaker.tensorflow.serving.TensorFlowModel`` flow this test originally +relied on. """ from __future__ import annotations @@ -18,15 +23,25 @@ INSTANCE_TYPE = "ml.c5.xlarge" +def _decode(invoke_response) -> dict: + """Decode an ``invoke_endpoint`` response Body into a Python dict.""" + body = invoke_response["Body"].read() + if isinstance(body, (bytes, bytearray)): + body = body.decode("utf-8") + return json.loads(body) + + def test_single_model_predict( - sagemaker_session, + sagemaker_client, + sagemaker_runtime_client, sagemaker_role_arn, inference_image_uri, + default_bucket, + upload_to_s3, unique_name, cleanup_endpoint, + wait_for_endpoint, ): - from sagemaker.tensorflow.serving import TensorFlowModel - with tempfile.TemporaryDirectory(prefix="tf220-single-") as workdir: tar_path = build_sample_model( output_dir=workdir, @@ -34,55 +49,63 @@ def test_single_model_predict( model_name="model", ) - bucket = sagemaker_session.default_bucket() - key_prefix = f"tf220-inference-tests/{Path(tar_path).stem}-{unique_name('single')}" - model_data = sagemaker_session.upload_data( - path=tar_path, - bucket=bucket, - key_prefix=key_prefix, - ) + run_id = unique_name("single") + s3_key = f"tf220-inference-tests/{Path(tar_path).stem}-{run_id}/{Path(tar_path).name}" + model_data = upload_to_s3(tar_path, default_bucket, s3_key) endpoint_name = unique_name("tf220-single") model_name = unique_name("tf220-single-model") cleanup_endpoint(endpoint_name, model_name=model_name) - tf_model = TensorFlowModel( - model_data=model_data, - role=sagemaker_role_arn, - image_uri=inference_image_uri, - sagemaker_session=sagemaker_session, - name=model_name, + # Equivalent to v2 ``TensorFlowModel(image_uri=..., model_data=...).deploy(...)`` + # but expressed as the underlying control-plane API calls. + sagemaker_client.create_model( + ModelName=model_name, + ExecutionRoleArn=sagemaker_role_arn, + PrimaryContainer={ + "Image": inference_image_uri, + "ModelDataUrl": model_data, + }, + ) + + sagemaker_client.create_endpoint_config( + EndpointConfigName=endpoint_name, + ProductionVariants=[ + { + "VariantName": "AllTraffic", + "ModelName": model_name, + "InitialInstanceCount": 1, + "InstanceType": INSTANCE_TYPE, + "InitialVariantWeight": 1.0, + } + ], ) - predictor = tf_model.deploy( - initial_instance_count=1, - instance_type=INSTANCE_TYPE, - endpoint_name=endpoint_name, + sagemaker_client.create_endpoint( + EndpointName=endpoint_name, + EndpointConfigName=endpoint_name, ) + wait_for_endpoint(endpoint_name) - try: - payload = {"instances": [[1.0, 2.0, 3.0]]} - response = predictor.predict(payload) - - # The TFS predictor may return a dict already, or a JSON string — - # normalize both shapes. - if isinstance(response, (bytes, str)): - response = json.loads(response) - - assert "predictions" in response, f"missing predictions key in {response!r}" - predictions = response["predictions"] - assert predictions and isinstance(predictions, list) - - # Output signature is {"output": x * 2.0} -> TFS surfaces the tensor - # under the signature output key when there is a single named tensor; - # some TFS versions instead return the raw list. Handle both. - first = predictions[0] - if isinstance(first, dict) and "output" in first: - values = first["output"] - else: - values = first - - assert values == pytest.approx([2.0, 4.0, 6.0]), f"got {values!r}" - finally: - # cleanup_endpoint teardown handles resources; no manual delete needed. - pass + payload = json.dumps({"instances": [[1.0, 2.0, 3.0]]}) + response = sagemaker_runtime_client.invoke_endpoint( + EndpointName=endpoint_name, + ContentType="application/json", + Body=payload, + ) + body = _decode(response) + + assert "predictions" in body, f"missing predictions key in {body!r}" + predictions = body["predictions"] + assert predictions and isinstance(predictions, list) + + # Output signature is {"output": x * 2.0} -> TFS surfaces the tensor + # under the signature output key when there is a single named tensor; + # some TFS versions instead return the raw list. Handle both. + first = predictions[0] + if isinstance(first, dict) and "output" in first: + values = first["output"] + else: + values = first + + assert values == pytest.approx([2.0, 4.0, 6.0]), f"got {values!r}" From 0426f698938ddd1984a00d0ed27fa74d44a32dde Mon Sep 17 00:00:00 2001 From: Bhanu Teja Goshikonda Date: Mon, 15 Jun 2026 16:23:25 -0700 Subject: [PATCH 11/11] fix: rewrite inference integration tests using SageMaker SDK v3 Replaces commit 4b99d0fb which used boto3 directly. The SageMaker Python SDK v3 is the supported entrypoint for these tests; v3 removed the v2 sagemaker.Session, sagemaker.tensorflow.serving.TensorFlowModel, and sagemaker.multidatamodel.MultiDataModel classes the old fixtures relied on. For DLC integration tests we already supply image_uri and a pre-built model.tar.gz, so ModelBuilder's auto-detection adds no value. We use the v3 sagemaker-core resource layer directly (the same surface ModelBuilder calls underneath): - conftest sagemaker_session fixture uses sagemaker.core.helper.session_helper.Session (default_bucket / upload_data); cleanup_endpoint uses Endpoint.get(...).delete(), EndpointConfig.get(...).delete(), Model.get(...).delete() - single-model test uses Model.create(primary_container=ContainerDefinition(...)) + EndpointConfig.create([ProductionVariant(...)]) + Endpoint.create() with endpoint.wait_for_status("InService") and endpoint.invoke(...) - MME test expresses the multi-model contract directly: ContainerDefinition(mode="MultiModel", model_data_url=) and endpoint.invoke(target_model="modelN.tar.gz") - sagemaker>=3.0.0 retained in requirements.txt --- .../integration/inference/conftest.py | 148 +++++------------- .../integration/inference/requirements.txt | 1 + .../inference/test_multi_model_endpoint.py | 137 ++++++++-------- .../inference/test_single_model_endpoint.py | 112 +++++++------ 4 files changed, 177 insertions(+), 221 deletions(-) diff --git a/test/tensorflow/integration/inference/conftest.py b/test/tensorflow/integration/inference/conftest.py index a3d70acf59bb..506170347040 100644 --- a/test/tensorflow/integration/inference/conftest.py +++ b/test/tensorflow/integration/inference/conftest.py @@ -1,11 +1,15 @@ """Pytest fixtures for TF 2.20 inference integration tests on SageMaker. -These fixtures use boto3 directly (sagemaker, sagemaker-runtime, s3 clients) -rather than the SageMaker Python SDK. The PyPI ``sagemaker`` package v3.x -removed the legacy v2 surfaces this suite relied on (``sagemaker.Session``, -``sagemaker.tensorflow.serving.TensorFlowModel``, ``sagemaker.multidatamodel``), -and the v3 ``ModelBuilder`` flow is heavier than what these integration tests -need. Going boto3-only keeps the tests transparent and SDK-version-independent. +Uses the SageMaker Python SDK v3 (``sagemaker>=3.0.0``) — the v2 Estimator / +Model / Predictor classes were removed in v3 in favor of the unified +``ModelBuilder`` and the ``sagemaker-core`` resource layer +(``Endpoint``, ``EndpointConfig``, ``Model``, ``ContainerDefinition``, +``ProductionVariant``). For these DLC tests we already have a custom +``image_uri`` and a pre-built ``model.tar.gz``, so the simplest v3 path is +the resource layer directly: ``Model.create -> EndpointConfig.create -> +Endpoint.create -> endpoint.invoke()``. ``ModelBuilder`` is the right choice +when the SDK should auto-detect the framework / container / packaging — for +us, those are all fixed by the test fixture inputs. Fixtures intentionally defer all AWS calls until test-execution time so that ``pytest --collect-only`` works in environments without AWS credentials. @@ -46,58 +50,29 @@ def inference_image_uri() -> str: @pytest.fixture(scope="session") def boto_session(aws_region: str): - """A boto3 session bound to the configured region.""" + """A boto3 session bound to the configured region. + + Used purely as a transport for ``sagemaker.core.helper.session_helper.Session`` + and for the underlying ``s3`` client when uploading model artifacts; no + SageMaker control-plane calls go through it directly. + """ import boto3 return boto3.Session(region_name=aws_region) @pytest.fixture(scope="session") -def sagemaker_client(boto_session): - """Low-level SageMaker control-plane client (create/delete model, endpoint, ...).""" - return boto_session.client("sagemaker") - - -@pytest.fixture(scope="session") -def sagemaker_runtime_client(boto_session): - """SageMaker runtime client used to invoke endpoints.""" - return boto_session.client("sagemaker-runtime") - - -@pytest.fixture(scope="session") -def s3_client(boto_session): - """S3 client used to upload sample model tarballs.""" - return boto_session.client("s3") - - -@pytest.fixture(scope="session") -def default_bucket(boto_session, aws_region: str, s3_client) -> str: - """Resolve the ``sagemaker--`` default bucket, creating it if absent. +def sagemaker_session(boto_session): + """A SageMaker SDK v3 session. - Mirrors the behaviour of the v2 SDK's ``Session.default_bucket()`` so test - bodies can keep using a single, predictable bucket without callers having - to plumb one in. + ``sagemaker.core.helper.session_helper.Session`` is the v3 replacement for + the v2 ``sagemaker.Session``. We use it for ``default_bucket()`` and + ``upload_data()``; resource-layer ``create()`` calls accept it via the + ``session=`` kwarg. """ - sts = boto_session.client("sts") - account_id = sts.get_caller_identity()["Account"] - bucket = f"sagemaker-{aws_region}-{account_id}" - - try: - s3_client.head_bucket(Bucket=bucket) - except Exception: - # Bucket missing or inaccessible — try to create it. us-east-1 must omit - # LocationConstraint; every other region requires it. - create_kwargs: dict = {"Bucket": bucket} - if aws_region != "us-east-1": - create_kwargs["CreateBucketConfiguration"] = {"LocationConstraint": aws_region} - try: - s3_client.create_bucket(**create_kwargs) - except Exception: - # Race or pre-existing-but-403 — leave the original error to surface - # at upload time rather than masking it here. - pass - - return bucket + from sagemaker.core.helper.session_helper import Session + + return Session(boto_session=boto_session) @pytest.fixture @@ -115,61 +90,20 @@ def _make(prefix: str) -> str: @pytest.fixture -def upload_to_s3(s3_client): - """Yield-style helper that uploads a local file to ``s3:///``. - - Returns the resulting ``s3://`` URI. Failures propagate; teardown is - intentionally not provided since SageMaker model artifacts are typically - left in the bucket for forensic inspection. - """ - - def _upload(local_path: str, bucket: str, key: str) -> str: - s3_client.upload_file(local_path, bucket, key) - return f"s3://{bucket}/{key}" - - return _upload - - -@pytest.fixture -def wait_for_endpoint(sagemaker_client): - """Poll ``describe_endpoint`` until the endpoint reaches ``InService``. - - Raises ``RuntimeError`` if the endpoint enters ``Failed`` / ``OutOfService`` - or if the wait exceeds ``timeout_seconds`` (default 1800s = 30 min, which - matches typical first-pull cold-start latency for inference DLCs). - """ - - def _wait(endpoint_name: str, timeout_seconds: int = 1800, poll_seconds: int = 30) -> None: - deadline = time.time() + timeout_seconds - while time.time() < deadline: - resp = sagemaker_client.describe_endpoint(EndpointName=endpoint_name) - status = resp["EndpointStatus"] - if status == "InService": - return - if status in {"Failed", "OutOfService"}: - reason = resp.get("FailureReason", "") - raise RuntimeError( - f"endpoint {endpoint_name} entered terminal state {status}: {reason}" - ) - time.sleep(poll_seconds) - raise RuntimeError( - f"endpoint {endpoint_name} did not reach InService within {timeout_seconds}s" - ) - - return _wait - - -@pytest.fixture -def cleanup_endpoint(sagemaker_client): +def cleanup_endpoint(boto_session): """Yield-style fixture that tears down endpoint, endpoint config, and model. + Uses the v3 ``sagemaker-core`` resource layer (``Endpoint.get(...).delete()``, + etc.) rather than raw boto3 SDK calls, so cleanup code matches the deploy + code in the tests. The ``session=`` kwarg on resource ``get`` / ``create`` + methods accepts a raw ``boto3.session.Session`` (see + ``sagemaker.core.utils.utils.SageMakerClient``); pass ``boto_session`` + rather than the helper ``Session``. + Usage: def test_x(cleanup_endpoint, ...): cleanup_endpoint(endpoint_name, model_name=model_name) # ... deploy + predict ... - - Endpoint config is registered under the same name as the endpoint, matching - what the test bodies pass to ``create_endpoint_config``. """ registered: list[dict] = [] @@ -178,22 +112,26 @@ def _register(endpoint_name: str, model_name: str | None = None) -> None: yield _register + # Import lazily so collection works without the SDK installed. + from sagemaker.core.resources import Endpoint, EndpointConfig, Model + for item in registered: endpoint_name = item["endpoint_name"] model_name = item["model_name"] - for delete_call, kwargs in ( - (sagemaker_client.delete_endpoint, {"EndpointName": endpoint_name}), - (sagemaker_client.delete_endpoint_config, {"EndpointConfigName": endpoint_name}), + # Endpoint config name == endpoint name in our deploy flow below. + for resource_cls, get_kwargs in ( + (Endpoint, {"endpoint_name": endpoint_name}), + (EndpointConfig, {"endpoint_config_name": endpoint_name}), ): try: - delete_call(**kwargs) + resource_cls.get(session=boto_session, **get_kwargs).delete() except Exception: - # Swallow NotFound / already-deleted; teardown is best-effort. + # Best-effort teardown: swallow NotFound / already-deleted. pass if model_name: try: - sagemaker_client.delete_model(ModelName=model_name) + Model.get(model_name=model_name, session=boto_session).delete() except Exception: pass diff --git a/test/tensorflow/integration/inference/requirements.txt b/test/tensorflow/integration/inference/requirements.txt index 7ae26195bc74..f60dca57bf56 100644 --- a/test/tensorflow/integration/inference/requirements.txt +++ b/test/tensorflow/integration/inference/requirements.txt @@ -1,2 +1,3 @@ boto3 pytest +sagemaker>=3.0.0 diff --git a/test/tensorflow/integration/inference/test_multi_model_endpoint.py b/test/tensorflow/integration/inference/test_multi_model_endpoint.py index 559d14f5864c..ebea37e42d58 100644 --- a/test/tensorflow/integration/inference/test_multi_model_endpoint.py +++ b/test/tensorflow/integration/inference/test_multi_model_endpoint.py @@ -2,13 +2,16 @@ Builds two tiny SavedModels (``y = 2x`` and ``y = 3x``), uploads both to a shared S3 prefix, deploys a SageMaker MME backed by the v2 inference image, -and asserts that ``TargetModel`` routes invocations correctly. - -Uses boto3 directly (sagemaker, sagemaker-runtime, s3 clients) rather than -the SageMaker Python SDK, because the v3.x ``sagemaker`` package removed -the v2 ``MultiDataModel`` / ``TensorFlowModel`` classes this test originally -relied on. MME is a thin server-side construct anyway: ``Mode: MultiModel`` -on the PrimaryContainer plus a model_data S3 prefix. +and asserts that ``target_model`` routes invocations correctly. + +Uses the SageMaker Python SDK v3 ``sagemaker-core`` resource layer — v3 +removed ``sagemaker.multidatamodel.MultiDataModel``. The native MME wire +contract (``ContainerDefinition.mode = "MultiModel"``, +``model_data_url = s3://bucket/prefix/``, plus the +``X-Amzn-SageMaker-Target-Model`` header on invoke) is unchanged, so we +express it directly: ``Model.create`` with ``mode="MultiModel"`` and an S3 +prefix in ``model_data_url``, then ``endpoint.invoke(target_model=...)`` +which sets the runtime header for us. """ from __future__ import annotations @@ -24,14 +27,6 @@ INSTANCE_TYPE = "ml.c5.xlarge" -def _decode(invoke_response) -> dict: - """Decode an ``invoke_endpoint`` response Body into a Python dict.""" - body = invoke_response["Body"].read() - if isinstance(body, (bytes, bytearray)): - body = body.decode("utf-8") - return json.loads(body) - - def _values_from_predictions(predictions) -> list: """Pull the numeric output list out of either signature-keyed or raw rows.""" assert predictions and isinstance(predictions, list) @@ -42,16 +37,21 @@ def _values_from_predictions(predictions) -> list: def test_mme_two_models( - sagemaker_client, - sagemaker_runtime_client, + boto_session, + sagemaker_session, sagemaker_role_arn, inference_image_uri, - default_bucket, - upload_to_s3, unique_name, cleanup_endpoint, - wait_for_endpoint, ): + from sagemaker.core.resources import ( + ContainerDefinition, + Endpoint, + EndpointConfig, + Model, + ProductionVariant, + ) + with tempfile.TemporaryDirectory(prefix="tf220-mme-") as workdir: workdir_path = Path(workdir) @@ -66,73 +66,78 @@ def test_mme_two_models( output_dir=model2_dir, multiplier=3.0, model_name="model", tar_filename="model2.tar.gz" ) + bucket = sagemaker_session.default_bucket() run_id = unique_name("mme") s3_key_prefix = f"tf220-inference-tests/mme-models/{run_id}" # Upload each tarball under the shared MME prefix so the runtime can - # resolve TargetModel relative to the same S3 location. - upload_to_s3(model1_tar, default_bucket, f"{s3_key_prefix}/model1.tar.gz") - upload_to_s3(model2_tar, default_bucket, f"{s3_key_prefix}/model2.tar.gz") - s3_model_prefix = f"s3://{default_bucket}/{s3_key_prefix}/" + # resolve target_model relative to the same S3 location. + sagemaker_session.upload_data(path=model1_tar, bucket=bucket, key_prefix=s3_key_prefix) + sagemaker_session.upload_data(path=model2_tar, bucket=bucket, key_prefix=s3_key_prefix) + s3_model_prefix = f"s3://{bucket}/{s3_key_prefix}/" endpoint_name = unique_name("tf220-mme") - base_model_name = unique_name("tf220-mme-model") - cleanup_endpoint(endpoint_name, model_name=base_model_name) - - # ``Mode: MultiModel`` plus an S3 prefix is the entire MME contract on - # the control plane; the runtime resolves ``TargetModel`` relative to - # ModelDataUrl. - sagemaker_client.create_model( - ModelName=base_model_name, - ExecutionRoleArn=sagemaker_role_arn, - PrimaryContainer={ - "Image": inference_image_uri, - "ModelDataUrl": s3_model_prefix, - "Mode": "MultiModel", - }, + model_name = unique_name("tf220-mme-model") + cleanup_endpoint(endpoint_name, model_name=model_name) + + # 1. Create a multi-model SageMaker Model. The MME contract is + # expressed at the container definition level: mode="MultiModel" + # plus an S3 *prefix* (not a single tar) in model_data_url. + Model.create( + model_name=model_name, + primary_container=ContainerDefinition( + image=inference_image_uri, + mode="MultiModel", + model_data_url=s3_model_prefix, + ), + execution_role_arn=sagemaker_role_arn, + session=boto_session, ) - sagemaker_client.create_endpoint_config( - EndpointConfigName=endpoint_name, - ProductionVariants=[ - { - "VariantName": "AllTraffic", - "ModelName": base_model_name, - "InitialInstanceCount": 1, - "InstanceType": INSTANCE_TYPE, - "InitialVariantWeight": 1.0, - } + # 2. Endpoint config + endpoint — same shape as single-model. + EndpointConfig.create( + endpoint_config_name=endpoint_name, + production_variants=[ + ProductionVariant( + variant_name="AllTraffic", + model_name=model_name, + initial_instance_count=1, + instance_type=INSTANCE_TYPE, + ), ], + session=boto_session, ) - sagemaker_client.create_endpoint( - EndpointName=endpoint_name, - EndpointConfigName=endpoint_name, + endpoint = Endpoint.create( + endpoint_name=endpoint_name, + endpoint_config_name=endpoint_name, + session=boto_session, ) - wait_for_endpoint(endpoint_name) + endpoint.wait_for_status("InService") payload = json.dumps({"instances": [[1.0, 2.0, 3.0]]}) - # Invoke model1 (x * 2.0) - resp1 = sagemaker_runtime_client.invoke_endpoint( - EndpointName=endpoint_name, - ContentType="application/json", - TargetModel="model1.tar.gz", - Body=payload, + # 3. Invoke each model by name. ``target_model`` maps to the + # X-Amzn-SageMaker-Target-Model header that selects the tarball + # within the MME's S3 prefix. + resp1 = endpoint.invoke( + body=payload, + content_type="application/json", + accept="application/json", + target_model="model1.tar.gz", ) - body1 = _decode(resp1) + body1 = json.loads(resp1.body.read().decode("utf-8")) assert "predictions" in body1, f"model1 response missing predictions: {body1!r}" values1 = _values_from_predictions(body1["predictions"]) assert values1 == pytest.approx([2.0, 4.0, 6.0]), f"model1 got {values1!r}" - # Invoke model2 (x * 3.0) - resp2 = sagemaker_runtime_client.invoke_endpoint( - EndpointName=endpoint_name, - ContentType="application/json", - TargetModel="model2.tar.gz", - Body=payload, + resp2 = endpoint.invoke( + body=payload, + content_type="application/json", + accept="application/json", + target_model="model2.tar.gz", ) - body2 = _decode(resp2) + body2 = json.loads(resp2.body.read().decode("utf-8")) assert "predictions" in body2, f"model2 response missing predictions: {body2!r}" values2 = _values_from_predictions(body2["predictions"]) assert values2 == pytest.approx([3.0, 6.0, 9.0]), f"model2 got {values2!r}" diff --git a/test/tensorflow/integration/inference/test_single_model_endpoint.py b/test/tensorflow/integration/inference/test_single_model_endpoint.py index 4e77168851bc..c632377a4d8a 100644 --- a/test/tensorflow/integration/inference/test_single_model_endpoint.py +++ b/test/tensorflow/integration/inference/test_single_model_endpoint.py @@ -4,10 +4,13 @@ endpoint backed by the v2 inference image under test, and asserts the predicted values. -Uses boto3 directly (sagemaker, sagemaker-runtime, s3 clients) rather than -the SageMaker Python SDK, because the v3.x ``sagemaker`` package removed the -v2 ``sagemaker.tensorflow.serving.TensorFlowModel`` flow this test originally -relied on. +Uses the SageMaker Python SDK v3 ``sagemaker-core`` resource layer +(``Endpoint``, ``EndpointConfig``, ``Model``, ``ContainerDefinition``, +``ProductionVariant``) — the v2 ``TensorFlowModel`` / ``Predictor`` classes +were removed in v3. ``ModelBuilder`` is the v3 entry point for +auto-detected deployments, but for DLC tests we already supply the +``image_uri`` and a pre-built ``model.tar.gz``, so we go straight to the +resource layer (the same surface ``ModelBuilder`` calls underneath). """ from __future__ import annotations @@ -23,25 +26,22 @@ INSTANCE_TYPE = "ml.c5.xlarge" -def _decode(invoke_response) -> dict: - """Decode an ``invoke_endpoint`` response Body into a Python dict.""" - body = invoke_response["Body"].read() - if isinstance(body, (bytes, bytearray)): - body = body.decode("utf-8") - return json.loads(body) - - def test_single_model_predict( - sagemaker_client, - sagemaker_runtime_client, + boto_session, + sagemaker_session, sagemaker_role_arn, inference_image_uri, - default_bucket, - upload_to_s3, unique_name, cleanup_endpoint, - wait_for_endpoint, ): + from sagemaker.core.resources import ( + ContainerDefinition, + Endpoint, + EndpointConfig, + Model, + ProductionVariant, + ) + with tempfile.TemporaryDirectory(prefix="tf220-single-") as workdir: tar_path = build_sample_model( output_dir=workdir, @@ -49,54 +49,66 @@ def test_single_model_predict( model_name="model", ) - run_id = unique_name("single") - s3_key = f"tf220-inference-tests/{Path(tar_path).stem}-{run_id}/{Path(tar_path).name}" - model_data = upload_to_s3(tar_path, default_bucket, s3_key) + # Upload the tarball via the v3 helper Session — same default-bucket / + # upload_data ergonomics as v2. + bucket = sagemaker_session.default_bucket() + key_prefix = f"tf220-inference-tests/{Path(tar_path).stem}-{unique_name('single')}" + model_data = sagemaker_session.upload_data( + path=tar_path, + bucket=bucket, + key_prefix=key_prefix, + ) endpoint_name = unique_name("tf220-single") model_name = unique_name("tf220-single-model") cleanup_endpoint(endpoint_name, model_name=model_name) - # Equivalent to v2 ``TensorFlowModel(image_uri=..., model_data=...).deploy(...)`` - # but expressed as the underlying control-plane API calls. - sagemaker_client.create_model( - ModelName=model_name, - ExecutionRoleArn=sagemaker_role_arn, - PrimaryContainer={ - "Image": inference_image_uri, - "ModelDataUrl": model_data, - }, + # 1. Create the SageMaker Model — points at our DLC image and the + # uploaded SavedModel tar.gz. + Model.create( + model_name=model_name, + primary_container=ContainerDefinition( + image=inference_image_uri, + model_data_url=model_data, + ), + execution_role_arn=sagemaker_role_arn, + session=boto_session, ) - sagemaker_client.create_endpoint_config( - EndpointConfigName=endpoint_name, - ProductionVariants=[ - { - "VariantName": "AllTraffic", - "ModelName": model_name, - "InitialInstanceCount": 1, - "InstanceType": INSTANCE_TYPE, - "InitialVariantWeight": 1.0, - } + # 2. Create the EndpointConfig with a single ProductionVariant. + EndpointConfig.create( + endpoint_config_name=endpoint_name, + production_variants=[ + ProductionVariant( + variant_name="AllTraffic", + model_name=model_name, + initial_instance_count=1, + instance_type=INSTANCE_TYPE, + ), ], + session=boto_session, ) - sagemaker_client.create_endpoint( - EndpointName=endpoint_name, - EndpointConfigName=endpoint_name, + # 3. Create the Endpoint and wait for it to come InService. + endpoint = Endpoint.create( + endpoint_name=endpoint_name, + endpoint_config_name=endpoint_name, + session=boto_session, ) - wait_for_endpoint(endpoint_name) + endpoint.wait_for_status("InService") + # 4. Invoke. ``Endpoint.invoke`` returns an InvokeEndpointOutput whose + # ``body`` is a streaming bytes-like object. payload = json.dumps({"instances": [[1.0, 2.0, 3.0]]}) - response = sagemaker_runtime_client.invoke_endpoint( - EndpointName=endpoint_name, - ContentType="application/json", - Body=payload, + result = endpoint.invoke( + body=payload, + content_type="application/json", + accept="application/json", ) - body = _decode(response) + response = json.loads(result.body.read().decode("utf-8")) - assert "predictions" in body, f"missing predictions key in {body!r}" - predictions = body["predictions"] + assert "predictions" in response, f"missing predictions key in {response!r}" + predictions = response["predictions"] assert predictions and isinstance(predictions, list) # Output signature is {"output": x * 2.0} -> TFS surfaces the tensor