Skip to content

Commit b5f4e37

Browse files
committed
fix(sagemaker-templates): Fix batch inference parameter resolution
The batch inference template was passing ParameterString objects directly to job_arguments, which caused them to be serialized as literal JSON strings instead of being resolved at runtime. Changes: - Use ProcessingInput with parameter as source for dynamic S3 paths - Pass S3 URL parameter to script so it can extract filename - Script constructs local path from filename - Remove S3 download logic (ProcessingInput handles it) This fixes the issue where InputDataUrl parameter was passed as '{"Get": "Parameters.InputDataUrl"}' instead of the actual S3 path, causing preprocessing jobs to fail with IndexError. Resolves parameter resolution for batch inference pipelines.
1 parent d098ee1 commit b5f4e37

2 files changed

Lines changed: 7 additions & 17 deletions

File tree

  • modules/sagemaker/sagemaker-templates/templates/batch_inference/seed_code/build_app

modules/sagemaker/sagemaker-templates/templates/batch_inference/seed_code/build_app/ml_pipelines/transformer/pipeline.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ def get_pipeline(
144144
inputs=[
145145
ProcessingInput(
146146
source=input_data,
147-
destination="/opt/ml/processing/input/data.csv",
147+
destination="/opt/ml/processing/input",
148148
),
149149
],
150150
outputs=[
@@ -153,7 +153,7 @@ def get_pipeline(
153153
code="source_scripts/preprocessing.py",
154154
job_arguments=[
155155
"--input-data",
156-
"/opt/ml/processing/input/data.csv",
156+
input_data,
157157
"--do-train-test-split",
158158
"False",
159159
],

modules/sagemaker/sagemaker-templates/templates/batch_inference/seed_code/build_app/source_scripts/preprocessing.py

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -62,23 +62,13 @@ def merge_two_dicts(x: Dict[str, Any], y: Dict[str, Any]) -> Dict[str, Any]:
6262
base_dir = "/opt/ml/processing"
6363
pathlib.Path(f"{base_dir}/data").mkdir(parents=True, exist_ok=True)
6464
input_data = args.input_data
65-
logger.info("Input data path: %s", input_data)
65+
logger.info("Input data S3 URL: %s", input_data)
6666

67-
# Check if input_data is an S3 URL or a local path
68-
if input_data.startswith("s3://"):
69-
# Download from S3
70-
bucket = input_data.split("/")[2]
71-
key = "/".join(input_data.split("/")[3:])
72-
logger.info("Downloading data from bucket: %s, key: %s", bucket, key)
73-
fn = f"{base_dir}/data/abalone-dataset.csv"
74-
s3 = boto3.resource("s3")
75-
s3.Bucket(bucket).download_file(key, fn)
76-
else:
77-
# Use local path (file already downloaded by ProcessingInput)
78-
logger.info("Using local input data path")
79-
fn = input_data
67+
# Extract filename from S3 URL - file is already downloaded by ProcessingInput
68+
filename = input_data.split('/')[-1]
69+
fn = f"/opt/ml/processing/input/{filename}"
70+
logger.info("Reading file: %s", fn)
8071

81-
logger.debug("Reading downloaded data.")
8272
df = pd.read_csv(
8373
fn,
8474
header=None,

0 commit comments

Comments
 (0)