Skip to content

Commit d098ee1

Browse files
committed
fix(sagemaker-templates): Fix batch inference parameter resolution
The batch inference template was passing ParameterString objects directly to job_arguments, which caused them to be serialized as literal JSON strings instead of being resolved at runtime. Changes: - Use ProcessingInput with parameter as source for dynamic S3 paths - Specify explicit destination filename to avoid path ambiguity - Follows the same pattern as xgboost_abalone template This fixes the issue where InputDataUrl parameter was passed as '{"Get": "Parameters.InputDataUrl"}' instead of the actual S3 path, causing preprocessing jobs to fail with IndexError. Resolves parameter resolution for batch inference pipelines.
1 parent 6140600 commit d098ee1

2 files changed

Lines changed: 22 additions & 9 deletions

File tree

  • modules/sagemaker/sagemaker-templates/templates/batch_inference/seed_code/build_app

modules/sagemaker/sagemaker-templates/templates/batch_inference/seed_code/build_app/ml_pipelines/transformer/pipeline.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import sagemaker.session
77
from sagemaker import ModelPackage
88
from sagemaker.inputs import TransformInput
9-
from sagemaker.processing import ProcessingOutput
9+
from sagemaker.processing import ProcessingInput, ProcessingOutput
1010
from sagemaker.sklearn.processing import SKLearnProcessor
1111
from sagemaker.transformer import Transformer
1212
from sagemaker.workflow.execution_variables import ExecutionVariables
@@ -141,13 +141,19 @@ def get_pipeline(
141141
step_process = ProcessingStep(
142142
name="PreprocessData",
143143
processor=sklearn_processor,
144+
inputs=[
145+
ProcessingInput(
146+
source=input_data,
147+
destination="/opt/ml/processing/input/data.csv",
148+
),
149+
],
144150
outputs=[
145151
ProcessingOutput(output_name="output_data", source="/opt/ml/processing/output_data"),
146152
],
147153
code="source_scripts/preprocessing.py",
148154
job_arguments=[
149155
"--input-data",
150-
input_data,
156+
"/opt/ml/processing/input/data.csv",
151157
"--do-train-test-split",
152158
"False",
153159
],

modules/sagemaker/sagemaker-templates/templates/batch_inference/seed_code/build_app/source_scripts/preprocessing.py

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,20 @@ def merge_two_dicts(x: Dict[str, Any], y: Dict[str, Any]) -> Dict[str, Any]:
6363
pathlib.Path(f"{base_dir}/data").mkdir(parents=True, exist_ok=True)
6464
input_data = args.input_data
6565
logger.info("Input data path: %s", input_data)
66-
bucket = input_data.split("/")[2]
67-
key = "/".join(input_data.split("/")[3:])
68-
69-
logger.info("Downloading data from bucket: %s, key: %s", bucket, key)
70-
fn = f"{base_dir}/data/abalone-dataset.csv"
71-
s3 = boto3.resource("s3")
72-
s3.Bucket(bucket).download_file(key, fn)
66+
67+
# Check if input_data is an S3 URL or a local path
68+
if input_data.startswith("s3://"):
69+
# Download from S3
70+
bucket = input_data.split("/")[2]
71+
key = "/".join(input_data.split("/")[3:])
72+
logger.info("Downloading data from bucket: %s, key: %s", bucket, key)
73+
fn = f"{base_dir}/data/abalone-dataset.csv"
74+
s3 = boto3.resource("s3")
75+
s3.Bucket(bucket).download_file(key, fn)
76+
else:
77+
# Use local path (file already downloaded by ProcessingInput)
78+
logger.info("Using local input data path")
79+
fn = input_data
7380

7481
logger.debug("Reading downloaded data.")
7582
df = pd.read_csv(

0 commit comments

Comments
 (0)