Skip to content

Commit 6cc7bcc

Browse files
wutimotwutimot
authored
Merge branch 'master' into feature/emr-serverless-step
2 parents a2374aa + efa8d7d commit 6cc7bcc

File tree

5 files changed

+1060
-144
lines changed

5 files changed

+1060
-144
lines changed

‎.github/workflows/pr-checks-master.yml‎

Lines changed: 63 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -74,32 +74,79 @@ jobs:
7474
echo "Changed files:"
7575
echo "$CHANGES"
7676
77-
SUBMODULES=[]
77+
# Function to extract dependencies from pyproject.toml
78+
get_dependencies() {
79+
local module=$1
80+
grep "sagemaker-" "$module/pyproject.toml" | grep -o 'sagemaker-[a-z]*' | sort -u
81+
}
82+
83+
# Function to find all modules that depend on a given module (recursively)
84+
find_dependents() {
85+
local target=$1
86+
local all_modules=("sagemaker-core" "sagemaker-train" "sagemaker-serve" "sagemaker-mlops")
87+
local dependents=()
88+
89+
for module in "${all_modules[@]}"; do
90+
if [ "$module" != "$target" ]; then
91+
if get_dependencies "$module" | grep -q "^$target$"; then
92+
dependents+=("$module")
93+
fi
94+
fi
95+
done
96+
97+
echo "${dependents[@]}"
98+
}
99+
100+
# Initialize set of submodules to test (using associative array)
101+
declare -A SUBMODULES_SET
102+
103+
# Function to recursively add module and all its dependents
104+
add_module_and_dependents() {
105+
local module=$1
106+
107+
if [ -z "${SUBMODULES_SET[$module]}" ]; then
108+
SUBMODULES_SET["$module"]=1
109+
echo "Adding $module to test set"
110+
111+
# Find all modules that depend on this one and add them recursively
112+
local dependents=$(find_dependents "$module")
113+
for dependent in $dependents; do
114+
add_module_and_dependents "$dependent"
115+
done
116+
fi
117+
}
118+
119+
# Check which submodules changed and add them plus their dependents
120+
if echo "$CHANGES" | grep -q "^sagemaker-core/"; then
121+
echo "sagemaker-core changed - will add core and all dependents"
122+
add_module_and_dependents "sagemaker-core"
123+
fi
78124
79125
if echo "$CHANGES" | grep -q "^sagemaker-train/"; then
80-
SUBMODULES='["sagemaker-train"]'
126+
echo "sagemaker-train changed - will add train and all dependents"
127+
add_module_and_dependents "sagemaker-train"
81128
fi
129+
82130
if echo "$CHANGES" | grep -q "^sagemaker-serve/"; then
83-
if [ "$SUBMODULES" = '[]' ]; then
84-
SUBMODULES='["sagemaker-serve"]'
85-
else
86-
SUBMODULES=$(echo $SUBMODULES | sed 's/\]$/,"sagemaker-serve"\]/')
87-
fi
131+
echo "sagemaker-serve changed - will add serve and all dependents"
132+
add_module_and_dependents "sagemaker-serve"
88133
fi
134+
89135
if echo "$CHANGES" | grep -q "^sagemaker-mlops/"; then
90-
if [ "$SUBMODULES" = '[]' ]; then
91-
SUBMODULES='["sagemaker-mlops"]'
92-
else
93-
SUBMODULES=$(echo $SUBMODULES | sed 's/\]$/,"sagemaker-mlops"\]/')
94-
fi
136+
echo "sagemaker-mlops changed - will add mlops"
137+
add_module_and_dependents "sagemaker-mlops"
95138
fi
96-
if echo "$CHANGES" | grep -q "^sagemaker-core/"; then
139+
140+
# Convert associative array to JSON array
141+
SUBMODULES='[]'
142+
for submodule in "${!SUBMODULES_SET[@]}"; do
97143
if [ "$SUBMODULES" = '[]' ]; then
98-
SUBMODULES='["sagemaker-core"]'
144+
SUBMODULES="[\"$submodule\"]"
99145
else
100-
SUBMODULES=$(echo $SUBMODULES | sed 's/\]$/,"sagemaker-core"\]/')
146+
SUBMODULES=$(echo $SUBMODULES | sed "s/\]$/,\"$submodule\"\]/")
101147
fi
102-
fi
148+
done
149+
103150
echo "Final SUBMODULES: $SUBMODULES"
104151
echo "submodules=$SUBMODULES" >> $GITHUB_OUTPUT
105152

‎sagemaker-core/src/sagemaker/core/processing.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -625,6 +625,8 @@ def submit(request):
625625
from sagemaker.core.utils.code_injection.codec import transform
626626

627627
transformed = transform(serialized_request, "CreateProcessingJobRequest")
628+
# Remove tags from transformed dict as ProcessingJob resource doesn't accept it
629+
transformed.pop("tags", None)
628630
return ProcessingJob(**transformed)
629631

630632
def _get_process_args(self, inputs, outputs, experiment_config):

‎sagemaker-core/tests/unit/test_processing.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1088,6 +1088,48 @@ def test_get_process_args_with_pipeline_variable_role(self, mock_session):
10881088
assert args["role_arn"] == role_var
10891089

10901090

1091+
class TestProcessorStartNewWithTags:
1092+
def test_start_new_removes_tags_from_processing_job(self, mock_session):
1093+
"""Test that tags are removed from transformed dict before ProcessingJob creation"""
1094+
processor = Processor(
1095+
role="arn:aws:iam::123456789012:role/SageMakerRole",
1096+
image_uri="test-image:latest",
1097+
instance_count=1,
1098+
instance_type="ml.m5.xlarge",
1099+
tags=[("Key", "Value")],
1100+
sagemaker_session=mock_session,
1101+
)
1102+
processor._current_job_name = "test-job"
1103+
1104+
with patch.object(
1105+
processor,
1106+
"_get_process_args",
1107+
return_value={
1108+
"job_name": "test-job",
1109+
"inputs": [],
1110+
"output_config": {"Outputs": []},
1111+
"resources": {"ClusterConfig": {}},
1112+
"stopping_condition": None,
1113+
"app_specification": {"ImageUri": "test-image"},
1114+
"environment": None,
1115+
"network_config": None,
1116+
"role_arn": "arn:aws:iam::123456789012:role/SageMakerRole",
1117+
"tags": [{"Key": "Key", "Value": "Value"}],
1118+
},
1119+
):
1120+
with patch("sagemaker.core.processing.serialize", return_value={"tags": [{"Key": "Key", "Value": "Value"}]}):
1121+
with patch("sagemaker.core.processing.ProcessingJob") as mock_job_class:
1122+
with patch(
1123+
"sagemaker.core.utils.code_injection.codec.transform",
1124+
return_value={"processing_job_name": "test-job", "tags": [{"Key": "Key", "Value": "Value"}]},
1125+
):
1126+
processor._start_new([], [], None)
1127+
# Verify ProcessingJob was called without tags
1128+
mock_job_class.assert_called_once()
1129+
call_kwargs = mock_job_class.call_args[1]
1130+
assert "tags" not in call_kwargs
1131+
1132+
10911133
# Additional tests from test_processing_extended.py
10921134
class TestProcessorBasics:
10931135
"""Test cases for basic Processor functionality"""

0 commit comments

Comments
 (0)