Skip to content

Commit 02a29f2

Browse files
ranmanclaude
ranman
and
claude
committed
Switch to Docker container Lambda, add EXIF extraction, and harden error paths
CDK/Lambda: - Replace fragile Lambda layer with DockerImageFunction (single shared image) - Add lambda/Dockerfile and root .dockerignore - Add exclude list to fromImageAsset to prevent recursive cdk.out staging - Fix handler routing to use ACTION env var (CDK sets it per function) - Make merge_manifests resilient to missing partial keys (skips with counter) - Fix CDK test mock to extend Construct with grantPull/addResourceMetadata - Add autoCleanup prop for dev/test stack teardown EXIF metadata extraction: - Add 12 EXIF fields to ImageRecord (camera, lens, focal length, exposure, GPS, etc.) - Extract via PIL getexif() with IFD0 + ExifIFD sub-IFD fallback - Compute distortion_risk from 35mm focal length (<18mm=high, 18-24=medium) - Flag GPS presence for privacy awareness - Add EXIF summary stats to DatasetSummary (camera counts, distortion, GPS) - Add --skip-exif CLI flag and ScanConfig option - 8 new EXIF tests with programmatic EXIF fixture images Error-path tests: - Test ACTION env var routing and case-insensitive fallback - Test analyze_batch with corrupt images (is_corrupt=True, not errors) - Test merge with missing S3 keys and failed Map state results - Integration test scaffolding 197 Python tests, 10 CDK tests — all passing. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 411373d commit 02a29f2

39 files changed

Lines changed: 6913 additions & 338 deletions

‎.dockerignore‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
.git
2+
.venv
3+
__pycache__
4+
*.pyc
5+
cdk/
6+
tests/
7+
.github/
8+
.claude/
9+
*.egg-info
10+
dist/
11+
build/
12+
node_modules/
13+
.ruff_cache/
14+
.mypy_cache/
15+
.pytest_cache/
16+
htmlcov/

‎cdk/bin/app.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ import * as cdk from 'aws-cdk-lib';
33
import { ImgedaStack } from '../lib/imgeda-stack';
44

55
const app = new cdk.App();
6+
7+
// Use -c autoCleanup=true for test deployments that should auto-delete on destroy
8+
const autoCleanup = app.node.tryGetContext('autoCleanup') === 'true';
9+
610
new ImgedaStack(app, 'ImgedaStack', {
711
description: 'Serverless image dataset EDA pipeline using Step Functions',
12+
autoCleanup,
813
});

‎cdk/lib/imgeda-stack.ts‎

Lines changed: 103 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,37 @@ import * as sfn from 'aws-cdk-lib/aws-stepfunctions';
55
import * as tasks from 'aws-cdk-lib/aws-stepfunctions-tasks';
66
import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch';
77
import { Construct } from 'constructs';
8+
import * as path from 'path';
9+
10+
export interface ImgedaStackProps extends cdk.StackProps {
11+
/** Auto-delete bucket contents on stack destroy. Use for dev/test only. */
12+
readonly autoCleanup?: boolean;
13+
}
814

915
export class ImgedaStack extends cdk.Stack {
1016
public readonly inputBucket: s3.Bucket;
1117
public readonly outputBucket: s3.Bucket;
1218
public readonly stateMachine: sfn.StateMachine;
1319

14-
constructor(scope: Construct, id: string, props?: cdk.StackProps) {
20+
constructor(scope: Construct, id: string, props?: ImgedaStackProps) {
1521
super(scope, id, props);
1622

23+
const autoCleanup = props?.autoCleanup ?? false;
24+
const removalPolicy = autoCleanup
25+
? cdk.RemovalPolicy.DESTROY
26+
: cdk.RemovalPolicy.RETAIN;
27+
1728
// --- S3 Buckets ---
1829
this.inputBucket = new s3.Bucket(this, 'InputBucket', {
19-
removalPolicy: cdk.RemovalPolicy.RETAIN,
30+
removalPolicy,
31+
autoDeleteObjects: autoCleanup,
2032
encryption: s3.BucketEncryption.S3_MANAGED,
2133
blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
2234
});
2335

2436
this.outputBucket = new s3.Bucket(this, 'OutputBucket', {
25-
removalPolicy: cdk.RemovalPolicy.RETAIN,
37+
removalPolicy,
38+
autoDeleteObjects: autoCleanup,
2639
encryption: s3.BucketEncryption.S3_MANAGED,
2740
blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
2841
lifecycleRules: [
@@ -33,74 +46,68 @@ export class ImgedaStack extends cdk.Stack {
3346
],
3447
});
3548

36-
// --- Lambda Layer ---
37-
const imgedaLayer = new lambda.LayerVersion(this, 'ImgedaLayer', {
38-
code: lambda.Code.fromAsset('../', {
39-
bundling: {
40-
image: lambda.Runtime.PYTHON_3_12.bundlingImage,
41-
command: [
42-
'bash', '-c',
43-
'pip install . -t /asset-output/python && cp -r src/imgeda /asset-output/python/',
44-
],
45-
},
46-
}),
47-
compatibleRuntimes: [lambda.Runtime.PYTHON_3_12],
48-
description: 'imgeda package and dependencies',
49+
// --- Docker Image Lambda ---
50+
// All 5 Lambda functions share one Docker image built from lambda/Dockerfile.
51+
// Each function sets an ACTION env var to route to the correct handler.
52+
const projectRoot = path.resolve(__dirname, '../..');
53+
const imageCode = lambda.DockerImageCode.fromImageAsset(projectRoot, {
54+
file: 'lambda/Dockerfile',
55+
exclude: [
56+
'cdk',
57+
'tests',
58+
'.venv',
59+
'.git',
60+
'node_modules',
61+
'.github',
62+
'.claude',
63+
'dist',
64+
'build',
65+
'*.egg-info',
66+
'.ruff_cache',
67+
'.mypy_cache',
68+
'.pytest_cache',
69+
'htmlcov',
70+
],
4971
});
5072

51-
// --- Lambda Functions ---
52-
const commonLambdaProps: Partial<lambda.FunctionProps> = {
53-
runtime: lambda.Runtime.PYTHON_3_12,
73+
const listImagesFn = new lambda.DockerImageFunction(this, 'ListImagesFn', {
74+
code: imageCode,
5475
memorySize: 1024,
5576
timeout: cdk.Duration.minutes(5),
56-
layers: [imgedaLayer],
57-
handler: 'imgeda.lambda_handler.handler.handler',
58-
};
59-
60-
const listImagesFn = new lambda.Function(this, 'ListImagesFn', {
61-
...commonLambdaProps,
62-
code: lambda.Code.fromInline('# handler in layer'),
63-
handler: 'imgeda.lambda_handler.handler.handler',
6477
description: 'List images in S3 bucket and split into batches',
65-
timeout: cdk.Duration.minutes(5),
6678
environment: { ACTION: 'list_images' },
6779
});
6880

69-
const analyzeBatchFn = new lambda.Function(this, 'AnalyzeBatchFn', {
70-
...commonLambdaProps,
71-
code: lambda.Code.fromInline('# handler in layer'),
72-
handler: 'imgeda.lambda_handler.handler.handler',
73-
description: 'Analyze a batch of images',
81+
const analyzeBatchFn = new lambda.DockerImageFunction(this, 'AnalyzeBatchFn', {
82+
code: imageCode,
7483
memorySize: 2048,
7584
timeout: cdk.Duration.minutes(10),
7685
ephemeralStorageSize: cdk.Size.gibibytes(2),
86+
description: 'Analyze a batch of images',
7787
environment: { ACTION: 'analyze_batch' },
7888
});
7989

80-
const mergeManifestsFn = new lambda.Function(this, 'MergeManifestsFn', {
81-
...commonLambdaProps,
82-
code: lambda.Code.fromInline('# handler in layer'),
83-
handler: 'imgeda.lambda_handler.handler.handler',
84-
description: 'Merge partial manifests into final JSONL',
90+
const mergeManifestsFn = new lambda.DockerImageFunction(this, 'MergeManifestsFn', {
91+
code: imageCode,
92+
memorySize: 1024,
8593
timeout: cdk.Duration.minutes(10),
94+
description: 'Merge partial manifests into final JSONL',
8695
environment: { ACTION: 'merge_manifests' },
8796
});
8897

89-
const aggregateFn = new lambda.Function(this, 'AggregateFn', {
90-
...commonLambdaProps,
91-
code: lambda.Code.fromInline('# handler in layer'),
92-
handler: 'imgeda.lambda_handler.handler.handler',
98+
const aggregateFn = new lambda.DockerImageFunction(this, 'AggregateFn', {
99+
code: imageCode,
100+
memorySize: 1024,
101+
timeout: cdk.Duration.minutes(5),
93102
description: 'Compute aggregate statistics from manifest',
94103
environment: { ACTION: 'aggregate' },
95104
});
96105

97-
const generatePlotsFn = new lambda.Function(this, 'GeneratePlotsFn', {
98-
...commonLambdaProps,
99-
code: lambda.Code.fromInline('# handler in layer'),
100-
handler: 'imgeda.lambda_handler.handler.handler',
101-
description: 'Generate visualization plots from manifest',
106+
const generatePlotsFn = new lambda.DockerImageFunction(this, 'GeneratePlotsFn', {
107+
code: imageCode,
102108
memorySize: 2048,
103109
timeout: cdk.Duration.minutes(10),
110+
description: 'Generate visualization plots from manifest',
104111
environment: { ACTION: 'generate_plots' },
105112
});
106113

@@ -114,36 +121,79 @@ export class ImgedaStack extends cdk.Stack {
114121
this.outputBucket.grantReadWrite(generatePlotsFn);
115122

116123
// --- Step Functions Workflow ---
124+
//
125+
// Execution input schema:
126+
// {
127+
// "input_bucket": "<bucket-with-images>",
128+
// "prefix": "images/",
129+
// "output_bucket": "<bucket-for-results>"
130+
// }
131+
//
132+
// Each task uses `resultPath` to preserve the full state across steps,
133+
// and `payload` to construct the handler-specific input from state fields.
134+
117135
const listImagesTask = new tasks.LambdaInvoke(this, 'ListImages', {
118136
lambdaFunction: listImagesFn,
119-
outputPath: '$.Payload',
137+
payload: sfn.TaskInput.fromObject({
138+
'bucket.$': '$.input_bucket',
139+
'prefix.$': '$.prefix',
140+
}),
141+
resultPath: '$.list_result',
142+
payloadResponseOnly: true,
120143
});
121144

122145
const analyzeBatchTask = new tasks.LambdaInvoke(this, 'AnalyzeBatch', {
123146
lambdaFunction: analyzeBatchFn,
124-
outputPath: '$.Payload',
147+
payloadResponseOnly: true,
125148
});
126149

127150
const analyzeMap = new sfn.Map(this, 'AnalyzeBatches', {
128-
itemsPath: '$.batches',
151+
itemsPath: '$.list_result.batches',
129152
maxConcurrency: 10,
130153
resultPath: '$.analyze_results',
154+
itemSelector: {
155+
'source_bucket.$': '$.input_bucket',
156+
'keys.$': '$$.Map.Item.Value',
157+
'output_bucket.$': '$.output_bucket',
158+
'output_key.$':
159+
"States.Format('partials/batch-{}.jsonl', $$.Map.Item.Index)",
160+
},
131161
});
132162
analyzeMap.itemProcessor(analyzeBatchTask);
133163

134164
const mergeTask = new tasks.LambdaInvoke(this, 'MergeManifests', {
135165
lambdaFunction: mergeManifestsFn,
136-
outputPath: '$.Payload',
166+
payload: sfn.TaskInput.fromObject({
167+
'bucket.$': '$.output_bucket',
168+
'analyze_results.$': '$.analyze_results',
169+
'output_key': 'manifests/manifest.jsonl',
170+
'input_dir.$':
171+
"States.Format('s3://{}/{}', $.input_bucket, $.prefix)",
172+
}),
173+
resultPath: '$.merge_result',
174+
payloadResponseOnly: true,
137175
});
138176

139177
const aggregateTask = new tasks.LambdaInvoke(this, 'Aggregate', {
140178
lambdaFunction: aggregateFn,
141-
outputPath: '$.Payload',
179+
payload: sfn.TaskInput.fromObject({
180+
'bucket.$': '$.output_bucket',
181+
'manifest_key.$': '$.merge_result.output_key',
182+
'output_key': 'summary/summary.json',
183+
}),
184+
resultPath: '$.aggregate_result',
185+
payloadResponseOnly: true,
142186
});
143187

144188
const generatePlotsTask = new tasks.LambdaInvoke(this, 'GeneratePlots', {
145189
lambdaFunction: generatePlotsFn,
146-
outputPath: '$.Payload',
190+
payload: sfn.TaskInput.fromObject({
191+
'bucket.$': '$.output_bucket',
192+
'manifest_key.$': '$.merge_result.output_key',
193+
'output_prefix': 'plots/',
194+
}),
195+
resultPath: '$.plots_result',
196+
payloadResponseOnly: true,
147197
});
148198

149199
const definition = listImagesTask

0 commit comments

Comments
 (0)