diff --git a/lib/layer/index.ts b/lib/layer/index.ts index 9505b293c..ba9469661 100644 --- a/lib/layer/index.ts +++ b/lib/layer/index.ts @@ -1,6 +1,8 @@ import * as cdk from "aws-cdk-lib"; import * as lambda from "aws-cdk-lib/aws-lambda"; import * as s3assets from "aws-cdk-lib/aws-s3-assets"; +import * as lpath from "path"; +import { execSync } from "child_process"; import { Construct } from "constructs"; interface LayerProps { @@ -26,6 +28,58 @@ export class Layer extends Construct { const layerAsset = new s3assets.Asset(this, "LayerAsset", { path, bundling: { + local: { + /* implements a local method of bundling that does not depend on Docker. Local + bundling is preferred over DIND for performance and security reasons. + see https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.ILocalBundling.html and + https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3_assets-readme.html#asset-bundling */ + tryBundle(outputDir: string, options: cdk.BundlingOptions) { + let canRunLocal = false; + let python = props.runtime.name; + + /* check if local machine architecture matches lambda runtime architecture. annoyingly, + Node refers to x86_64 CPUs as x64 instead of using the POSIX standard name. + https://nodejs.org/docs/latest-v18.x/api/process.html#processarch + https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_lambda.Architecture.html */ + if (!((process.arch == 'x64' && architecture.name == 'x86_64') || (process.arch == architecture.name))) { + console.log(`Can't do local bundling because local arch != target arch (${process.arch} != ${architecture.name})`); + // Local bundling is pointless if architectures don't match + return false; + } + + try { + // check if pip is available locally + const testCommand = `${python} -m pip -V` + console.log(`Checking for pip: ${testCommand}`) + // without the stdio arg no output is printed to console + execSync(testCommand, { stdio: 'inherit' }); + // no exception means command executed successfully + canRunLocal = true; + } catch { + // execSync throws Error in case return value of child process is non-zero. + // Actual output should be printed to the console. + console.warn(`Unable to do local bundling! ${python} with pip must be on path.`); + } + + if (canRunLocal) { + const pkgDir = lpath.posix.join(outputDir, "python"); + const command = `${python} -m pip install -r ${lpath.posix.join(path, "requirements.txt")} -t ${pkgDir} ${autoUpgrade ? '-U' : ''}`; + try { + console.debug(`Local bundling: ${command}`); + // this is where the work gets done + execSync(command, { stdio: 'inherit' }); + return true; + } catch (ex) { + // execSync throws Error in case return value of child process + // is non-zero. It'll be printed to the console because of the + // stdio argument. + console.log(`Local bundling attempt failed: ${ex}`) + } + } + // if we get here then Docker will be used as configured below + return false; + } + }, image: runtime.bundlingImage, platform: architecture.dockerPlatform, command: [ diff --git a/lib/shared/shared-asset-bundler.ts b/lib/shared/shared-asset-bundler.ts index 960fe2d37..e13c71079 100644 --- a/lib/shared/shared-asset-bundler.ts +++ b/lib/shared/shared-asset-bundler.ts @@ -3,6 +3,7 @@ import { BundlingOutput, DockerImage, aws_s3_assets, + BundlingOptions } from "aws-cdk-lib"; import { Code, S3Code } from "aws-cdk-lib/aws-lambda"; import { Asset } from "aws-cdk-lib/aws-s3-assets"; @@ -10,6 +11,7 @@ import { md5hash } from "aws-cdk-lib/core/lib/helpers-internal"; import { Construct } from "constructs"; import * as path from "path"; import * as fs from "fs"; +import { execSync } from "child_process"; function calculateHash(paths: string[]): string { return paths.reduce((mh, p) => { @@ -33,6 +35,47 @@ function calculateHash(paths: string[]): string { export class SharedAssetBundler extends Construct { private readonly sharedAssets: string[]; private readonly WORKING_PATH = "/asset-input/"; + // see static init block below + private static useLocalBundler: boolean = false; + /** The container image we'll use if Local Bundling is not possible. */ + private static containerImage: DockerImage; + + /** + * Check if possible to use local bundling instead of Docker. Sets `useLocalBundler` to + * true if local environment supports bundling. Referenced below in method bundleWithAsset(...). + */ + static { + const command = "zip -v"; + console.log(`Checking for zip: ${command}`); + // check if zip is available locally + try { + // without stdio option command output does not appear in console + execSync(command, { stdio: 'inherit' }); + // no exception means command executed successfully + this.useLocalBundler = true; + } catch { + /* execSync throws Error in case return value of child process + is non-zero. Actual output should be printed to the console. */ + console.warn("`zip` is required for local bundling; falling back to default method."); + } + + try { + /** Build Alpine image from local definition. */ + this.containerImage = DockerImage.fromBuild(path.posix.join(__dirname, "alpine-zip")); + } catch (erx) { + // this will result in an exception if Docker is unavailable + if (this.useLocalBundler) { + /* we don't actually need the container if local bundling succeeds, but + it is a required parameter in the method below. + https://hub.docker.com/_/scratch/ */ + this.containerImage = DockerImage.fromRegistry("scratch"); + } else { + // Build will fail anyway so no point suppressing the exception + throw erx; + } + } + } + /** * Instantiate a new SharedAssetBundler. You then invoke `bundleWithAsset(pathToAsset)` to * bundle your asset code with the common code. @@ -51,21 +94,52 @@ export class SharedAssetBundler extends Construct { bundleWithAsset(assetPath: string): Asset { console.log(`Bundling asset ${assetPath}`); + // necessary for access from anonymous class + const thisAssets = this.sharedAssets; + const asset = new aws_s3_assets.Asset( this, md5hash(assetPath).slice(0, 6), { path: assetPath, bundling: { - image: DockerImage.fromBuild( - path.posix.join(__dirname, "alpine-zip") - ), - command: [ - "zip", - "-r", - path.posix.join("/asset-output", "asset.zip"), - ".", - ], + local: { + /* implements a local method of bundling that does not depend on Docker. Local + bundling is preferred over DIND for performance and security reasons. + see https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.ILocalBundling.html and + https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3_assets-readme.html#asset-bundling */ + tryBundle(outputDir: string, options: BundlingOptions) { + if (SharedAssetBundler.useLocalBundler) { + // base command to execute + const command = `zip -r ${path.posix.join(outputDir, "asset.zip")} . `; + + try { + console.debug(`Local bundling: ${assetPath}`); + // cd to dir of current asset and zip contents + execSync(`cd ${assetPath} && `.concat(command), {stdio: 'inherit'}); + // do the same for each dir in shared assets array + thisAssets.forEach((a)=>{ + /* Complete the command for this specific shared asset path; for example: + `cd ${assetPath}/.. && ${command} -i ${assetPath.basename}/*` */ + const cx = `cd ${path.posix.join(a, '..')} && `.concat(command).concat(`-i "${path.basename(a)}/*"`); + //execute the command in child process + execSync(cx, {stdio: 'inherit'}); + }); + // no exception means command executed successfully + return true; + } catch (ex) { + // execSync throws Error in case return value of child process + // is non-zero. It'll be printed to the console because of the + // stdio argument. + console.log(`local bundling attempt failed: ${ex}`) + } + } + // if we get here then Docker will be used as configured below + return false; + } + }, + image: SharedAssetBundler.containerImage, + command: ["zip", "-r", path.posix.join("/asset-output", "asset.zip"), "."], volumes: this.sharedAssets.map((f) => ({ containerPath: path.posix.join(this.WORKING_PATH, path.basename(f)), hostPath: f, @@ -77,6 +151,7 @@ export class SharedAssetBundler extends Construct { assetHashType: AssetHashType.CUSTOM, } ); + console.log(`Successfully bundled ${asset.toString()} shared assets for ${assetPath} as ${asset.s3ObjectKey}.`); return asset; }