diff --git a/packages/aws-cdk-lib/aws-fsx/README.md b/packages/aws-cdk-lib/aws-fsx/README.md index ab2f623d221d3..5bc6f9771740d 100644 --- a/packages/aws-cdk-lib/aws-fsx/README.md +++ b/packages/aws-cdk-lib/aws-fsx/README.md @@ -178,11 +178,64 @@ fs.connections.allowDefaultPortFrom(inst); ### Lustre Data Repository Association support -The LustreFilesystem Construct supports one [Data Repository Association](https://docs.aws.amazon.com/fsx/latest/LustreGuide/fsx-data-repositories.html) (DRA) to an S3 bucket. This allows Lustre hierarchical storage management to S3 buckets, which in turn makes it possible to use S3 as a permanent backing store, and use FSx for Lustre as a temporary high performance cache. +A [Data Repository Association](https://docs.aws.amazon.com/fsx/latest/LustreGuide/overview-dra-data-repo.html) (DRA) +links an S3 bucket (or a prefix within one) to a path on the Lustre file system. This enables bidirectional data +movement: files written to the file system path can be automatically exported to S3, and objects appearing in S3 can +be automatically imported into the file system. -Note: CloudFormation does not currently support for `PERSISTENT_2` filesystems, and so neither does CDK. +Data repository associations are supported on FSx for Lustre 2.12 and 2.15 file systems with `SCRATCH_2`, +`PERSISTENT_1`, or `PERSISTENT_2` deployment types. They are **not** supported on `SCRATCH_1` file systems. -The following example illustrates setting up a DRA to an S3 bucket, including automated metadata import whenever a file is changed, created or deleted in the S3 bucket: +> **Note:** DRAs and the legacy `importPath`/`exportPath`/`autoImportPolicy` properties on `LustreConfiguration` are +> mutually exclusive. Use one approach or the other, not both. + +#### Using the L2 DataRepositoryAssociation construct (recommended) + +```ts +import { aws_s3 as s3 } from 'aws-cdk-lib'; + +declare const vpc: ec2.Vpc; +declare const bucket: s3.Bucket; + +const fileSystem = new fsx.LustreFileSystem(this, 'FsxLustreFileSystem', { + lustreConfiguration: { deploymentType: fsx.LustreDeploymentType.SCRATCH_2 }, + storageCapacityGiB: 1200, + vpc, + vpcSubnet: vpc.privateSubnets[0], + fileSystemTypeVersion: fsx.FileSystemTypeVersion.V_2_15, +}); + +new fsx.DataRepositoryAssociation(this, 'DataRepositoryAssociation', { + fileSystem, + fileSystemPath: '/data', + bucket, + bucketPrefix: 'my-prefix', + s3: { + autoImportPolicy: { + events: [ + fsx.DataRepositoryEventType.NEW, + fsx.DataRepositoryEventType.CHANGED, + fsx.DataRepositoryEventType.DELETED, + ], + }, + autoExportPolicy: { + events: [ + fsx.DataRepositoryEventType.NEW, + fsx.DataRepositoryEventType.CHANGED, + fsx.DataRepositoryEventType.DELETED, + ], + }, + }, +}); +``` + +The `DataRepositoryAssociation` construct automatically grants `fsx.amazonaws.com` read/write access to the bucket and +ensures the bucket policy is created before the association via an explicit CloudFormation dependency. + +#### Using legacy importPath/exportPath (SCRATCH_1 and PERSISTENT_1 only) + +The older `importPath`/`exportPath` approach on `LustreConfiguration` is still available for `SCRATCH_1` and older +`PERSISTENT_1` workloads, but Data Repository Associations are preferred for all new file systems: ```ts import { aws_s3 as s3 } from 'aws-cdk-lib'; @@ -197,7 +250,7 @@ const lustreConfiguration = { autoImportPolicy: fsx.LustreAutoImportPolicy.NEW_CHANGED_DELETED, }; -const fs = new fsx.LustreFileSystem(this, "FsxLustreFileSystem", { +const fs = new fsx.LustreFileSystem(this, 'FsxLustreFileSystem', { vpc: vpc, vpcSubnet: vpc.privateSubnets[0], storageCapacityGiB: 1200, diff --git a/packages/aws-cdk-lib/aws-fsx/lib/data-repository-association.ts b/packages/aws-cdk-lib/aws-fsx/lib/data-repository-association.ts new file mode 100644 index 0000000000000..a33494b6053eb --- /dev/null +++ b/packages/aws-cdk-lib/aws-fsx/lib/data-repository-association.ts @@ -0,0 +1,231 @@ +import type { Construct } from 'constructs'; +import type { IFileSystem } from './file-system'; +import { CfnDataRepositoryAssociation } from './fsx.generated'; + +import { ServicePrincipal } from '../../aws-iam'; +import type { IBucket } from '../../aws-s3'; +import { RemovalPolicy, Resource, Token, ValidationError } from '../../core'; +import { addConstructMetadata } from '../../core/lib/metadata-resource'; +import { lit } from '../../core/lib/private/literal-string'; +import { propertyInjectable } from '../../core/lib/prop-injectable'; + +/** + * Event types that trigger automatic import or export for a DataRepositoryAssociation. + */ +export enum DataRepositoryEventType { + /** + * New files or directories added to the S3 bucket / file system. + */ + NEW = 'NEW', + + /** + * Files or directories changed in the S3 bucket / file system. + */ + CHANGED = 'CHANGED', + + /** + * Files or directories deleted in the S3 bucket / file system. + */ + DELETED = 'DELETED', +} + +/** + * Automatic import policy: which S3 events trigger import into the Lustre file system. + */ +export interface S3AutoImportPolicy { + /** + * The S3 events that will trigger an automatic import into the file system. + */ + readonly events: DataRepositoryEventType[]; +} + +/** + * Automatic export policy: which file system events trigger export to S3. + */ +export interface S3AutoExportPolicy { + /** + * The file system events that will trigger an automatic export to S3. + */ + readonly events: DataRepositoryEventType[]; +} + +/** + * S3 data repository configuration for a DataRepositoryAssociation. + */ +export interface S3DataRepositoryConfiguration { + /** + * Defines which S3 events automatically import new file metadata into the Lustre file system. + * + * @default - no automatic import + */ + readonly autoImportPolicy?: S3AutoImportPolicy; + + /** + * Defines which file system events automatically export changed file metadata to S3. + * + * @default - no automatic export + */ + readonly autoExportPolicy?: S3AutoExportPolicy; +} + +/** + * Properties for a DataRepositoryAssociation. + */ +export interface DataRepositoryAssociationProps { + /** + * The Lustre file system to associate with the S3 data repository. + */ + readonly fileSystem: IFileSystem; + + /** + * The path on the Lustre file system to associate with the data repository. + * Must begin with `/` and be unique within the file system. + * + * Example: `/data` + */ + readonly fileSystemPath: string; + + /** + * The S3 bucket to use as the data repository. + */ + readonly bucket: IBucket; + + /** + * The prefix within the S3 bucket to associate with the file system path. + * + * @default - the root of the bucket (`s3:///`) + */ + readonly bucketPrefix?: string; + + /** + * S3 auto-import and auto-export policies for this association. + * + * @default - no automatic import or export + */ + readonly s3?: S3DataRepositoryConfiguration; + + /** + * For files imported from S3, the stripe count and maximum amount of data per + * file (in MiB) stored on a single physical disk. + * + * Allowed values: 1 to 512,000 MiB. + * + * @default 1024 + */ + readonly importedFileChunkSizeMiB?: number; + + /** + * Whether to run a data repository task to import S3 metadata after the + * association is created. + * + * @default false + */ + readonly batchImportMetaDataOnCreate?: boolean; + + /** + * The removal policy for this resource. + * + * @default RemovalPolicy.RETAIN + */ + readonly removalPolicy?: RemovalPolicy; +} + +/** + * Interface for a DataRepositoryAssociation. + */ +export interface IDataRepositoryAssociation { + /** + * The ID of the data repository association. + */ + readonly associationId: string; +} + +/** + * An L2 construct for an FSx for Lustre DataRepositoryAssociation. + * + * Links an S3 bucket to a path on a Lustre file system so that data can be + * automatically imported and exported between S3 and the file system. + * + * Data repository associations are supported on FSx for Lustre 2.12 and 2.15 + * file systems with SCRATCH_2, PERSISTENT_1, or PERSISTENT_2 deployment types. + * + * @see https://docs.aws.amazon.com/fsx/latest/LustreGuide/overview-dra-data-repo.html + * + * @resource AWS::FSx::DataRepositoryAssociation + */ +@propertyInjectable +export class DataRepositoryAssociation extends Resource implements IDataRepositoryAssociation { + /** + * Uniquely identifies this class. + */ + public static readonly PROPERTY_INJECTION_ID: string = 'aws-cdk-lib.aws-fsx.DataRepositoryAssociation'; + + /** + * The ID of the data repository association. + * @attribute + */ + public readonly associationId: string; + + constructor(scope: Construct, id: string, props: DataRepositoryAssociationProps) { + super(scope, id); + addConstructMetadata(this, props); + + this.validateProps(props); + + const dataRepositoryPath = props.bucketPrefix + ? `s3://${props.bucket.bucketName}/${props.bucketPrefix.replace(/^\/+/, '')}` + : `s3://${props.bucket.bucketName}/`; + + const resource = new CfnDataRepositoryAssociation(this, 'Resource', { + fileSystemId: props.fileSystem.fileSystemId, + fileSystemPath: props.fileSystemPath, + dataRepositoryPath, + importedFileChunkSize: props.importedFileChunkSizeMiB, + batchImportMetaDataOnCreate: props.batchImportMetaDataOnCreate, + s3: props.s3, + }); + resource.applyRemovalPolicy(props.removalPolicy ?? RemovalPolicy.RETAIN); + + this.associationId = resource.ref; + + // Grant FSx service principal read/write access to the bucket so it can + // fulfil import and export requests on behalf of the file system. + // Add an explicit dependency so CloudFormation creates the bucket policy before the DRA. + const grant = props.bucket.grantReadWrite(new ServicePrincipal('fsx.amazonaws.com')); + resource.node.addDependency(grant); + } + + private validateProps(props: DataRepositoryAssociationProps): void { + this.validateFileSystemPath(props.fileSystemPath); + this.validateImportedFileChunkSize(props.importedFileChunkSizeMiB); + if (props.s3?.autoImportPolicy) { + this.validateEventTypes(props.s3.autoImportPolicy.events, 'autoImportPolicy'); + } + if (props.s3?.autoExportPolicy) { + this.validateEventTypes(props.s3.autoExportPolicy.events, 'autoExportPolicy'); + } + } + + private validateFileSystemPath(fileSystemPath: string): void { + if (Token.isUnresolved(fileSystemPath)) return; + if (!fileSystemPath.startsWith('/')) { + throw new ValidationError(lit`FileSystemPathMustStartWithSlash`, `fileSystemPath must begin with "/", got: "${fileSystemPath}"`, this); + } + if (fileSystemPath.length > 4096) { + throw new ValidationError(lit`FileSystemPathExceedsMaxLength`, `fileSystemPath cannot exceed 4096 characters, got length: ${fileSystemPath.length}`, this); + } + } + + private validateImportedFileChunkSize(importedFileChunkSizeMiB?: number): void { + if (importedFileChunkSizeMiB === undefined) return; + if (importedFileChunkSizeMiB < 1 || importedFileChunkSizeMiB > 512000) { + throw new ValidationError(lit`ImportedFileChunkSizeInvalid`, `importedFileChunkSizeMiB must be between 1 and 512,000 MiB, got: ${importedFileChunkSizeMiB}`, this); + } + } + + private validateEventTypes(events: DataRepositoryEventType[], field: string): void { + if (events.length === 0) { + throw new ValidationError(lit`EventTypesMustNotBeEmpty`, `${field}.events must contain at least one event type`, this); + } + } +} diff --git a/packages/aws-cdk-lib/aws-fsx/lib/index.ts b/packages/aws-cdk-lib/aws-fsx/lib/index.ts index 5e3e2611f445a..7065334f95401 100644 --- a/packages/aws-cdk-lib/aws-fsx/lib/index.ts +++ b/packages/aws-cdk-lib/aws-fsx/lib/index.ts @@ -1,4 +1,5 @@ export * from './daily-automatic-backup-start-time'; +export * from './data-repository-association'; export * from './file-system'; export * from './fsx.generated'; export * from './lustre-file-system'; diff --git a/packages/aws-cdk-lib/aws-fsx/test/data-repository-association.test.ts b/packages/aws-cdk-lib/aws-fsx/test/data-repository-association.test.ts new file mode 100644 index 0000000000000..1976c8728c10a --- /dev/null +++ b/packages/aws-cdk-lib/aws-fsx/test/data-repository-association.test.ts @@ -0,0 +1,217 @@ +import { Template } from '../../assertions'; +import { Bucket } from '../../aws-s3'; +import { Stack } from '../../core'; +import { Subnet, Vpc } from '../../aws-ec2'; +import { + DataRepositoryAssociation, + DataRepositoryEventType, + FileSystemTypeVersion, + LustreDeploymentType, + LustreFileSystem, +} from '../lib'; + +describe('DataRepositoryAssociation', () => { + let stack: Stack; + let vpc: Vpc; + let fileSystem: LustreFileSystem; + let bucket: Bucket; + + beforeEach(() => { + stack = new Stack(); + vpc = new Vpc(stack, 'VPC'); + const vpcSubnet = new Subnet(stack, 'Subnet', { + availabilityZone: 'us-east-1a', + cidrBlock: vpc.vpcCidrBlock, + vpcId: vpc.vpcId, + }); + fileSystem = new LustreFileSystem(stack, 'FileSystem', { + lustreConfiguration: { + deploymentType: LustreDeploymentType.SCRATCH_2, + }, + storageCapacityGiB: 1200, + vpc, + vpcSubnet, + fileSystemTypeVersion: FileSystemTypeVersion.V_2_15, + }); + bucket = new Bucket(stack, 'Bucket'); + }); + + test('creates a DataRepositoryAssociation with required props', () => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::FSx::DataRepositoryAssociation', { + FileSystemId: { Ref: 'FileSystem8A8E25C0' }, + FileSystemPath: '/data', + }); + }); + + test('sets dataRepositoryPath to bucket root when no prefix is provided', () => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::FSx::DataRepositoryAssociation', { + DataRepositoryPath: { + 'Fn::Join': ['', ['s3://', { Ref: 'Bucket83908E77' }, '/']], + }, + }); + }); + + test('sets dataRepositoryPath with bucketPrefix', () => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + bucketPrefix: 'primary', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::FSx::DataRepositoryAssociation', { + DataRepositoryPath: { + 'Fn::Join': ['', ['s3://', { Ref: 'Bucket83908E77' }, '/primary']], + }, + }); + }); + + test('strips all leading slashes from bucketPrefix', () => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + bucketPrefix: '//primary/', + }); + + Template.fromStack(stack).hasResourceProperties('AWS::FSx::DataRepositoryAssociation', { + DataRepositoryPath: { + 'Fn::Join': ['', ['s3://', { Ref: 'Bucket83908E77' }, '/primary/']], + }, + }); + }); + + test('sets importedFileChunkSizeMiB and batchImportMetaDataOnCreate', () => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + importedFileChunkSizeMiB: 2048, + batchImportMetaDataOnCreate: true, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::FSx::DataRepositoryAssociation', { + ImportedFileChunkSize: 2048, + BatchImportMetaDataOnCreate: true, + }); + }); + + test('sets auto import and export policies', () => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + s3: { + autoImportPolicy: { events: [DataRepositoryEventType.NEW, DataRepositoryEventType.CHANGED] }, + autoExportPolicy: { events: [DataRepositoryEventType.NEW, DataRepositoryEventType.CHANGED, DataRepositoryEventType.DELETED] }, + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::FSx::DataRepositoryAssociation', { + S3: { + AutoImportPolicy: { Events: ['NEW', 'CHANGED'] }, + AutoExportPolicy: { Events: ['NEW', 'CHANGED', 'DELETED'] }, + }, + }); + }); + + test('grants fsx.amazonaws.com read-write access to the bucket via bucket policy', () => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::S3::BucketPolicy', { + PolicyDocument: { + Statement: [ + { + Principal: { Service: 'fsx.amazonaws.com' }, + }, + ], + }, + }); + }); + + describe('validation', () => { + test('throws when fileSystemPath does not start with /', () => { + expect(() => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: 'data', + bucket, + }); + }).toThrow(/fileSystemPath must begin with "\/"/); + }); + + test('throws when fileSystemPath exceeds 4096 characters', () => { + expect(() => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/' + 'a'.repeat(4096), + bucket, + }); + }).toThrow(/fileSystemPath cannot exceed 4096 characters/); + }); + + test('throws when importedFileChunkSizeMiB is below 1', () => { + expect(() => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + importedFileChunkSizeMiB: 0, + }); + }).toThrow(/importedFileChunkSizeMiB must be between 1 and 512,000/); + }); + + test('throws when importedFileChunkSizeMiB exceeds 512000', () => { + expect(() => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + importedFileChunkSizeMiB: 512001, + }); + }).toThrow(/importedFileChunkSizeMiB must be between 1 and 512,000/); + }); + + test('throws when autoImportPolicy.events is empty', () => { + expect(() => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + s3: { + autoImportPolicy: { events: [] }, + }, + }); + }).toThrow(/autoImportPolicy.events must contain at least one event type/); + }); + + test('throws when autoExportPolicy.events is empty', () => { + expect(() => { + new DataRepositoryAssociation(stack, 'DRA', { + fileSystem, + fileSystemPath: '/data', + bucket, + s3: { + autoExportPolicy: { events: [] }, + }, + }); + }).toThrow(/autoExportPolicy.events must contain at least one event type/); + }); + }); +});