diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/job.ts index 4102e53068b70..ea8d87ed11401 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/job.ts @@ -8,6 +8,8 @@ import { Code } from '../code'; import { MetricType, JobState, WorkerType, GlueVersion } from '../constants'; import { IConnection } from '../connection'; import { ISecurityConfiguration } from '../security-configuration'; +import { CfnJob, CfnJobProps } from 'aws-cdk-lib/aws-glue'; +import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; /** * Interface representing a new or an imported Glue Job @@ -25,6 +27,11 @@ export interface IJob extends cdk.IResource, iam.IGrantable { */ readonly jobArn: string; + /** + * The IAM role associated with this job. + */ + readonly role?: iam.IRole; + /** * Defines a CloudWatch event rule triggered when something happens with this job. * @@ -131,8 +138,20 @@ export interface ContinuousLoggingProps { * event-driven flow using the job. */ export abstract class JobBase extends cdk.Resource implements IJob { + /** + * Returns the job arn + */ + protected static buildJobArn(scope: constructs.Construct, jobName: string) : string { + return cdk.Stack.of(scope).formatArn({ + service: 'glue', + resource: 'job', + resourceName: jobName, + }); + } + public abstract readonly jobArn: string; public abstract readonly jobName: string; + public abstract readonly role?: iam.IRole; public abstract readonly grantPrincipal: iam.IPrincipal; /** @@ -267,17 +286,6 @@ export abstract class JobBase extends cdk.Resource implements IJob { private metricJobStateRule(id: string, jobState: JobState): events.Rule { return this.node.tryFindChild(id) as events.Rule ?? this.onStateChange(id, jobState); } - - /** - * Returns the job arn - */ - protected buildJobArn(scope: constructs.Construct, jobName: string) : string { - return cdk.Stack.of(scope).formatArn({ - service: 'glue', - resource: 'job', - resourceName: jobName, - }); - } } /** @@ -448,6 +456,26 @@ export interface JobProps { * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html **/ readonly continuousLogging?: ContinuousLoggingProps; + + /** + * Specifies configuration properties of a notification (optional). + * After a job run starts, the number of minutes to wait before sending a job run delay notification. + * + * @default - undefined + */ + readonly notifyDelayAfter?: cdk.Duration; + + /** + * Specifies whether job run queuing is enabled for the job runs for this job. + * A value of true means job run queuing is enabled for the job runs. + * If false or not populated, the job runs will not be considered for queueing. + * If this field does not match the value set in the job run, then the value from + * the job run field will be used. This property must be set to false for flex jobs. + * If this property is enabled, maxRetries must be set to zero. + * + * @default - no job run queuing + */ + readonly jobRunQueuingEnabled?: boolean; } /** @@ -466,7 +494,8 @@ export abstract class Job extends JobBase { public static fromJobAttributes(scope: constructs.Construct, id: string, attrs: JobAttributes): IJob { class Import extends JobBase { public readonly jobName = attrs.jobName; - public readonly jobArn = this.buildJobArn(scope, attrs.jobName); + public readonly jobArn = Job.buildJobArn(scope, attrs.jobName); + public readonly role = attrs.role; public readonly grantPrincipal = attrs.role ?? new iam.UnknownPrincipal({ resource: this }); } @@ -474,9 +503,43 @@ export abstract class Job extends JobBase { } /** - * The IAM role Glue assumes to run this job. - */ - public readonly abstract role: iam.IRole; + * Utility method to help with creating the CfnJob resource. + * It handles common/shared JobProps, while allowing CfnJobProps overrides. + * + * @param scope the scope to create the resource in. + * @param props the JobProps to use for the resource. + * @param cfnProps the CfnJobProps overrides to use for the resource. + * @protected + */ + protected static setupJobResource(scope: constructs.Construct, props: JobProps, cfnProps: CfnJobProps) { + return new CfnJob(scope, 'Resource', { + name: props.jobName, + description: props.description, + executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, + timeout: props.timeout?.toMinutes(), + connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, + securityConfiguration: props.securityConfiguration?.securityConfigurationName, + maxRetries: props.jobRunQueuingEnabled ? 0 : props.maxRetries, + jobRunQueuingEnabled: props.jobRunQueuingEnabled ? props.jobRunQueuingEnabled : false, + notificationProperty: props.notifyDelayAfter ? { notifyDelayAfter: props.notifyDelayAfter.toMinutes() } : undefined, + tags: props.tags, + ...cfnProps, + }); + } + + public readonly role?: iam.IRole; + public readonly grantPrincipal: iam.IPrincipal; + + protected constructor(scope: constructs.Construct, id: string, props: JobProps) { + super(scope, id, { + physicalName: props.jobName, + }); + // Enhanced CDK Analytics Telemetry + addConstructMetadata(this, props); + + this.role = props.role; + this.grantPrincipal = props.role; + } /** * Check no usage of reserved arguments. @@ -497,11 +560,10 @@ export abstract class Job extends JobBase { /** * Setup Continuous Logging Properties - * @param role The IAM role to use for continuous logging * @param props The properties for continuous logging configuration * @returns String containing the args for the continuous logging command */ - protected setupContinuousLogging(role: iam.IRole, props: ContinuousLoggingProps | undefined) : any { + protected setupContinuousLogging(props: ContinuousLoggingProps | undefined) : any { // If the developer has explicitly disabled continuous logging return no args if (props && !props.enabled) { return {}; @@ -519,7 +581,7 @@ export abstract class Job extends JobBase { // If the developer provided a log group, add its name to the args and update the role. if (props?.logGroup) { args['--continuous-log-logGroup'] = props.logGroup.logGroupName; - props.logGroup.grantWrite(role); + props.logGroup.grantWrite(this); } if (props?.logStreamPrefix) { @@ -534,7 +596,7 @@ export abstract class Job extends JobBase { } protected codeS3ObjectUrl(code: Code) { - const s3Location = code.bind(this, this.role).s3Location; + const s3Location = code.bind(this, this).s3Location; return `s3://${s3Location.bucketName}/${s3Location.objectKey}`; } } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-etl-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-etl-job.ts index 970599d2872c3..fd6e759ec62c4 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-etl-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-etl-job.ts @@ -1,9 +1,8 @@ -import { CfnJob } from 'aws-cdk-lib/aws-glue'; import { Construct } from 'constructs'; import { JobType, GlueVersion, JobLanguage, PythonVersion, WorkerType } from '../constants'; import { Code } from '../code'; import { SparkJob, SparkJobProps } from './spark-job'; -import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; +import { Job } from './job'; /** * Properties for creating a Python Spark ETL job @@ -16,43 +15,6 @@ export interface PySparkEtlJobProps extends SparkJobProps { * @default - no extra files */ readonly extraPythonFiles?: Code[]; - - /** - * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. - * - * @default - no extra files specified. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraFiles?: Code[]; - - /** - * Extra Jars S3 URL (optional) - * S3 URL where additional jar dependencies are located - * @default - no extra jar files - */ - readonly extraJars?: Code[]; - - /** - * Setting this value to true prioritizes the customer's extra JAR files in the classpath. - * - * @default false - priority is not given to user-provided jars - * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraJarsFirst?: boolean; - - /** - * Specifies whether job run queuing is enabled for the job runs for this job. - * A value of true means job run queuing is enabled for the job runs. - * If false or not populated, the job runs will not be considered for queueing. - * If this field does not match the value set in the job run, then the value from - * the job run field will be used. This property must be set to false for flex jobs. - * If this property is enabled, maxRetries must be set to zero. - * - * @default false - */ - readonly jobRunQueuingEnabled?: boolean; } /** @@ -76,8 +38,6 @@ export class PySparkEtlJob extends SparkJob { */ constructor(scope: Construct, id: string, props: PySparkEtlJobProps) { super(scope, id, props); - // Enhanced CDK Analytics Telemetry - addConstructMetadata(this, props); // Combine command line arguments into a single line item const defaultArguments = { @@ -85,10 +45,8 @@ export class PySparkEtlJob extends SparkJob { ...this.nonExecutableCommonArguments(props), }; - const jobResource = new CfnJob(this, 'Resource', { - name: props.jobName, - description: props.description, - role: this.role.roleArn, + const jobResource = Job.setupJobResource(this, props, { + role: this.role!.roleArn, command: { name: JobType.ETL, scriptLocation: this.codeS3ObjectUrl(props.script), @@ -97,18 +55,11 @@ export class PySparkEtlJob extends SparkJob { glueVersion: props.glueVersion ?? GlueVersion.V4_0, workerType: props.workerType ?? WorkerType.G_1X, numberOfWorkers: props.numberOfWorkers ? props.numberOfWorkers : 10, - maxRetries: props.jobRunQueuingEnabled ? 0 : props.maxRetries, - jobRunQueuingEnabled: props.jobRunQueuingEnabled ? props.jobRunQueuingEnabled : false, - executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, - timeout: props.timeout?.toMinutes(), - connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, - securityConfiguration: props.securityConfiguration?.securityConfigurationName, - tags: props.tags, defaultArguments, }); const resourceName = this.getResourceNameAttribute(jobResource.ref); - this.jobArn = this.buildJobArn(this, resourceName); + this.jobArn = Job.buildJobArn(this, resourceName); this.jobName = resourceName; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-flex-etl-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-flex-etl-job.ts index bd4d5d0ec753e..e49be910eb612 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-flex-etl-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-flex-etl-job.ts @@ -1,22 +1,14 @@ -import { CfnJob } from 'aws-cdk-lib/aws-glue'; import { Construct } from 'constructs'; import { JobType, GlueVersion, JobLanguage, PythonVersion, WorkerType, ExecutionClass } from '../constants'; -import * as cdk from 'aws-cdk-lib/core'; import { Code } from '../code'; import { SparkJob, SparkJobProps } from './spark-job'; import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; +import { Job } from './job'; /** * Properties for PySparkFlexEtlJob */ export interface PySparkFlexEtlJobProps extends SparkJobProps { - /** - * Specifies configuration properties of a notification (optional). - * After a job run starts, the number of minutes to wait before sending a job run delay notification. - * @default - undefined - */ - readonly notifyDelayAfter?: cdk.Duration; - /** * Extra Python Files S3 URL (optional) * S3 URL where additional python dependencies are located @@ -24,31 +16,6 @@ export interface PySparkFlexEtlJobProps extends SparkJobProps { * @default - no extra files */ readonly extraPythonFiles?: Code[]; - - /** - * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. - * - * @default - no extra files specified. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraFiles?: Code[]; - - /** - * Extra Jars S3 URL (optional) - * S3 URL where additional jar dependencies are located - * @default - no extra jar files - */ - readonly extraJars?: Code[]; - - /** - * Setting this value to true prioritizes the customer's extra JAR files in the classpath. - * - * @default false - priority is not given to user-provided jars - * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraJarsFirst?: boolean; } /** @@ -81,10 +48,8 @@ export class PySparkFlexEtlJob extends SparkJob { ...this.nonExecutableCommonArguments(props), }; - const jobResource = new CfnJob(this, 'Resource', { - name: props.jobName, - description: props.description, - role: this.role.roleArn, + const jobResource = Job.setupJobResource(this, props, { + role: this.role!.roleArn, command: { name: JobType.ETL, scriptLocation: this.codeS3ObjectUrl(props.script), @@ -94,20 +59,13 @@ export class PySparkFlexEtlJob extends SparkJob { workerType: props.workerType ? props.workerType : WorkerType.G_1X, numberOfWorkers: props.numberOfWorkers ? props.numberOfWorkers : 10, maxRetries: props.maxRetries, - executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, - notificationProperty: props.notifyDelayAfter ? { notifyDelayAfter: props.notifyDelayAfter.toMinutes() } : undefined, - timeout: props.timeout?.toMinutes(), - connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, - securityConfiguration: props.securityConfiguration?.securityConfigurationName, - tags: props.tags, executionClass: ExecutionClass.FLEX, jobRunQueuingEnabled: false, defaultArguments, - }); const resourceName = this.getResourceNameAttribute(jobResource.ref); - this.jobArn = this.buildJobArn(this, resourceName); + this.jobArn = Job.buildJobArn(this, resourceName); this.jobName = resourceName; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-streaming-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-streaming-job.ts index e4d4184ee061e..a5d46986a7aa4 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-streaming-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-streaming-job.ts @@ -1,9 +1,8 @@ -import { CfnJob } from 'aws-cdk-lib/aws-glue'; import { Construct } from 'constructs'; import { JobType, GlueVersion, JobLanguage, PythonVersion, WorkerType } from '../constants'; import { Code } from '../code'; import { SparkJob, SparkJobProps } from './spark-job'; -import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; +import { Job } from './job'; /** * Properties for creating a Python Spark ETL job @@ -16,43 +15,6 @@ export interface PySparkStreamingJobProps extends SparkJobProps { * @default - no extra files */ readonly extraPythonFiles?: Code[]; - - /** - * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. - * - * @default - no extra files specified. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraFiles?: Code[]; - - /** - * Extra Jars S3 URL (optional) - * S3 URL where additional jar dependencies are located - * @default - no extra jar files - */ - readonly extraJars?: Code[]; - - /** - * Setting this value to true prioritizes the customer's extra JAR files in the classpath. - * - * @default false - priority is not given to user-provided jars - * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraJarsFirst?: boolean; - - /** - * Specifies whether job run queuing is enabled for the job runs for this job. - * A value of true means job run queuing is enabled for the job runs. - * If false or not populated, the job runs will not be considered for queueing. - * If this field does not match the value set in the job run, then the value from - * the job run field will be used. This property must be set to false for flex jobs. - * If this property is enabled, maxRetries must be set to zero. - * - * @default - no job run queuing - */ - readonly jobRunQueuingEnabled?: boolean; } /** @@ -76,8 +38,6 @@ export class PySparkStreamingJob extends SparkJob { */ constructor(scope: Construct, id: string, props: PySparkStreamingJobProps) { super(scope, id, props); - // Enhanced CDK Analytics Telemetry - addConstructMetadata(this, props); // Combine command line arguments into a single line item const defaultArguments = { @@ -85,10 +45,8 @@ export class PySparkStreamingJob extends SparkJob { ...this.nonExecutableCommonArguments(props), }; - const jobResource = new CfnJob(this, 'Resource', { - name: props.jobName, - description: props.description, - role: this.role.roleArn, + const jobResource = Job.setupJobResource(this, props, { + role: this.role!.roleArn, command: { name: JobType.STREAMING, scriptLocation: this.codeS3ObjectUrl(props.script), @@ -97,18 +55,11 @@ export class PySparkStreamingJob extends SparkJob { glueVersion: props.glueVersion ? props.glueVersion : GlueVersion.V4_0, workerType: props.workerType ? props.workerType : WorkerType.G_1X, numberOfWorkers: props.numberOfWorkers ? props.numberOfWorkers : 10, - maxRetries: props.jobRunQueuingEnabled ? 0 : props.maxRetries, - jobRunQueuingEnabled: props.jobRunQueuingEnabled ? props.jobRunQueuingEnabled : false, - executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, - timeout: props.timeout?.toMinutes(), - connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, - securityConfiguration: props.securityConfiguration?.securityConfigurationName, - tags: props.tags, defaultArguments, }); const resourceName = this.getResourceNameAttribute(jobResource.ref); - this.jobArn = this.buildJobArn(this, resourceName); + this.jobArn = Job.buildJobArn(this, resourceName); this.jobName = resourceName; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/python-shell-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/python-shell-job.ts index 4b2e96376e9d9..6f1c1d88e4de3 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/python-shell-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/python-shell-job.ts @@ -1,9 +1,6 @@ -import { CfnJob } from 'aws-cdk-lib/aws-glue'; -import * as iam from 'aws-cdk-lib/aws-iam'; import { Job, JobProps } from './job'; import { Construct } from 'constructs'; import { JobType, GlueVersion, PythonVersion, MaxCapacity, JobLanguage } from '../constants'; -import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; /** * Properties for creating a Python Shell job @@ -22,18 +19,6 @@ export interface PythonShellJobProps extends JobProps { * @default 0.0625 */ readonly maxCapacity?: MaxCapacity; - - /** - * Specifies whether job run queuing is enabled for the job runs for this job. - * A value of true means job run queuing is enabled for the job runs. - * If false or not populated, the job runs will not be considered for queueing. - * If this field does not match the value set in the job run, then the value from - * the job run field will be used. This property must be set to false for flex jobs. - * If this property is enabled, maxRetries must be set to zero. - * - * @default false - */ - readonly jobRunQueuingEnabled?: boolean; } /** @@ -46,23 +31,15 @@ export interface PythonShellJobProps extends JobProps { export class PythonShellJob extends Job { public readonly jobArn: string; public readonly jobName: string; - public readonly role: iam.IRole; - public readonly grantPrincipal: iam.IPrincipal; /** * PythonShellJob constructor */ constructor(scope: Construct, id: string, props: PythonShellJobProps) { - super(scope, id, { physicalName: props.jobName }); - // Enhanced CDK Analytics Telemetry - addConstructMetadata(this, props); - - // Set up role and permissions for principal - this.role = props.role; - this.grantPrincipal = this.role; + super(scope, id, props); // Enable CloudWatch metrics and continuous logging by default as a best practice - const continuousLoggingArgs = this.setupContinuousLogging(this.role, props.continuousLogging); + const continuousLoggingArgs = this.setupContinuousLogging(props.continuousLogging); const profilingMetricsArgs = { '--enable-metrics': '' }; const observabilityMetricsArgs = { '--enable-observability-metrics': 'true' }; @@ -78,10 +55,8 @@ export class PythonShellJob extends Job { ...this.checkNoReservedArgs(props.defaultArguments), }; - const jobResource = new CfnJob(this, 'Resource', { - name: props.jobName, - description: props.description, - role: this.role.roleArn, + const jobResource = Job.setupJobResource(this, props, { + role: this.role!.roleArn, command: { name: JobType.PYTHON_SHELL, scriptLocation: this.codeS3ObjectUrl(props.script), @@ -90,17 +65,11 @@ export class PythonShellJob extends Job { glueVersion: props.glueVersion ? props.glueVersion : GlueVersion.V3_0, maxCapacity: props.maxCapacity ? props.maxCapacity : MaxCapacity.DPU_1_16TH, maxRetries: props.jobRunQueuingEnabled ? 0 : props.maxRetries ? props.maxRetries : 0, - jobRunQueuingEnabled: props.jobRunQueuingEnabled ? props.jobRunQueuingEnabled : false, - executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, - timeout: props.timeout?.toMinutes(), - connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, - securityConfiguration: props.securityConfiguration?.securityConfigurationName, - tags: props.tags, defaultArguments, }); const resourceName = this.getResourceNameAttribute(jobResource.ref); - this.jobArn = this.buildJobArn(this, resourceName); + this.jobArn = Job.buildJobArn(this, resourceName); this.jobName = resourceName; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/ray-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/ray-job.ts index baf8ce877877f..220d79869fa3f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/ray-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/ray-job.ts @@ -1,9 +1,6 @@ -import { CfnJob } from 'aws-cdk-lib/aws-glue'; -import * as iam from 'aws-cdk-lib/aws-iam'; import { Job, JobProps } from './job'; import { Construct } from 'constructs'; import { JobType, GlueVersion, WorkerType, Runtime } from '../constants'; -import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; /** * Properties for creating a Ray Glue job @@ -15,18 +12,6 @@ export interface RayJobProps extends JobProps { * @default - Runtime version will default to Ray2.4 */ readonly runtime?: Runtime; - - /** - * Specifies whether job run queuing is enabled for the job runs for this job. - * A value of true means job run queuing is enabled for the job runs. - * If false or not populated, the job runs will not be considered for queueing. - * If this field does not match the value set in the job run, then the value from - * the job run field will be used. This property must be set to false for flex jobs. - * If this property is enabled, maxRetries must be set to zero. - * - * @default - no job run queuing - */ - readonly jobRunQueuingEnabled?: boolean; } /** @@ -40,27 +25,15 @@ export interface RayJobProps extends JobProps { export class RayJob extends Job { public readonly jobArn: string; public readonly jobName: string; - public readonly role: iam.IRole; - public readonly grantPrincipal: iam.IPrincipal; /** * RayJob constructor */ constructor(scope: Construct, id: string, props: RayJobProps) { - super(scope, id, { - physicalName: props.jobName, - }); - // Enhanced CDK Analytics Telemetry - addConstructMetadata(this, props); - - this.jobName = props.jobName ?? ''; - - // Set up role and permissions for principal - this.role = props.role; - this.grantPrincipal = this.role; + super(scope, id, props); // Enable CloudWatch metrics and continuous logging by default as a best practice - const continuousLoggingArgs = this.setupContinuousLogging(this.role, props.continuousLogging); + const continuousLoggingArgs = this.setupContinuousLogging(props.continuousLogging); const profilingMetricsArgs = { '--enable-metrics': '' }; const observabilityMetricsArgs = { '--enable-observability-metrics': 'true' }; @@ -76,10 +49,8 @@ export class RayJob extends Job { throw new Error('Ray jobs only support Z.2X worker type'); } - const jobResource = new CfnJob(this, 'Resource', { - name: props.jobName, - description: props.description, - role: this.role.roleArn, + const jobResource = Job.setupJobResource(this, props, { + role: this.role!.roleArn, command: { name: JobType.RAY, scriptLocation: this.codeS3ObjectUrl(props.script), @@ -88,18 +59,11 @@ export class RayJob extends Job { glueVersion: GlueVersion.V4_0, workerType: props.workerType ? props.workerType : WorkerType.Z_2X, numberOfWorkers: props.numberOfWorkers ? props.numberOfWorkers: 3, - maxRetries: props.jobRunQueuingEnabled ? 0 : props.maxRetries, - jobRunQueuingEnabled: props.jobRunQueuingEnabled ? props.jobRunQueuingEnabled : false, - executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, - timeout: props.timeout?.toMinutes(), - connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, - securityConfiguration: props.securityConfiguration?.securityConfigurationName, - tags: props.tags, defaultArguments, }); const resourceName = this.getResourceNameAttribute(jobResource.ref); - this.jobArn = this.buildJobArn(this, resourceName); + this.jobArn = Job.buildJobArn(this, resourceName); this.jobName = resourceName; } } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-etl-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-etl-job.ts index d227ad33f717c..ebfa98e1ff8c3 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-etl-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-etl-job.ts @@ -1,9 +1,7 @@ -import { CfnJob } from 'aws-cdk-lib/aws-glue'; import { Construct } from 'constructs'; import { JobType, GlueVersion, JobLanguage, WorkerType } from '../constants'; -import { Code } from '../code'; import { SparkJob, SparkJobProps } from './spark-job'; -import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; +import { Job } from './job'; /** * Properties for creating a Scala Spark ETL job @@ -15,43 +13,6 @@ export interface ScalaSparkEtlJobProps extends SparkJobProps { * Java scripts **/ readonly className: string; - - /** - * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. - * - * @default - no extra files specified. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraFiles?: Code[]; - - /** - * Extra Jars S3 URL (optional) - * S3 URL where additional jar dependencies are located - * @default - no extra jar files - */ - readonly extraJars?: Code[]; - - /** - * Setting this value to true prioritizes the customer's extra JAR files in the classpath. - * - * @default false - priority is not given to user-provided jars - * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraJarsFirst?: boolean; - - /** - * Specifies whether job run queuing is enabled for the job runs for this job. - * A value of true means job run queuing is enabled for the job runs. - * If false or not populated, the job runs will not be considered for queueing. - * If this field does not match the value set in the job run, then the value from - * the job run field will be used. This property must be set to false for flex jobs. - * If this property is enabled, maxRetries must be set to zero. - * - * @default - no job run queuing - */ - readonly jobRunQueuingEnabled?: boolean; } /** @@ -75,8 +36,6 @@ export class ScalaSparkEtlJob extends SparkJob { */ constructor(scope: Construct, id: string, props: ScalaSparkEtlJobProps) { super(scope, id, props); - // Enhanced CDK Analytics Telemetry - addConstructMetadata(this, props); // Combine command line arguments into a single line item const defaultArguments = { @@ -88,10 +47,8 @@ export class ScalaSparkEtlJob extends SparkJob { throw new Error('Both workerType and numberOfWorkers must be set'); } - const jobResource = new CfnJob(this, 'Resource', { - name: props.jobName, - description: props.description, - role: this.role.roleArn, + const jobResource = Job.setupJobResource(this, props, { + role: this.role!.roleArn, command: { name: JobType.ETL, scriptLocation: this.codeS3ObjectUrl(props.script), @@ -99,18 +56,11 @@ export class ScalaSparkEtlJob extends SparkJob { glueVersion: props.glueVersion ? props.glueVersion : GlueVersion.V4_0, workerType: props.workerType ? props.workerType : WorkerType.G_1X, numberOfWorkers: props.numberOfWorkers ? props.numberOfWorkers : 10, - maxRetries: props.jobRunQueuingEnabled ? 0 : props.maxRetries, - jobRunQueuingEnabled: props.jobRunQueuingEnabled ? props.jobRunQueuingEnabled : false, - executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, - timeout: props.timeout?.toMinutes(), - connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, - securityConfiguration: props.securityConfiguration?.securityConfigurationName, - tags: props.tags, defaultArguments, }); const resourceName = this.getResourceNameAttribute(jobResource.ref); - this.jobArn = this.buildJobArn(this, resourceName); + this.jobArn = Job.buildJobArn(this, resourceName); this.jobName = resourceName; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-flex-etl-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-flex-etl-job.ts index 6cfe243a695e6..be73fc78ccbd7 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-flex-etl-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-flex-etl-job.ts @@ -1,10 +1,7 @@ -import { CfnJob } from 'aws-cdk-lib/aws-glue'; import { Construct } from 'constructs'; import { JobType, GlueVersion, JobLanguage, WorkerType, ExecutionClass } from '../constants'; -import * as cdk from 'aws-cdk-lib/core'; -import { Code } from '../code'; import { SparkJob, SparkJobProps } from './spark-job'; -import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; +import { Job } from './job'; /** * Flex Jobs class @@ -20,44 +17,12 @@ import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; * */ export interface ScalaSparkFlexEtlJobProps extends SparkJobProps { - /** - * Specifies configuration properties of a notification (optional). - * After a job run starts, the number of minutes to wait before sending a job run delay notification. - * @default - undefined - */ - readonly notifyDelayAfter?: cdk.Duration; - /** * The fully qualified Scala class name that serves as the entry point for the job. * * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className: string; - - /** - * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. - * - * @default - no extra files specified. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraFiles?: Code[]; - - /** - * Extra Jars S3 URL (optional) - * S3 URL where additional jar dependencies are located - * @default - no extra jar files - */ - readonly extraJars?: Code[]; - - /** - * Setting this value to true prioritizes the customer's extra JAR files in the classpath. - * - * @default false - priority is not given to user-provided jars - * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraJarsFirst?: boolean; } /** @@ -81,8 +46,6 @@ export class ScalaSparkFlexEtlJob extends SparkJob { */ constructor(scope: Construct, id: string, props: ScalaSparkFlexEtlJobProps) { super(scope, id, props); - // Enhanced CDK Analytics Telemetry - addConstructMetadata(this, props); // Combine command line arguments into a single line item const defaultArguments = { @@ -90,10 +53,8 @@ export class ScalaSparkFlexEtlJob extends SparkJob { ...this.nonExecutableCommonArguments(props), }; - const jobResource = new CfnJob(this, 'Resource', { - name: props.jobName, - description: props.description, - role: this.role.roleArn, + const jobResource = Job.setupJobResource(this, props, { + role: this.role!.roleArn, command: { name: JobType.ETL, scriptLocation: this.codeS3ObjectUrl(props.script), @@ -102,19 +63,13 @@ export class ScalaSparkFlexEtlJob extends SparkJob { workerType: props.workerType ? props.workerType : WorkerType.G_1X, numberOfWorkers: props.numberOfWorkers ? props.numberOfWorkers : 10, maxRetries: props.maxRetries, - executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, - notificationProperty: props.notifyDelayAfter ? { notifyDelayAfter: props.notifyDelayAfter.toMinutes() } : undefined, - timeout: props.timeout?.toMinutes(), - connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, - securityConfiguration: props.securityConfiguration?.securityConfigurationName, - tags: props.tags, executionClass: ExecutionClass.FLEX, jobRunQueuingEnabled: false, defaultArguments, }); const resourceName = this.getResourceNameAttribute(jobResource.ref); - this.jobArn = this.buildJobArn(this, resourceName); + this.jobArn = Job.buildJobArn(this, resourceName); this.jobName = resourceName; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-streaming-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-streaming-job.ts index 51c259b902a29..4d84248e327b3 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-streaming-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/scala-spark-streaming-job.ts @@ -1,9 +1,7 @@ -import { CfnJob } from 'aws-cdk-lib/aws-glue'; import { Construct } from 'constructs'; import { JobType, GlueVersion, JobLanguage, WorkerType } from '../constants'; -import { Code } from '../code'; import { SparkJob, SparkJobProps } from './spark-job'; -import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; +import { Job } from './job'; /** * Properties for creating a Scala Spark ETL job @@ -15,43 +13,6 @@ export interface ScalaSparkStreamingJobProps extends SparkJobProps { * Java scripts **/ readonly className: string; - - /** - * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. - * - * @default - no extra files specified. - * - * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraFiles?: Code[]; - - /** - * Extra Jars S3 URL (optional) - * S3 URL where additional jar dependencies are located - * @default - no extra jar files - */ - readonly extraJars?: Code[]; - - /** - * Setting this value to true prioritizes the customer's extra JAR files in the classpath. - * - * @default false - priority is not given to user-provided jars - * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html - */ - readonly extraJarsFirst?: boolean; - - /** - * Specifies whether job run queuing is enabled for the job runs for this job. - * A value of true means job run queuing is enabled for the job runs. - * If false or not populated, the job runs will not be considered for queueing. - * If this field does not match the value set in the job run, then the value from - * the job run field will be used. This property must be set to false for flex jobs. - * If this property is enabled, maxRetries must be set to zero. - * - * @default - no job run queuing - */ - readonly jobRunQueuingEnabled?: boolean; } /** @@ -75,8 +36,6 @@ export class ScalaSparkStreamingJob extends SparkJob { */ constructor(scope: Construct, id: string, props: ScalaSparkStreamingJobProps) { super(scope, id, props); - // Enhanced CDK Analytics Telemetry - addConstructMetadata(this, props); // Combine command line arguments into a single line item const defaultArguments = { @@ -88,10 +47,8 @@ export class ScalaSparkStreamingJob extends SparkJob { throw new Error('Both workerType and numberOfWorkers must be set'); } - const jobResource = new CfnJob(this, 'Resource', { - name: props.jobName, - description: props.description, - role: this.role.roleArn, + const jobResource = Job.setupJobResource(this, props, { + role: this.role!.roleArn, command: { name: JobType.STREAMING, scriptLocation: this.codeS3ObjectUrl(props.script), @@ -99,18 +56,11 @@ export class ScalaSparkStreamingJob extends SparkJob { glueVersion: props.glueVersion ? props.glueVersion : GlueVersion.V4_0, workerType: props.workerType ? props.workerType : WorkerType.G_1X, numberOfWorkers: props.numberOfWorkers ? props.numberOfWorkers : 10, - maxRetries: props.jobRunQueuingEnabled ? 0 : props.maxRetries, - jobRunQueuingEnabled: props.jobRunQueuingEnabled ? props.jobRunQueuingEnabled : false, - executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined, - timeout: props.timeout?.toMinutes(), - connections: props.connections ? { connections: props.connections.map((connection) => connection.connectionName) } : undefined, - securityConfiguration: props.securityConfiguration?.securityConfigurationName, - tags: props.tags, defaultArguments, }); const resourceName = this.getResourceNameAttribute(jobResource.ref); - this.jobArn = this.buildJobArn(this, resourceName); + this.jobArn = Job.buildJobArn(this, resourceName); this.jobName = resourceName; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/spark-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/spark-job.ts index 9e1ceaaa203bb..f2534cfea210d 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/spark-job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/spark-job.ts @@ -1,6 +1,4 @@ -import * as iam from 'aws-cdk-lib/aws-iam'; import * as s3 from 'aws-cdk-lib/aws-s3'; -import { addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; import * as constructs from 'constructs'; import { Code } from '../code'; import { Job, JobProps } from './job'; @@ -101,15 +99,37 @@ export interface SparkJobProps extends JobProps { * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly sparkUI?: SparkUIProps; + + /** + * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. + * + * @default - no extra files specified. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + */ + readonly extraFiles?: Code[]; + + /** + * Extra Jars S3 URL (optional) + * S3 URL where additional jar dependencies are located + * @default - no extra jar files + */ + readonly extraJars?: Code[]; + + /** + * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * + * @default false - priority is not given to user-provided jars + * + * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + */ + readonly extraJarsFirst?: boolean; } /** * Base class for different types of Spark Jobs. */ export abstract class SparkJob extends Job { - public readonly role: iam.IRole; - public readonly grantPrincipal: iam.IPrincipal; - /** * The Spark UI logs location if Spark UI monitoring and debugging is enabled. * @@ -118,22 +138,15 @@ export abstract class SparkJob extends Job { */ public readonly sparkUILoggingLocation?: SparkUILoggingLocation; - constructor(scope: constructs.Construct, id: string, props: SparkJobProps) { - super(scope, id, { - physicalName: props.jobName, - }); - // Enhanced CDK Analytics Telemetry - addConstructMetadata(this, props); - - this.role = props.role; - this.grantPrincipal = this.role; + protected constructor(scope: constructs.Construct, id: string, props: SparkJobProps) { + super(scope, id, props); this.sparkUILoggingLocation = props.sparkUI ? this.setupSparkUILoggingLocation(props.sparkUI) : undefined; } protected nonExecutableCommonArguments(props: SparkJobProps): {[key: string]: string} { // Enable CloudWatch metrics and continuous logging by default as a best practice - const continuousLoggingArgs = this.setupContinuousLogging(this.role, props.continuousLogging); + const continuousLoggingArgs = this.setupContinuousLogging(props.continuousLogging); const profilingMetricsArgs = { '--enable-metrics': '' }; const observabilityMetricsArgs = { '--enable-observability-metrics': 'true' }; diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/aws-glue-job-pyspark-etl.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/aws-glue-job-pyspark-etl.assets.json index 32888bf8f533a..1eb7aef813360 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/aws-glue-job-pyspark-etl.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/aws-glue-job-pyspark-etl.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,7 @@ } } }, - "6ae52026df1ea2305a8d7321f0dd63581ab7f4c58a118f28964977e2c73af55a": { + "7f8a3f6526b1baec270fab32079680dcb65f351c84cd130f096e1deb166f071e": { "source": { "path": "aws-glue-job-pyspark-etl.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "6ae52026df1ea2305a8d7321f0dd63581ab7f4c58a118f28964977e2c73af55a.json", + "objectKey": "7f8a3f6526b1baec270fab32079680dcb65f351c84cd130f096e1deb166f071e.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/aws-glue-job-pyspark-etl.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/aws-glue-job-pyspark-etl.template.json index 0a0ee7f338ab2..42e4beb2abf44 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/aws-glue-job-pyspark-etl.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/aws-glue-job-pyspark-etl.template.json @@ -157,6 +157,9 @@ "JobRunQueuingEnabled": true, "MaxRetries": 0, "Name": "Optional Override PySpark ETL Job", + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, "NumberOfWorkers": 20, "Role": { "Fn::GetAtt": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/awsgluejobpysparketlintegtestDefaultTestDeployAssertED1ACE14.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/awsgluejobpysparketlintegtestDefaultTestDeployAssertED1ACE14.assets.json index 7a5adc4449cd0..bff14d8c7e20a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/awsgluejobpysparketlintegtestDefaultTestDeployAssertED1ACE14.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/awsgluejobpysparketlintegtestDefaultTestDeployAssertED1ACE14.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/cdk.out index c6e612584e352..1e02a2deb191b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"38.0.1"} \ No newline at end of file +{"version":"40.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/integ.json index 624540faf2914..71e17bdcced03 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "testCases": { "aws-glue-job-pyspark-etl-integ-test/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/manifest.json index 90c354b0fffa8..6fb9f38336eda 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "artifacts": { "aws-glue-job-pyspark-etl.assets": { "type": "cdk:asset-manifest", @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/6ae52026df1ea2305a8d7321f0dd63581ab7f4c58a118f28964977e2c73af55a.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/7f8a3f6526b1baec270fab32079680dcb65f351c84cd130f096e1deb166f071e.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -34,24 +34,136 @@ "aws-glue-job-pyspark-etl.assets" ], "metadata": { + "/aws-glue-job-pyspark-etl/IAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": { + "assumedBy": { + "principalAccount": "*", + "assumeRoleAction": "*" + }, + "managedPolicies": [ + { + "managedPolicyArn": "*" + } + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + } + ], + "/aws-glue-job-pyspark-etl/IAMServiceRole/ImportIAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pyspark-etl/IAMServiceRole/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRole61C662C4" } ], + "/aws-glue-job-pyspark-etl/IAMServiceRole/DefaultPolicy": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + } + ], "/aws-glue-job-pyspark-etl/IAMServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRoleDefaultPolicy379D1A0E" } ], + "/aws-glue-job-pyspark-etl/BasicPySparkETLJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pyspark-etl/BasicPySparkETLJob/Resource": [ { "type": "aws:cdk:logicalId", "data": "BasicPySparkETLJob833DD8C4" } ], + "/aws-glue-job-pyspark-etl/OverridePySparkETLJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pyspark-etl/OverridePySparkETLJob/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/tree.json index 8a313dd69fc3b..c4104265a035e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.js.snapshot/tree.json @@ -1,378 +1 @@ -{ - "version": "tree-0.1", - "tree": { - "id": "App", - "path": "", - "children": { - "aws-glue-job-pyspark-etl": { - "id": "aws-glue-job-pyspark-etl", - "path": "aws-glue-job-pyspark-etl", - "children": { - "IAMServiceRole": { - "id": "IAMServiceRole", - "path": "aws-glue-job-pyspark-etl/IAMServiceRole", - "children": { - "ImportIAMServiceRole": { - "id": "ImportIAMServiceRole", - "path": "aws-glue-job-pyspark-etl/IAMServiceRole/ImportIAMServiceRole", - "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pyspark-etl/IAMServiceRole/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Role", - "aws:cdk:cloudformation:props": { - "assumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "managedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSGlueServiceRole" - ] - ] - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" - } - }, - "DefaultPolicy": { - "id": "DefaultPolicy", - "path": "aws-glue-job-pyspark-etl/IAMServiceRole/DefaultPolicy", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pyspark-etl/IAMServiceRole/DefaultPolicy/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Policy", - "aws:cdk:cloudformation:props": { - "policyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - } - ] - ] - } - ] - } - ], - "Version": "2012-10-17" - }, - "policyName": "IAMServiceRoleDefaultPolicy379D1A0E", - "roles": [ - { - "Ref": "IAMServiceRole61C662C4" - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" - } - }, - "BasicPySparkETLJob": { - "id": "BasicPySparkETLJob", - "path": "aws-glue-job-pyspark-etl/BasicPySparkETLJob", - "children": { - "Code2907ea7be4a583708cfffc21b3df1dfa": { - "id": "Code2907ea7be4a583708cfffc21b3df1dfa", - "path": "aws-glue-job-pyspark-etl/BasicPySparkETLJob/Code2907ea7be4a583708cfffc21b3df1dfa", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job-pyspark-etl/BasicPySparkETLJob/Code2907ea7be4a583708cfffc21b3df1dfa/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job-pyspark-etl/BasicPySparkETLJob/Code2907ea7be4a583708cfffc21b3df1dfa/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pyspark-etl/BasicPySparkETLJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "glueetl", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3" - }, - "defaultArguments": { - "--job-language": "python", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true" - }, - "glueVersion": "4.0", - "jobRunQueuingEnabled": false, - "numberOfWorkers": 10, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PySparkEtlJob", - "version": "0.0.0" - } - }, - "OverridePySparkETLJob": { - "id": "OverridePySparkETLJob", - "path": "aws-glue-job-pyspark-etl/OverridePySparkETLJob", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pyspark-etl/OverridePySparkETLJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "glueetl", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3" - }, - "defaultArguments": { - "--job-language": "python", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true", - "arg1": "value1", - "arg2": "value2" - }, - "description": "Optional Override PySpark ETL Job", - "glueVersion": "3.0", - "jobRunQueuingEnabled": true, - "maxRetries": 0, - "name": "Optional Override PySpark ETL Job", - "numberOfWorkers": 20, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "tags": { - "key": "value" - }, - "timeout": 15, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PySparkEtlJob", - "version": "0.0.0" - } - }, - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-pyspark-etl/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-pyspark-etl/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - }, - "aws-glue-job-pyspark-etl-integ-test": { - "id": "aws-glue-job-pyspark-etl-integ-test", - "path": "aws-glue-job-pyspark-etl-integ-test", - "children": { - "DefaultTest": { - "id": "DefaultTest", - "path": "aws-glue-job-pyspark-etl-integ-test/DefaultTest", - "children": { - "Default": { - "id": "Default", - "path": "aws-glue-job-pyspark-etl-integ-test/DefaultTest/Default", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - }, - "DeployAssert": { - "id": "DeployAssert", - "path": "aws-glue-job-pyspark-etl-integ-test/DefaultTest/DeployAssert", - "children": { - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-pyspark-etl-integ-test/DefaultTest/DeployAssert/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-pyspark-etl-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", - "version": "0.0.0" - } - }, - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.App", - "version": "0.0.0" - } - } -} \ No newline at end of file +{"version":"tree-0.1","tree":{"id":"App","path":"","children":{"aws-glue-job-pyspark-etl":{"id":"aws-glue-job-pyspark-etl","path":"aws-glue-job-pyspark-etl","children":{"IAMServiceRole":{"id":"IAMServiceRole","path":"aws-glue-job-pyspark-etl/IAMServiceRole","children":{"ImportIAMServiceRole":{"id":"ImportIAMServiceRole","path":"aws-glue-job-pyspark-etl/IAMServiceRole/ImportIAMServiceRole","constructInfo":{"fqn":"aws-cdk-lib.Resource","version":"0.0.0","metadata":["*"]}},"Resource":{"id":"Resource","path":"aws-glue-job-pyspark-etl/IAMServiceRole/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Role","aws:cdk:cloudformation:props":{"assumeRolePolicyDocument":{"Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":"glue.amazonaws.com"}}],"Version":"2012-10-17"},"managedPolicyArns":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":iam::aws:policy/service-role/AWSGlueServiceRole"]]}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnRole","version":"0.0.0"}},"DefaultPolicy":{"id":"DefaultPolicy","path":"aws-glue-job-pyspark-etl/IAMServiceRole/DefaultPolicy","children":{"Resource":{"id":"Resource","path":"aws-glue-job-pyspark-etl/IAMServiceRole/DefaultPolicy/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Policy","aws:cdk:cloudformation:props":{"policyDocument":{"Statement":[{"Action":["s3:GetBucket*","s3:GetObject*","s3:List*"],"Effect":"Allow","Resource":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/*"]]},{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}]]}]}],"Version":"2012-10-17"},"policyName":"IAMServiceRoleDefaultPolicy379D1A0E","roles":[{"Ref":"IAMServiceRole61C662C4"}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnPolicy","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Policy","version":"0.0.0","metadata":["*",{"attachToRole":["*"]},{"attachToRole":["*"]},{"addStatements":[{}]},{"addStatements":[{}]}]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Role","version":"0.0.0","metadata":[{"assumedBy":{"principalAccount":"*","assumeRoleAction":"*"},"managedPolicies":[{"managedPolicyArn":"*"}]},{"addToPrincipalPolicy":[{}]},{"attachInlinePolicy":["*"]},{"attachInlinePolicy":["*"]},{"addToPrincipalPolicy":[{}]}]}},"BasicPySparkETLJob":{"id":"BasicPySparkETLJob","path":"aws-glue-job-pyspark-etl/BasicPySparkETLJob","children":{"Code1de73119980d143bf808e10735354e19":{"id":"Code1de73119980d143bf808e10735354e19","path":"aws-glue-job-pyspark-etl/BasicPySparkETLJob/Code1de73119980d143bf808e10735354e19","children":{"Stage":{"id":"Stage","path":"aws-glue-job-pyspark-etl/BasicPySparkETLJob/Code1de73119980d143bf808e10735354e19/Stage","constructInfo":{"fqn":"aws-cdk-lib.AssetStaging","version":"0.0.0"}},"AssetBucket":{"id":"AssetBucket","path":"aws-glue-job-pyspark-etl/BasicPySparkETLJob/Code1de73119980d143bf808e10735354e19/AssetBucket","constructInfo":{"fqn":"aws-cdk-lib.aws_s3.BucketBase","version":"0.0.0","metadata":[]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_s3_assets.Asset","version":"0.0.0"}},"Resource":{"id":"Resource","path":"aws-glue-job-pyspark-etl/BasicPySparkETLJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"glueetl","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3"},"defaultArguments":{"--job-language":"python","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true"},"glueVersion":"4.0","jobRunQueuingEnabled":false,"numberOfWorkers":10,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PySparkEtlJob","version":"0.0.0","metadata":["*","*"]}},"OverridePySparkETLJob":{"id":"OverridePySparkETLJob","path":"aws-glue-job-pyspark-etl/OverridePySparkETLJob","children":{"Resource":{"id":"Resource","path":"aws-glue-job-pyspark-etl/OverridePySparkETLJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"glueetl","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3"},"defaultArguments":{"--job-language":"python","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true","arg1":"value1","arg2":"value2"},"description":"Optional Override PySpark ETL Job","glueVersion":"3.0","jobRunQueuingEnabled":true,"maxRetries":0,"name":"Optional Override PySpark ETL Job","notificationProperty":{"notifyDelayAfter":5},"numberOfWorkers":20,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"tags":{"key":"value"},"timeout":15,"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PySparkEtlJob","version":"0.0.0","metadata":["*","*"]}},"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-pyspark-etl/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-pyspark-etl/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}},"aws-glue-job-pyspark-etl-integ-test":{"id":"aws-glue-job-pyspark-etl-integ-test","path":"aws-glue-job-pyspark-etl-integ-test","children":{"DefaultTest":{"id":"DefaultTest","path":"aws-glue-job-pyspark-etl-integ-test/DefaultTest","children":{"Default":{"id":"Default","path":"aws-glue-job-pyspark-etl-integ-test/DefaultTest/Default","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}},"DeployAssert":{"id":"DeployAssert","path":"aws-glue-job-pyspark-etl-integ-test/DefaultTest/DeployAssert","children":{"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-pyspark-etl-integ-test/DefaultTest/DeployAssert/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-pyspark-etl-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTestCase","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTest","version":"0.0.0"}},"Tree":{"id":"Tree","path":"Tree","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}}},"constructInfo":{"fqn":"aws-cdk-lib.App","version":"0.0.0"}}} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.ts index 775e5230d8db9..99cebe711b0ca 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-etl.ts @@ -52,6 +52,7 @@ new glue.PySparkEtlJob(stack, 'OverridePySparkETLJob', { key: 'value', }, jobRunQueuingEnabled: true, + notifyDelayAfter: cdk.Duration.minutes(5), }); new integ.IntegTest(app, 'aws-glue-job-pyspark-etl-integ-test', { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/aws-glue-job-pysparkflex-etl.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/aws-glue-job-pysparkflex-etl.assets.json index cd109b9b42881..9f24da0d432d8 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/aws-glue-job-pysparkflex-etl.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/aws-glue-job-pysparkflex-etl.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,7 @@ } } }, - "afa3fdc251199a9fb365e05d666fa02e1d45fff2007575ac4e684653d10597fb": { + "ed4cadb4f1ce505f83365c3de27300ffc496e63411e79cb221cc4a07da9d6604": { "source": { "path": "aws-glue-job-pysparkflex-etl.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "afa3fdc251199a9fb365e05d666fa02e1d45fff2007575ac4e684653d10597fb.json", + "objectKey": "ed4cadb4f1ce505f83365c3de27300ffc496e63411e79cb221cc4a07da9d6604.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/aws-glue-job-pysparkflex-etl.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/aws-glue-job-pysparkflex-etl.template.json index 55b82ea5637d3..fb16447610f21 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/aws-glue-job-pysparkflex-etl.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/aws-glue-job-pysparkflex-etl.template.json @@ -158,6 +158,9 @@ "GlueVersion": "3.0", "JobRunQueuingEnabled": false, "Name": "Optional Override PySpark Flex Etl Job", + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, "NumberOfWorkers": 20, "Role": { "Fn::GetAtt": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/awsgluejobpysparkflexetlintegtestDefaultTestDeployAssert3F3EC951.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/awsgluejobpysparkflexetlintegtestDefaultTestDeployAssert3F3EC951.assets.json index 5689ec7c7578d..7b46d11a8e2f1 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/awsgluejobpysparkflexetlintegtestDefaultTestDeployAssert3F3EC951.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/awsgluejobpysparkflexetlintegtestDefaultTestDeployAssert3F3EC951.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/cdk.out index c6e612584e352..1e02a2deb191b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"38.0.1"} \ No newline at end of file +{"version":"40.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/integ.json index 4871676a6b628..276be7d481c62 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "testCases": { "aws-glue-job-pysparkflex-etl-integ-test/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/manifest.json index b94253bb2aeb6..49f67ad3d594f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "artifacts": { "aws-glue-job-pysparkflex-etl.assets": { "type": "cdk:asset-manifest", @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/afa3fdc251199a9fb365e05d666fa02e1d45fff2007575ac4e684653d10597fb.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/ed4cadb4f1ce505f83365c3de27300ffc496e63411e79cb221cc4a07da9d6604.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -34,24 +34,136 @@ "aws-glue-job-pysparkflex-etl.assets" ], "metadata": { + "/aws-glue-job-pysparkflex-etl/IAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": { + "assumedBy": { + "principalAccount": "*", + "assumeRoleAction": "*" + }, + "managedPolicies": [ + { + "managedPolicyArn": "*" + } + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + } + ], + "/aws-glue-job-pysparkflex-etl/IAMServiceRole/ImportIAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pysparkflex-etl/IAMServiceRole/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRole61C662C4" } ], + "/aws-glue-job-pysparkflex-etl/IAMServiceRole/DefaultPolicy": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + } + ], "/aws-glue-job-pysparkflex-etl/IAMServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRoleDefaultPolicy379D1A0E" } ], + "/aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Resource": [ { "type": "aws:cdk:logicalId", "data": "BasicPySparkFlexEtlJobC50DC250" } ], + "/aws-glue-job-pysparkflex-etl/OverridePySparkFlexEtlJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pysparkflex-etl/OverridePySparkFlexEtlJob/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/tree.json index 79cc1e97d1416..d02022e0cdd26 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.js.snapshot/tree.json @@ -1,379 +1 @@ -{ - "version": "tree-0.1", - "tree": { - "id": "App", - "path": "", - "children": { - "aws-glue-job-pysparkflex-etl": { - "id": "aws-glue-job-pysparkflex-etl", - "path": "aws-glue-job-pysparkflex-etl", - "children": { - "IAMServiceRole": { - "id": "IAMServiceRole", - "path": "aws-glue-job-pysparkflex-etl/IAMServiceRole", - "children": { - "ImportIAMServiceRole": { - "id": "ImportIAMServiceRole", - "path": "aws-glue-job-pysparkflex-etl/IAMServiceRole/ImportIAMServiceRole", - "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pysparkflex-etl/IAMServiceRole/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Role", - "aws:cdk:cloudformation:props": { - "assumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "managedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSGlueServiceRole" - ] - ] - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" - } - }, - "DefaultPolicy": { - "id": "DefaultPolicy", - "path": "aws-glue-job-pysparkflex-etl/IAMServiceRole/DefaultPolicy", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pysparkflex-etl/IAMServiceRole/DefaultPolicy/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Policy", - "aws:cdk:cloudformation:props": { - "policyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - } - ] - ] - } - ] - } - ], - "Version": "2012-10-17" - }, - "policyName": "IAMServiceRoleDefaultPolicy379D1A0E", - "roles": [ - { - "Ref": "IAMServiceRole61C662C4" - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" - } - }, - "BasicPySparkFlexEtlJob": { - "id": "BasicPySparkFlexEtlJob", - "path": "aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob", - "children": { - "Code2907ea7be4a583708cfffc21b3df1dfa": { - "id": "Code2907ea7be4a583708cfffc21b3df1dfa", - "path": "aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Code2907ea7be4a583708cfffc21b3df1dfa", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Code2907ea7be4a583708cfffc21b3df1dfa/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Code2907ea7be4a583708cfffc21b3df1dfa/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "glueetl", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3" - }, - "defaultArguments": { - "--job-language": "python", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true" - }, - "executionClass": "FLEX", - "glueVersion": "3.0", - "jobRunQueuingEnabled": false, - "numberOfWorkers": 10, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PySparkFlexEtlJob", - "version": "0.0.0" - } - }, - "OverridePySparkFlexEtlJob": { - "id": "OverridePySparkFlexEtlJob", - "path": "aws-glue-job-pysparkflex-etl/OverridePySparkFlexEtlJob", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pysparkflex-etl/OverridePySparkFlexEtlJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "glueetl", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3" - }, - "defaultArguments": { - "--job-language": "python", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true", - "arg1": "value1", - "arg2": "value2" - }, - "description": "Optional Override PySpark Flex Etl Job", - "executionClass": "FLEX", - "glueVersion": "3.0", - "jobRunQueuingEnabled": false, - "name": "Optional Override PySpark Flex Etl Job", - "numberOfWorkers": 20, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "tags": { - "key": "value" - }, - "timeout": 15, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PySparkFlexEtlJob", - "version": "0.0.0" - } - }, - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-pysparkflex-etl/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-pysparkflex-etl/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - }, - "aws-glue-job-pysparkflex-etl-integ-test": { - "id": "aws-glue-job-pysparkflex-etl-integ-test", - "path": "aws-glue-job-pysparkflex-etl-integ-test", - "children": { - "DefaultTest": { - "id": "DefaultTest", - "path": "aws-glue-job-pysparkflex-etl-integ-test/DefaultTest", - "children": { - "Default": { - "id": "Default", - "path": "aws-glue-job-pysparkflex-etl-integ-test/DefaultTest/Default", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - }, - "DeployAssert": { - "id": "DeployAssert", - "path": "aws-glue-job-pysparkflex-etl-integ-test/DefaultTest/DeployAssert", - "children": { - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-pysparkflex-etl-integ-test/DefaultTest/DeployAssert/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-pysparkflex-etl-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", - "version": "0.0.0" - } - }, - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.App", - "version": "0.0.0" - } - } -} \ No newline at end of file +{"version":"tree-0.1","tree":{"id":"App","path":"","children":{"aws-glue-job-pysparkflex-etl":{"id":"aws-glue-job-pysparkflex-etl","path":"aws-glue-job-pysparkflex-etl","children":{"IAMServiceRole":{"id":"IAMServiceRole","path":"aws-glue-job-pysparkflex-etl/IAMServiceRole","children":{"ImportIAMServiceRole":{"id":"ImportIAMServiceRole","path":"aws-glue-job-pysparkflex-etl/IAMServiceRole/ImportIAMServiceRole","constructInfo":{"fqn":"aws-cdk-lib.Resource","version":"0.0.0","metadata":["*"]}},"Resource":{"id":"Resource","path":"aws-glue-job-pysparkflex-etl/IAMServiceRole/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Role","aws:cdk:cloudformation:props":{"assumeRolePolicyDocument":{"Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":"glue.amazonaws.com"}}],"Version":"2012-10-17"},"managedPolicyArns":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":iam::aws:policy/service-role/AWSGlueServiceRole"]]}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnRole","version":"0.0.0"}},"DefaultPolicy":{"id":"DefaultPolicy","path":"aws-glue-job-pysparkflex-etl/IAMServiceRole/DefaultPolicy","children":{"Resource":{"id":"Resource","path":"aws-glue-job-pysparkflex-etl/IAMServiceRole/DefaultPolicy/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Policy","aws:cdk:cloudformation:props":{"policyDocument":{"Statement":[{"Action":["s3:GetBucket*","s3:GetObject*","s3:List*"],"Effect":"Allow","Resource":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/*"]]},{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}]]}]}],"Version":"2012-10-17"},"policyName":"IAMServiceRoleDefaultPolicy379D1A0E","roles":[{"Ref":"IAMServiceRole61C662C4"}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnPolicy","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Policy","version":"0.0.0","metadata":["*",{"attachToRole":["*"]},{"attachToRole":["*"]},{"addStatements":[{}]},{"addStatements":[{}]}]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Role","version":"0.0.0","metadata":[{"assumedBy":{"principalAccount":"*","assumeRoleAction":"*"},"managedPolicies":[{"managedPolicyArn":"*"}]},{"addToPrincipalPolicy":[{}]},{"attachInlinePolicy":["*"]},{"attachInlinePolicy":["*"]},{"addToPrincipalPolicy":[{}]}]}},"BasicPySparkFlexEtlJob":{"id":"BasicPySparkFlexEtlJob","path":"aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob","children":{"Code1de73119980d143bf808e10735354e19":{"id":"Code1de73119980d143bf808e10735354e19","path":"aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Code1de73119980d143bf808e10735354e19","children":{"Stage":{"id":"Stage","path":"aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Code1de73119980d143bf808e10735354e19/Stage","constructInfo":{"fqn":"aws-cdk-lib.AssetStaging","version":"0.0.0"}},"AssetBucket":{"id":"AssetBucket","path":"aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Code1de73119980d143bf808e10735354e19/AssetBucket","constructInfo":{"fqn":"aws-cdk-lib.aws_s3.BucketBase","version":"0.0.0","metadata":[]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_s3_assets.Asset","version":"0.0.0"}},"Resource":{"id":"Resource","path":"aws-glue-job-pysparkflex-etl/BasicPySparkFlexEtlJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"glueetl","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3"},"defaultArguments":{"--job-language":"python","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true"},"executionClass":"FLEX","glueVersion":"3.0","jobRunQueuingEnabled":false,"numberOfWorkers":10,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PySparkFlexEtlJob","version":"0.0.0","metadata":["*","*"]}},"OverridePySparkFlexEtlJob":{"id":"OverridePySparkFlexEtlJob","path":"aws-glue-job-pysparkflex-etl/OverridePySparkFlexEtlJob","children":{"Resource":{"id":"Resource","path":"aws-glue-job-pysparkflex-etl/OverridePySparkFlexEtlJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"glueetl","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3"},"defaultArguments":{"--job-language":"python","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true","arg1":"value1","arg2":"value2"},"description":"Optional Override PySpark Flex Etl Job","executionClass":"FLEX","glueVersion":"3.0","jobRunQueuingEnabled":false,"name":"Optional Override PySpark Flex Etl Job","notificationProperty":{"notifyDelayAfter":5},"numberOfWorkers":20,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"tags":{"key":"value"},"timeout":15,"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PySparkFlexEtlJob","version":"0.0.0","metadata":["*","*"]}},"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-pysparkflex-etl/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-pysparkflex-etl/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}},"aws-glue-job-pysparkflex-etl-integ-test":{"id":"aws-glue-job-pysparkflex-etl-integ-test","path":"aws-glue-job-pysparkflex-etl-integ-test","children":{"DefaultTest":{"id":"DefaultTest","path":"aws-glue-job-pysparkflex-etl-integ-test/DefaultTest","children":{"Default":{"id":"Default","path":"aws-glue-job-pysparkflex-etl-integ-test/DefaultTest/Default","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}},"DeployAssert":{"id":"DeployAssert","path":"aws-glue-job-pysparkflex-etl-integ-test/DefaultTest/DeployAssert","children":{"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-pysparkflex-etl-integ-test/DefaultTest/DeployAssert/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-pysparkflex-etl-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTestCase","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTest","version":"0.0.0"}},"Tree":{"id":"Tree","path":"Tree","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}}},"constructInfo":{"fqn":"aws-cdk-lib.App","version":"0.0.0"}}} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.ts index 14b53a7490304..c54a305cdffe1 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-flex-etl.ts @@ -57,6 +57,7 @@ new glue.PySparkFlexEtlJob(stack, 'OverridePySparkFlexEtlJob', { tags: { key: 'value', }, + notifyDelayAfter: cdk.Duration.minutes(5), }); new integ.IntegTest(app, 'aws-glue-job-pysparkflex-etl-integ-test', { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/aws-glue-job-pyspark-streaming.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/aws-glue-job-pyspark-streaming.assets.json index 3cc272907910d..076d323dbd919 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/aws-glue-job-pyspark-streaming.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/aws-glue-job-pyspark-streaming.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,7 @@ } } }, - "09d72521949f27a084c5d8161fed094012e7bd42e95d2e7d292737f4c6118077": { + "dace170bf34ed434f946e2a63a0500e78b90c2543fafb2af084e14e1efc76172": { "source": { "path": "aws-glue-job-pyspark-streaming.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "09d72521949f27a084c5d8161fed094012e7bd42e95d2e7d292737f4c6118077.json", + "objectKey": "dace170bf34ed434f946e2a63a0500e78b90c2543fafb2af084e14e1efc76172.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/aws-glue-job-pyspark-streaming.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/aws-glue-job-pyspark-streaming.template.json index cf50764326545..74fbcd701cd20 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/aws-glue-job-pyspark-streaming.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/aws-glue-job-pyspark-streaming.template.json @@ -157,6 +157,9 @@ "JobRunQueuingEnabled": true, "MaxRetries": 0, "Name": "Optional Override PySpark Streaming Job", + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, "NumberOfWorkers": 20, "Role": { "Fn::GetAtt": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/awsgluejobpysparkstreamingintegtestDefaultTestDeployAssert242E520E.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/awsgluejobpysparkstreamingintegtestDefaultTestDeployAssert242E520E.assets.json index 482082291c1a3..7c1590423bd84 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/awsgluejobpysparkstreamingintegtestDefaultTestDeployAssert242E520E.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/awsgluejobpysparkstreamingintegtestDefaultTestDeployAssert242E520E.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/cdk.out index c6e612584e352..1e02a2deb191b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"38.0.1"} \ No newline at end of file +{"version":"40.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/integ.json index a67021772725a..174dd546af91d 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "testCases": { "aws-glue-job-pyspark-streaming-integ-test/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/manifest.json index 7a70749f4848f..cac064f4b2574 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "artifacts": { "aws-glue-job-pyspark-streaming.assets": { "type": "cdk:asset-manifest", @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/09d72521949f27a084c5d8161fed094012e7bd42e95d2e7d292737f4c6118077.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/dace170bf34ed434f946e2a63a0500e78b90c2543fafb2af084e14e1efc76172.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -34,24 +34,136 @@ "aws-glue-job-pyspark-streaming.assets" ], "metadata": { + "/aws-glue-job-pyspark-streaming/IAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": { + "assumedBy": { + "principalAccount": "*", + "assumeRoleAction": "*" + }, + "managedPolicies": [ + { + "managedPolicyArn": "*" + } + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + } + ], + "/aws-glue-job-pyspark-streaming/IAMServiceRole/ImportIAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pyspark-streaming/IAMServiceRole/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRole61C662C4" } ], + "/aws-glue-job-pyspark-streaming/IAMServiceRole/DefaultPolicy": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + } + ], "/aws-glue-job-pyspark-streaming/IAMServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRoleDefaultPolicy379D1A0E" } ], + "/aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Resource": [ { "type": "aws:cdk:logicalId", "data": "BasicPySparkStreamingJobAFD3B477" } ], + "/aws-glue-job-pyspark-streaming/OverridePySparkStreamingJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-pyspark-streaming/OverridePySparkStreamingJob/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/tree.json index 87ac0cf6da073..34b8caa0db8ad 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.js.snapshot/tree.json @@ -1,378 +1 @@ -{ - "version": "tree-0.1", - "tree": { - "id": "App", - "path": "", - "children": { - "aws-glue-job-pyspark-streaming": { - "id": "aws-glue-job-pyspark-streaming", - "path": "aws-glue-job-pyspark-streaming", - "children": { - "IAMServiceRole": { - "id": "IAMServiceRole", - "path": "aws-glue-job-pyspark-streaming/IAMServiceRole", - "children": { - "ImportIAMServiceRole": { - "id": "ImportIAMServiceRole", - "path": "aws-glue-job-pyspark-streaming/IAMServiceRole/ImportIAMServiceRole", - "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pyspark-streaming/IAMServiceRole/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Role", - "aws:cdk:cloudformation:props": { - "assumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "managedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSGlueServiceRole" - ] - ] - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" - } - }, - "DefaultPolicy": { - "id": "DefaultPolicy", - "path": "aws-glue-job-pyspark-streaming/IAMServiceRole/DefaultPolicy", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pyspark-streaming/IAMServiceRole/DefaultPolicy/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Policy", - "aws:cdk:cloudformation:props": { - "policyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - } - ] - ] - } - ] - } - ], - "Version": "2012-10-17" - }, - "policyName": "IAMServiceRoleDefaultPolicy379D1A0E", - "roles": [ - { - "Ref": "IAMServiceRole61C662C4" - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" - } - }, - "BasicPySparkStreamingJob": { - "id": "BasicPySparkStreamingJob", - "path": "aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob", - "children": { - "Code2907ea7be4a583708cfffc21b3df1dfa": { - "id": "Code2907ea7be4a583708cfffc21b3df1dfa", - "path": "aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Code2907ea7be4a583708cfffc21b3df1dfa", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Code2907ea7be4a583708cfffc21b3df1dfa/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Code2907ea7be4a583708cfffc21b3df1dfa/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "gluestreaming", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3" - }, - "defaultArguments": { - "--job-language": "python", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true" - }, - "glueVersion": "4.0", - "jobRunQueuingEnabled": false, - "numberOfWorkers": 10, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PySparkStreamingJob", - "version": "0.0.0" - } - }, - "OverridePySparkStreamingJob": { - "id": "OverridePySparkStreamingJob", - "path": "aws-glue-job-pyspark-streaming/OverridePySparkStreamingJob", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-pyspark-streaming/OverridePySparkStreamingJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "gluestreaming", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3" - }, - "defaultArguments": { - "--job-language": "python", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true", - "arg1": "value1", - "arg2": "value2" - }, - "description": "Optional Override PySpark Streaming Job", - "glueVersion": "3.0", - "jobRunQueuingEnabled": true, - "maxRetries": 0, - "name": "Optional Override PySpark Streaming Job", - "numberOfWorkers": 20, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "tags": { - "key": "value" - }, - "timeout": 15, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PySparkStreamingJob", - "version": "0.0.0" - } - }, - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-pyspark-streaming/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-pyspark-streaming/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - }, - "aws-glue-job-pyspark-streaming-integ-test": { - "id": "aws-glue-job-pyspark-streaming-integ-test", - "path": "aws-glue-job-pyspark-streaming-integ-test", - "children": { - "DefaultTest": { - "id": "DefaultTest", - "path": "aws-glue-job-pyspark-streaming-integ-test/DefaultTest", - "children": { - "Default": { - "id": "Default", - "path": "aws-glue-job-pyspark-streaming-integ-test/DefaultTest/Default", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - }, - "DeployAssert": { - "id": "DeployAssert", - "path": "aws-glue-job-pyspark-streaming-integ-test/DefaultTest/DeployAssert", - "children": { - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-pyspark-streaming-integ-test/DefaultTest/DeployAssert/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-pyspark-streaming-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", - "version": "0.0.0" - } - }, - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.App", - "version": "0.0.0" - } - } -} \ No newline at end of file +{"version":"tree-0.1","tree":{"id":"App","path":"","children":{"aws-glue-job-pyspark-streaming":{"id":"aws-glue-job-pyspark-streaming","path":"aws-glue-job-pyspark-streaming","children":{"IAMServiceRole":{"id":"IAMServiceRole","path":"aws-glue-job-pyspark-streaming/IAMServiceRole","children":{"ImportIAMServiceRole":{"id":"ImportIAMServiceRole","path":"aws-glue-job-pyspark-streaming/IAMServiceRole/ImportIAMServiceRole","constructInfo":{"fqn":"aws-cdk-lib.Resource","version":"0.0.0","metadata":["*"]}},"Resource":{"id":"Resource","path":"aws-glue-job-pyspark-streaming/IAMServiceRole/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Role","aws:cdk:cloudformation:props":{"assumeRolePolicyDocument":{"Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":"glue.amazonaws.com"}}],"Version":"2012-10-17"},"managedPolicyArns":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":iam::aws:policy/service-role/AWSGlueServiceRole"]]}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnRole","version":"0.0.0"}},"DefaultPolicy":{"id":"DefaultPolicy","path":"aws-glue-job-pyspark-streaming/IAMServiceRole/DefaultPolicy","children":{"Resource":{"id":"Resource","path":"aws-glue-job-pyspark-streaming/IAMServiceRole/DefaultPolicy/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Policy","aws:cdk:cloudformation:props":{"policyDocument":{"Statement":[{"Action":["s3:GetBucket*","s3:GetObject*","s3:List*"],"Effect":"Allow","Resource":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/*"]]},{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}]]}]}],"Version":"2012-10-17"},"policyName":"IAMServiceRoleDefaultPolicy379D1A0E","roles":[{"Ref":"IAMServiceRole61C662C4"}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnPolicy","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Policy","version":"0.0.0","metadata":["*",{"attachToRole":["*"]},{"attachToRole":["*"]},{"addStatements":[{}]},{"addStatements":[{}]}]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Role","version":"0.0.0","metadata":[{"assumedBy":{"principalAccount":"*","assumeRoleAction":"*"},"managedPolicies":[{"managedPolicyArn":"*"}]},{"addToPrincipalPolicy":[{}]},{"attachInlinePolicy":["*"]},{"attachInlinePolicy":["*"]},{"addToPrincipalPolicy":[{}]}]}},"BasicPySparkStreamingJob":{"id":"BasicPySparkStreamingJob","path":"aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob","children":{"Code1de73119980d143bf808e10735354e19":{"id":"Code1de73119980d143bf808e10735354e19","path":"aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Code1de73119980d143bf808e10735354e19","children":{"Stage":{"id":"Stage","path":"aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Code1de73119980d143bf808e10735354e19/Stage","constructInfo":{"fqn":"aws-cdk-lib.AssetStaging","version":"0.0.0"}},"AssetBucket":{"id":"AssetBucket","path":"aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Code1de73119980d143bf808e10735354e19/AssetBucket","constructInfo":{"fqn":"aws-cdk-lib.aws_s3.BucketBase","version":"0.0.0","metadata":[]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_s3_assets.Asset","version":"0.0.0"}},"Resource":{"id":"Resource","path":"aws-glue-job-pyspark-streaming/BasicPySparkStreamingJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"gluestreaming","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3"},"defaultArguments":{"--job-language":"python","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true"},"glueVersion":"4.0","jobRunQueuingEnabled":false,"numberOfWorkers":10,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PySparkStreamingJob","version":"0.0.0","metadata":["*","*"]}},"OverridePySparkStreamingJob":{"id":"OverridePySparkStreamingJob","path":"aws-glue-job-pyspark-streaming/OverridePySparkStreamingJob","children":{"Resource":{"id":"Resource","path":"aws-glue-job-pyspark-streaming/OverridePySparkStreamingJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"gluestreaming","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3"},"defaultArguments":{"--job-language":"python","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true","arg1":"value1","arg2":"value2"},"description":"Optional Override PySpark Streaming Job","glueVersion":"3.0","jobRunQueuingEnabled":true,"maxRetries":0,"name":"Optional Override PySpark Streaming Job","notificationProperty":{"notifyDelayAfter":5},"numberOfWorkers":20,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"tags":{"key":"value"},"timeout":15,"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PySparkStreamingJob","version":"0.0.0","metadata":["*","*"]}},"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-pyspark-streaming/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-pyspark-streaming/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}},"aws-glue-job-pyspark-streaming-integ-test":{"id":"aws-glue-job-pyspark-streaming-integ-test","path":"aws-glue-job-pyspark-streaming-integ-test","children":{"DefaultTest":{"id":"DefaultTest","path":"aws-glue-job-pyspark-streaming-integ-test/DefaultTest","children":{"Default":{"id":"Default","path":"aws-glue-job-pyspark-streaming-integ-test/DefaultTest/Default","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}},"DeployAssert":{"id":"DeployAssert","path":"aws-glue-job-pyspark-streaming-integ-test/DefaultTest/DeployAssert","children":{"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-pyspark-streaming-integ-test/DefaultTest/DeployAssert/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-pyspark-streaming-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTestCase","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTest","version":"0.0.0"}},"Tree":{"id":"Tree","path":"Tree","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}}},"constructInfo":{"fqn":"aws-cdk-lib.App","version":"0.0.0"}}} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.ts index 0a77ff0bbe4ed..b9a24f5336e97 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-pyspark-streaming.ts @@ -52,6 +52,7 @@ new glue.PySparkStreamingJob(stack, 'OverridePySparkStreamingJob', { key: 'value', }, jobRunQueuingEnabled: true, + notifyDelayAfter: cdk.Duration.minutes(5), }); new integ.IntegTest(app, 'aws-glue-job-pyspark-streaming-integ-test', { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json index b8824f41b535f..53309de68ec4f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,7 @@ } } }, - "0222cd48958b3146476a2504ebb4e4e21ba80d0d900991f191243efc281b0da2": { + "fb32392cc12dc489bedf064fdbe024f89cad3dca60acce4a968ccbd4e4a86538": { "source": { "path": "aws-glue-job-python-shell.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "0222cd48958b3146476a2504ebb4e4e21ba80d0d900991f191243efc281b0da2.json", + "objectKey": "fb32392cc12dc489bedf064fdbe024f89cad3dca60acce4a968ccbd4e4a86538.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.template.json index a6f1e42af9768..c151a3ad7afdb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.template.json @@ -197,6 +197,9 @@ "MaxCapacity": 1, "MaxRetries": 0, "Name": "My Python 3.9 Shell Job", + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, "Role": { "Fn::GetAtt": [ "IAMServiceRole61C662C4", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json index 7700c57fa89f4..cfc9a3c901f8f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/cdk.out index c6e612584e352..1e02a2deb191b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"38.0.1"} \ No newline at end of file +{"version":"40.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/integ.json index 3d9776563e990..aae2e45e870fb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "testCases": { "aws-glue-job-python-shell-integ-test/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/manifest.json index 761a0e001bf2f..22fbb4c9388ad 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "artifacts": { "aws-glue-job-python-shell.assets": { "type": "cdk:asset-manifest", @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/0222cd48958b3146476a2504ebb4e4e21ba80d0d900991f191243efc281b0da2.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/fb32392cc12dc489bedf064fdbe024f89cad3dca60acce4a968ccbd4e4a86538.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -34,30 +34,156 @@ "aws-glue-job-python-shell.assets" ], "metadata": { + "/aws-glue-job-python-shell/IAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": { + "assumedBy": { + "principalAccount": "*", + "assumeRoleAction": "*" + }, + "managedPolicies": [ + { + "managedPolicyArn": "*" + } + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + } + ], + "/aws-glue-job-python-shell/IAMServiceRole/ImportIAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-python-shell/IAMServiceRole/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRole61C662C4" } ], + "/aws-glue-job-python-shell/IAMServiceRole/DefaultPolicy": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + } + ], "/aws-glue-job-python-shell/IAMServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRoleDefaultPolicy379D1A0E" } ], + "/aws-glue-job-python-shell/BasicShellJob39": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-python-shell/BasicShellJob39/Resource": [ { "type": "aws:cdk:logicalId", "data": "BasicShellJob39F2E7D12A" } ], + "/aws-glue-job-python-shell/BasicShellJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-python-shell/BasicShellJob/Resource": [ { "type": "aws:cdk:logicalId", "data": "BasicShellJobC7D0761E" } ], + "/aws-glue-job-python-shell/DetailedShellJob39": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-python-shell/DetailedShellJob39/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/tree.json index 87fd451845726..0053ff9b27b3c 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/tree.json @@ -1,433 +1 @@ -{ - "version": "tree-0.1", - "tree": { - "id": "App", - "path": "", - "children": { - "aws-glue-job-python-shell": { - "id": "aws-glue-job-python-shell", - "path": "aws-glue-job-python-shell", - "children": { - "IAMServiceRole": { - "id": "IAMServiceRole", - "path": "aws-glue-job-python-shell/IAMServiceRole", - "children": { - "ImportIAMServiceRole": { - "id": "ImportIAMServiceRole", - "path": "aws-glue-job-python-shell/IAMServiceRole/ImportIAMServiceRole", - "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-python-shell/IAMServiceRole/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Role", - "aws:cdk:cloudformation:props": { - "assumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "managedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSGlueServiceRole" - ] - ] - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" - } - }, - "DefaultPolicy": { - "id": "DefaultPolicy", - "path": "aws-glue-job-python-shell/IAMServiceRole/DefaultPolicy", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-python-shell/IAMServiceRole/DefaultPolicy/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Policy", - "aws:cdk:cloudformation:props": { - "policyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - } - ] - ] - } - ] - } - ], - "Version": "2012-10-17" - }, - "policyName": "IAMServiceRoleDefaultPolicy379D1A0E", - "roles": [ - { - "Ref": "IAMServiceRole61C662C4" - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" - } - }, - "BasicShellJob39": { - "id": "BasicShellJob39", - "path": "aws-glue-job-python-shell/BasicShellJob39", - "children": { - "Code2907ea7be4a583708cfffc21b3df1dfa": { - "id": "Code2907ea7be4a583708cfffc21b3df1dfa", - "path": "aws-glue-job-python-shell/BasicShellJob39/Code2907ea7be4a583708cfffc21b3df1dfa", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job-python-shell/BasicShellJob39/Code2907ea7be4a583708cfffc21b3df1dfa/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job-python-shell/BasicShellJob39/Code2907ea7be4a583708cfffc21b3df1dfa/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-python-shell/BasicShellJob39/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "pythonshell", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3.9" - }, - "defaultArguments": { - "--job-language": "python", - "library-set": "analytics", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true" - }, - "glueVersion": "3.0", - "jobRunQueuingEnabled": false, - "maxCapacity": 0.0625, - "maxRetries": 0, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PythonShellJob", - "version": "0.0.0" - } - }, - "BasicShellJob": { - "id": "BasicShellJob", - "path": "aws-glue-job-python-shell/BasicShellJob", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-python-shell/BasicShellJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "pythonshell", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3" - }, - "defaultArguments": { - "--job-language": "python", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true" - }, - "glueVersion": "1.0", - "jobRunQueuingEnabled": false, - "maxCapacity": 0.0625, - "maxRetries": 0, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PythonShellJob", - "version": "0.0.0" - } - }, - "DetailedShellJob39": { - "id": "DetailedShellJob39", - "path": "aws-glue-job-python-shell/DetailedShellJob39", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-python-shell/DetailedShellJob39/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "pythonshell", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" - ] - ] - }, - "pythonVersion": "3.9" - }, - "defaultArguments": { - "--job-language": "python", - "library-set": "analytics", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true", - "arg1": "value1", - "arg2": "value2" - }, - "description": "My detailed Python 3.9 Shell Job", - "glueVersion": "3.0", - "jobRunQueuingEnabled": true, - "maxCapacity": 1, - "maxRetries": 0, - "name": "My Python 3.9 Shell Job", - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "tags": { - "key": "value" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.PythonShellJob", - "version": "0.0.0" - } - }, - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-python-shell/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-python-shell/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - }, - "aws-glue-job-python-shell-integ-test": { - "id": "aws-glue-job-python-shell-integ-test", - "path": "aws-glue-job-python-shell-integ-test", - "children": { - "DefaultTest": { - "id": "DefaultTest", - "path": "aws-glue-job-python-shell-integ-test/DefaultTest", - "children": { - "Default": { - "id": "Default", - "path": "aws-glue-job-python-shell-integ-test/DefaultTest/Default", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - }, - "DeployAssert": { - "id": "DeployAssert", - "path": "aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert", - "children": { - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", - "version": "0.0.0" - } - }, - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.App", - "version": "0.0.0" - } - } -} \ No newline at end of file +{"version":"tree-0.1","tree":{"id":"App","path":"","children":{"aws-glue-job-python-shell":{"id":"aws-glue-job-python-shell","path":"aws-glue-job-python-shell","children":{"IAMServiceRole":{"id":"IAMServiceRole","path":"aws-glue-job-python-shell/IAMServiceRole","children":{"ImportIAMServiceRole":{"id":"ImportIAMServiceRole","path":"aws-glue-job-python-shell/IAMServiceRole/ImportIAMServiceRole","constructInfo":{"fqn":"aws-cdk-lib.Resource","version":"0.0.0","metadata":["*"]}},"Resource":{"id":"Resource","path":"aws-glue-job-python-shell/IAMServiceRole/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Role","aws:cdk:cloudformation:props":{"assumeRolePolicyDocument":{"Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":"glue.amazonaws.com"}}],"Version":"2012-10-17"},"managedPolicyArns":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":iam::aws:policy/service-role/AWSGlueServiceRole"]]}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnRole","version":"0.0.0"}},"DefaultPolicy":{"id":"DefaultPolicy","path":"aws-glue-job-python-shell/IAMServiceRole/DefaultPolicy","children":{"Resource":{"id":"Resource","path":"aws-glue-job-python-shell/IAMServiceRole/DefaultPolicy/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Policy","aws:cdk:cloudformation:props":{"policyDocument":{"Statement":[{"Action":["s3:GetBucket*","s3:GetObject*","s3:List*"],"Effect":"Allow","Resource":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/*"]]},{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}]]}]}],"Version":"2012-10-17"},"policyName":"IAMServiceRoleDefaultPolicy379D1A0E","roles":[{"Ref":"IAMServiceRole61C662C4"}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnPolicy","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Policy","version":"0.0.0","metadata":["*",{"attachToRole":["*"]},{"attachToRole":["*"]},{"addStatements":[{}]},{"addStatements":[{}]},{"addStatements":[{}]}]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Role","version":"0.0.0","metadata":[{"assumedBy":{"principalAccount":"*","assumeRoleAction":"*"},"managedPolicies":[{"managedPolicyArn":"*"}]},{"addToPrincipalPolicy":[{}]},{"attachInlinePolicy":["*"]},{"attachInlinePolicy":["*"]},{"addToPrincipalPolicy":[{}]},{"addToPrincipalPolicy":[{}]}]}},"BasicShellJob39":{"id":"BasicShellJob39","path":"aws-glue-job-python-shell/BasicShellJob39","children":{"Code1de73119980d143bf808e10735354e19":{"id":"Code1de73119980d143bf808e10735354e19","path":"aws-glue-job-python-shell/BasicShellJob39/Code1de73119980d143bf808e10735354e19","children":{"Stage":{"id":"Stage","path":"aws-glue-job-python-shell/BasicShellJob39/Code1de73119980d143bf808e10735354e19/Stage","constructInfo":{"fqn":"aws-cdk-lib.AssetStaging","version":"0.0.0"}},"AssetBucket":{"id":"AssetBucket","path":"aws-glue-job-python-shell/BasicShellJob39/Code1de73119980d143bf808e10735354e19/AssetBucket","constructInfo":{"fqn":"aws-cdk-lib.aws_s3.BucketBase","version":"0.0.0","metadata":[]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_s3_assets.Asset","version":"0.0.0"}},"Resource":{"id":"Resource","path":"aws-glue-job-python-shell/BasicShellJob39/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"pythonshell","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3.9"},"defaultArguments":{"--job-language":"python","library-set":"analytics","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true"},"glueVersion":"3.0","jobRunQueuingEnabled":false,"maxCapacity":0.0625,"maxRetries":0,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PythonShellJob","version":"0.0.0","metadata":["*"]}},"BasicShellJob":{"id":"BasicShellJob","path":"aws-glue-job-python-shell/BasicShellJob","children":{"Resource":{"id":"Resource","path":"aws-glue-job-python-shell/BasicShellJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"pythonshell","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3"},"defaultArguments":{"--job-language":"python","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true"},"glueVersion":"1.0","jobRunQueuingEnabled":false,"maxCapacity":0.0625,"maxRetries":0,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PythonShellJob","version":"0.0.0","metadata":["*"]}},"DetailedShellJob39":{"id":"DetailedShellJob39","path":"aws-glue-job-python-shell/DetailedShellJob39","children":{"Resource":{"id":"Resource","path":"aws-glue-job-python-shell/DetailedShellJob39/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"pythonshell","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"]]},"pythonVersion":"3.9"},"defaultArguments":{"--job-language":"python","library-set":"analytics","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true","arg1":"value1","arg2":"value2"},"description":"My detailed Python 3.9 Shell Job","glueVersion":"3.0","jobRunQueuingEnabled":true,"maxCapacity":1,"maxRetries":0,"name":"My Python 3.9 Shell Job","notificationProperty":{"notifyDelayAfter":5},"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"tags":{"key":"value"}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.PythonShellJob","version":"0.0.0","metadata":["*"]}},"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-python-shell/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-python-shell/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}},"aws-glue-job-python-shell-integ-test":{"id":"aws-glue-job-python-shell-integ-test","path":"aws-glue-job-python-shell-integ-test","children":{"DefaultTest":{"id":"DefaultTest","path":"aws-glue-job-python-shell-integ-test/DefaultTest","children":{"Default":{"id":"Default","path":"aws-glue-job-python-shell-integ-test/DefaultTest/Default","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}},"DeployAssert":{"id":"DeployAssert","path":"aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert","children":{"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTestCase","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTest","version":"0.0.0"}},"Tree":{"id":"Tree","path":"Tree","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}}},"constructInfo":{"fqn":"aws-cdk-lib.App","version":"0.0.0"}}} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.ts index 74cbbe207f892..07ea4e483005e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.ts @@ -56,6 +56,7 @@ new glue.PythonShellJob(stack, 'DetailedShellJob39', { key: 'value', }, jobRunQueuingEnabled: true, + notifyDelayAfter: cdk.Duration.minutes(5), }); new integ.IntegTest(app, 'aws-glue-job-python-shell-integ-test', { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/aws-glue-job-scalaspark-etl.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/aws-glue-job-scalaspark-etl.assets.json index 3dd9eff7a7a39..75357b4c96aea 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/aws-glue-job-scalaspark-etl.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/aws-glue-job-scalaspark-etl.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602": { "source": { @@ -14,7 +14,7 @@ } } }, - "0fb91515f8aa7d3f3ba56c07df77fa5be2fb94dc9ded6c60f3ec6740fc440ffe": { + "34d909e26276092df65a018edaae6bba157cbaf92e684cc5635d24f1a878e0c1": { "source": { "path": "aws-glue-job-scalaspark-etl.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "0fb91515f8aa7d3f3ba56c07df77fa5be2fb94dc9ded6c60f3ec6740fc440ffe.json", + "objectKey": "34d909e26276092df65a018edaae6bba157cbaf92e684cc5635d24f1a878e0c1.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/aws-glue-job-scalaspark-etl.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/aws-glue-job-scalaspark-etl.template.json index e608f9b6d5684..5c99b5b034a30 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/aws-glue-job-scalaspark-etl.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/aws-glue-job-scalaspark-etl.template.json @@ -157,6 +157,9 @@ "JobRunQueuingEnabled": true, "MaxRetries": 0, "Name": "Optional Override ScalaSpark ETL Job", + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, "NumberOfWorkers": 20, "Role": { "Fn::GetAtt": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/awsgluejobscalasparketlintegtestDefaultTestDeployAssertCA9A8121.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/awsgluejobscalasparketlintegtestDefaultTestDeployAssertCA9A8121.assets.json index 7c8b6b37d6df4..a8eaabf630772 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/awsgluejobscalasparketlintegtestDefaultTestDeployAssertCA9A8121.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/awsgluejobscalasparketlintegtestDefaultTestDeployAssertCA9A8121.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/cdk.out index c6e612584e352..1e02a2deb191b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"38.0.1"} \ No newline at end of file +{"version":"40.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/integ.json index 27d64a8d01bda..c4d358979210f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "testCases": { "aws-glue-job-scalaspark-etl-integ-test/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/manifest.json index b62303d9bc984..bc13ae6153253 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "artifacts": { "aws-glue-job-scalaspark-etl.assets": { "type": "cdk:asset-manifest", @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/0fb91515f8aa7d3f3ba56c07df77fa5be2fb94dc9ded6c60f3ec6740fc440ffe.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/34d909e26276092df65a018edaae6bba157cbaf92e684cc5635d24f1a878e0c1.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -34,24 +34,136 @@ "aws-glue-job-scalaspark-etl.assets" ], "metadata": { + "/aws-glue-job-scalaspark-etl/IAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": { + "assumedBy": { + "principalAccount": "*", + "assumeRoleAction": "*" + }, + "managedPolicies": [ + { + "managedPolicyArn": "*" + } + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + } + ], + "/aws-glue-job-scalaspark-etl/IAMServiceRole/ImportIAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalaspark-etl/IAMServiceRole/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRole61C662C4" } ], + "/aws-glue-job-scalaspark-etl/IAMServiceRole/DefaultPolicy": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + } + ], "/aws-glue-job-scalaspark-etl/IAMServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRoleDefaultPolicy379D1A0E" } ], + "/aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Resource": [ { "type": "aws:cdk:logicalId", "data": "BasicScalaSparkETLJob5F894E39" } ], + "/aws-glue-job-scalaspark-etl/OverrideScalaSparkETLJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalaspark-etl/OverrideScalaSparkETLJob/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/tree.json index ef6c9d1ff3900..464de4b9b6d23 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.js.snapshot/tree.json @@ -1,378 +1 @@ -{ - "version": "tree-0.1", - "tree": { - "id": "App", - "path": "", - "children": { - "aws-glue-job-scalaspark-etl": { - "id": "aws-glue-job-scalaspark-etl", - "path": "aws-glue-job-scalaspark-etl", - "children": { - "IAMServiceRole": { - "id": "IAMServiceRole", - "path": "aws-glue-job-scalaspark-etl/IAMServiceRole", - "children": { - "ImportIAMServiceRole": { - "id": "ImportIAMServiceRole", - "path": "aws-glue-job-scalaspark-etl/IAMServiceRole/ImportIAMServiceRole", - "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalaspark-etl/IAMServiceRole/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Role", - "aws:cdk:cloudformation:props": { - "assumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "managedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSGlueServiceRole" - ] - ] - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" - } - }, - "DefaultPolicy": { - "id": "DefaultPolicy", - "path": "aws-glue-job-scalaspark-etl/IAMServiceRole/DefaultPolicy", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalaspark-etl/IAMServiceRole/DefaultPolicy/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Policy", - "aws:cdk:cloudformation:props": { - "policyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - } - ] - ] - } - ] - } - ], - "Version": "2012-10-17" - }, - "policyName": "IAMServiceRoleDefaultPolicy379D1A0E", - "roles": [ - { - "Ref": "IAMServiceRole61C662C4" - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" - } - }, - "BasicScalaSparkETLJob": { - "id": "BasicScalaSparkETLJob", - "path": "aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob", - "children": { - "Codeb58a68516710fd95a65c427a7e567405": { - "id": "Codeb58a68516710fd95a65c427a7e567405", - "path": "aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Codeb58a68516710fd95a65c427a7e567405", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Codeb58a68516710fd95a65c427a7e567405/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Codeb58a68516710fd95a65c427a7e567405/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "glueetl", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar" - ] - ] - } - }, - "defaultArguments": { - "--job-language": "scala", - "--class": "com.example.HelloWorld", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true" - }, - "glueVersion": "4.0", - "jobRunQueuingEnabled": false, - "numberOfWorkers": 10, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkEtlJob", - "version": "0.0.0" - } - }, - "OverrideScalaSparkETLJob": { - "id": "OverrideScalaSparkETLJob", - "path": "aws-glue-job-scalaspark-etl/OverrideScalaSparkETLJob", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalaspark-etl/OverrideScalaSparkETLJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "glueetl", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar" - ] - ] - } - }, - "defaultArguments": { - "--job-language": "scala", - "--class": "com.example.HelloWorld", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true", - "arg1": "value1", - "arg2": "value2" - }, - "description": "Optional Override ScalaSpark ETL Job", - "glueVersion": "3.0", - "jobRunQueuingEnabled": true, - "maxRetries": 0, - "name": "Optional Override ScalaSpark ETL Job", - "numberOfWorkers": 20, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "tags": { - "key": "value" - }, - "timeout": 15, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkEtlJob", - "version": "0.0.0" - } - }, - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-scalaspark-etl/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-scalaspark-etl/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - }, - "aws-glue-job-scalaspark-etl-integ-test": { - "id": "aws-glue-job-scalaspark-etl-integ-test", - "path": "aws-glue-job-scalaspark-etl-integ-test", - "children": { - "DefaultTest": { - "id": "DefaultTest", - "path": "aws-glue-job-scalaspark-etl-integ-test/DefaultTest", - "children": { - "Default": { - "id": "Default", - "path": "aws-glue-job-scalaspark-etl-integ-test/DefaultTest/Default", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - }, - "DeployAssert": { - "id": "DeployAssert", - "path": "aws-glue-job-scalaspark-etl-integ-test/DefaultTest/DeployAssert", - "children": { - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-scalaspark-etl-integ-test/DefaultTest/DeployAssert/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-scalaspark-etl-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", - "version": "0.0.0" - } - }, - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.App", - "version": "0.0.0" - } - } -} \ No newline at end of file +{"version":"tree-0.1","tree":{"id":"App","path":"","children":{"aws-glue-job-scalaspark-etl":{"id":"aws-glue-job-scalaspark-etl","path":"aws-glue-job-scalaspark-etl","children":{"IAMServiceRole":{"id":"IAMServiceRole","path":"aws-glue-job-scalaspark-etl/IAMServiceRole","children":{"ImportIAMServiceRole":{"id":"ImportIAMServiceRole","path":"aws-glue-job-scalaspark-etl/IAMServiceRole/ImportIAMServiceRole","constructInfo":{"fqn":"aws-cdk-lib.Resource","version":"0.0.0","metadata":["*"]}},"Resource":{"id":"Resource","path":"aws-glue-job-scalaspark-etl/IAMServiceRole/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Role","aws:cdk:cloudformation:props":{"assumeRolePolicyDocument":{"Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":"glue.amazonaws.com"}}],"Version":"2012-10-17"},"managedPolicyArns":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":iam::aws:policy/service-role/AWSGlueServiceRole"]]}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnRole","version":"0.0.0"}},"DefaultPolicy":{"id":"DefaultPolicy","path":"aws-glue-job-scalaspark-etl/IAMServiceRole/DefaultPolicy","children":{"Resource":{"id":"Resource","path":"aws-glue-job-scalaspark-etl/IAMServiceRole/DefaultPolicy/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Policy","aws:cdk:cloudformation:props":{"policyDocument":{"Statement":[{"Action":["s3:GetBucket*","s3:GetObject*","s3:List*"],"Effect":"Allow","Resource":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/*"]]},{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}]]}]}],"Version":"2012-10-17"},"policyName":"IAMServiceRoleDefaultPolicy379D1A0E","roles":[{"Ref":"IAMServiceRole61C662C4"}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnPolicy","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Policy","version":"0.0.0","metadata":["*",{"attachToRole":["*"]},{"attachToRole":["*"]},{"addStatements":[{}]},{"addStatements":[{}]}]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Role","version":"0.0.0","metadata":[{"assumedBy":{"principalAccount":"*","assumeRoleAction":"*"},"managedPolicies":[{"managedPolicyArn":"*"}]},{"addToPrincipalPolicy":[{}]},{"attachInlinePolicy":["*"]},{"attachInlinePolicy":["*"]},{"addToPrincipalPolicy":[{}]}]}},"BasicScalaSparkETLJob":{"id":"BasicScalaSparkETLJob","path":"aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob","children":{"Codeb1ad5b8a47fabefcad19ad781568f6a7":{"id":"Codeb1ad5b8a47fabefcad19ad781568f6a7","path":"aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Codeb1ad5b8a47fabefcad19ad781568f6a7","children":{"Stage":{"id":"Stage","path":"aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Codeb1ad5b8a47fabefcad19ad781568f6a7/Stage","constructInfo":{"fqn":"aws-cdk-lib.AssetStaging","version":"0.0.0"}},"AssetBucket":{"id":"AssetBucket","path":"aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Codeb1ad5b8a47fabefcad19ad781568f6a7/AssetBucket","constructInfo":{"fqn":"aws-cdk-lib.aws_s3.BucketBase","version":"0.0.0","metadata":[]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_s3_assets.Asset","version":"0.0.0"}},"Resource":{"id":"Resource","path":"aws-glue-job-scalaspark-etl/BasicScalaSparkETLJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"glueetl","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar"]]}},"defaultArguments":{"--job-language":"scala","--class":"com.example.HelloWorld","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true"},"glueVersion":"4.0","jobRunQueuingEnabled":false,"numberOfWorkers":10,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.ScalaSparkEtlJob","version":"0.0.0","metadata":["*","*"]}},"OverrideScalaSparkETLJob":{"id":"OverrideScalaSparkETLJob","path":"aws-glue-job-scalaspark-etl/OverrideScalaSparkETLJob","children":{"Resource":{"id":"Resource","path":"aws-glue-job-scalaspark-etl/OverrideScalaSparkETLJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"glueetl","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar"]]}},"defaultArguments":{"--job-language":"scala","--class":"com.example.HelloWorld","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true","arg1":"value1","arg2":"value2"},"description":"Optional Override ScalaSpark ETL Job","glueVersion":"3.0","jobRunQueuingEnabled":true,"maxRetries":0,"name":"Optional Override ScalaSpark ETL Job","notificationProperty":{"notifyDelayAfter":5},"numberOfWorkers":20,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"tags":{"key":"value"},"timeout":15,"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.ScalaSparkEtlJob","version":"0.0.0","metadata":["*","*"]}},"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-scalaspark-etl/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-scalaspark-etl/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}},"aws-glue-job-scalaspark-etl-integ-test":{"id":"aws-glue-job-scalaspark-etl-integ-test","path":"aws-glue-job-scalaspark-etl-integ-test","children":{"DefaultTest":{"id":"DefaultTest","path":"aws-glue-job-scalaspark-etl-integ-test/DefaultTest","children":{"Default":{"id":"Default","path":"aws-glue-job-scalaspark-etl-integ-test/DefaultTest/Default","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}},"DeployAssert":{"id":"DeployAssert","path":"aws-glue-job-scalaspark-etl-integ-test/DefaultTest/DeployAssert","children":{"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-scalaspark-etl-integ-test/DefaultTest/DeployAssert/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-scalaspark-etl-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTestCase","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTest","version":"0.0.0"}},"Tree":{"id":"Tree","path":"Tree","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}}},"constructInfo":{"fqn":"aws-cdk-lib.App","version":"0.0.0"}}} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.ts index 914d11f2cab4b..65a034e319620 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-etl.ts @@ -55,6 +55,7 @@ new glue.ScalaSparkEtlJob(stack, 'OverrideScalaSparkETLJob', { key: 'value', }, jobRunQueuingEnabled: true, + notifyDelayAfter: cdk.Duration.minutes(5), }); new integ.IntegTest(app, 'aws-glue-job-scalaspark-etl-integ-test', { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/aws-glue-job-scalasparkflex-etl.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/aws-glue-job-scalasparkflex-etl.assets.json index 6850e99cb6703..2a3dee788cedb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/aws-glue-job-scalasparkflex-etl.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/aws-glue-job-scalasparkflex-etl.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602": { "source": { @@ -14,7 +14,7 @@ } } }, - "7c0d31e02ab980459f90299032315caaa05e6b665bdcd686783bdb95a6ff474f": { + "687a4f60a8510878e59000a144782aca8ac5b00974359c3b4baf3666a7921591": { "source": { "path": "aws-glue-job-scalasparkflex-etl.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "7c0d31e02ab980459f90299032315caaa05e6b665bdcd686783bdb95a6ff474f.json", + "objectKey": "687a4f60a8510878e59000a144782aca8ac5b00974359c3b4baf3666a7921591.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/aws-glue-job-scalasparkflex-etl.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/aws-glue-job-scalasparkflex-etl.template.json index 77f796377dad5..ee4ce797657e9 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/aws-glue-job-scalasparkflex-etl.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/aws-glue-job-scalasparkflex-etl.template.json @@ -158,6 +158,9 @@ "GlueVersion": "3.0", "JobRunQueuingEnabled": false, "Name": "Optional Override ScalaSpark Flex Etl Job", + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, "NumberOfWorkers": 20, "Role": { "Fn::GetAtt": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/awsgluejobscalasparkflexetlintegtestDefaultTestDeployAssert8009E6FC.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/awsgluejobscalasparkflexetlintegtestDefaultTestDeployAssert8009E6FC.assets.json index a35cda0283860..42c2e19f63485 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/awsgluejobscalasparkflexetlintegtestDefaultTestDeployAssert8009E6FC.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/awsgluejobscalasparkflexetlintegtestDefaultTestDeployAssert8009E6FC.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/cdk.out index c6e612584e352..1e02a2deb191b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"38.0.1"} \ No newline at end of file +{"version":"40.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/integ.json index 72bae81c5765a..1db6b584437d2 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "testCases": { "aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/manifest.json index f4043c0b87d19..a57aca76fb5bf 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "artifacts": { "aws-glue-job-scalasparkflex-etl.assets": { "type": "cdk:asset-manifest", @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/7c0d31e02ab980459f90299032315caaa05e6b665bdcd686783bdb95a6ff474f.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/687a4f60a8510878e59000a144782aca8ac5b00974359c3b4baf3666a7921591.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -34,24 +34,136 @@ "aws-glue-job-scalasparkflex-etl.assets" ], "metadata": { + "/aws-glue-job-scalasparkflex-etl/IAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": { + "assumedBy": { + "principalAccount": "*", + "assumeRoleAction": "*" + }, + "managedPolicies": [ + { + "managedPolicyArn": "*" + } + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + } + ], + "/aws-glue-job-scalasparkflex-etl/IAMServiceRole/ImportIAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalasparkflex-etl/IAMServiceRole/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRole61C662C4" } ], + "/aws-glue-job-scalasparkflex-etl/IAMServiceRole/DefaultPolicy": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + } + ], "/aws-glue-job-scalasparkflex-etl/IAMServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRoleDefaultPolicy379D1A0E" } ], + "/aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Resource": [ { "type": "aws:cdk:logicalId", "data": "BasicScalaSparkFlexEtlJobF8FD9EFB" } ], + "/aws-glue-job-scalasparkflex-etl/OverrideScalaSparkFlexEtlJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalasparkflex-etl/OverrideScalaSparkFlexEtlJob/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/tree.json index 6053a5c5c342d..22c3cabe314a9 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.js.snapshot/tree.json @@ -1,379 +1 @@ -{ - "version": "tree-0.1", - "tree": { - "id": "App", - "path": "", - "children": { - "aws-glue-job-scalasparkflex-etl": { - "id": "aws-glue-job-scalasparkflex-etl", - "path": "aws-glue-job-scalasparkflex-etl", - "children": { - "IAMServiceRole": { - "id": "IAMServiceRole", - "path": "aws-glue-job-scalasparkflex-etl/IAMServiceRole", - "children": { - "ImportIAMServiceRole": { - "id": "ImportIAMServiceRole", - "path": "aws-glue-job-scalasparkflex-etl/IAMServiceRole/ImportIAMServiceRole", - "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalasparkflex-etl/IAMServiceRole/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Role", - "aws:cdk:cloudformation:props": { - "assumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "managedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSGlueServiceRole" - ] - ] - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" - } - }, - "DefaultPolicy": { - "id": "DefaultPolicy", - "path": "aws-glue-job-scalasparkflex-etl/IAMServiceRole/DefaultPolicy", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalasparkflex-etl/IAMServiceRole/DefaultPolicy/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Policy", - "aws:cdk:cloudformation:props": { - "policyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - } - ] - ] - } - ] - } - ], - "Version": "2012-10-17" - }, - "policyName": "IAMServiceRoleDefaultPolicy379D1A0E", - "roles": [ - { - "Ref": "IAMServiceRole61C662C4" - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" - } - }, - "BasicScalaSparkFlexEtlJob": { - "id": "BasicScalaSparkFlexEtlJob", - "path": "aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob", - "children": { - "Codeb58a68516710fd95a65c427a7e567405": { - "id": "Codeb58a68516710fd95a65c427a7e567405", - "path": "aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Codeb58a68516710fd95a65c427a7e567405", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Codeb58a68516710fd95a65c427a7e567405/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Codeb58a68516710fd95a65c427a7e567405/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "glueetl", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar" - ] - ] - } - }, - "defaultArguments": { - "--job-language": "scala", - "--class": "com.example.HelloWorld", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true" - }, - "executionClass": "FLEX", - "glueVersion": "3.0", - "jobRunQueuingEnabled": false, - "numberOfWorkers": 10, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkFlexEtlJob", - "version": "0.0.0" - } - }, - "OverrideScalaSparkFlexEtlJob": { - "id": "OverrideScalaSparkFlexEtlJob", - "path": "aws-glue-job-scalasparkflex-etl/OverrideScalaSparkFlexEtlJob", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalasparkflex-etl/OverrideScalaSparkFlexEtlJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "glueetl", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar" - ] - ] - } - }, - "defaultArguments": { - "--job-language": "scala", - "--class": "com.example.HelloWorld", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true", - "arg1": "value1", - "arg2": "value2" - }, - "description": "Optional Override ScalaSpark Flex Etl Job", - "executionClass": "FLEX", - "glueVersion": "3.0", - "jobRunQueuingEnabled": false, - "name": "Optional Override ScalaSpark Flex Etl Job", - "numberOfWorkers": 20, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "tags": { - "key": "value" - }, - "timeout": 15, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkFlexEtlJob", - "version": "0.0.0" - } - }, - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-scalasparkflex-etl/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-scalasparkflex-etl/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - }, - "aws-glue-job-scalasparkflex-etl-integ-test": { - "id": "aws-glue-job-scalasparkflex-etl-integ-test", - "path": "aws-glue-job-scalasparkflex-etl-integ-test", - "children": { - "DefaultTest": { - "id": "DefaultTest", - "path": "aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest", - "children": { - "Default": { - "id": "Default", - "path": "aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest/Default", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - }, - "DeployAssert": { - "id": "DeployAssert", - "path": "aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest/DeployAssert", - "children": { - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest/DeployAssert/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", - "version": "0.0.0" - } - }, - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.App", - "version": "0.0.0" - } - } -} \ No newline at end of file +{"version":"tree-0.1","tree":{"id":"App","path":"","children":{"aws-glue-job-scalasparkflex-etl":{"id":"aws-glue-job-scalasparkflex-etl","path":"aws-glue-job-scalasparkflex-etl","children":{"IAMServiceRole":{"id":"IAMServiceRole","path":"aws-glue-job-scalasparkflex-etl/IAMServiceRole","children":{"ImportIAMServiceRole":{"id":"ImportIAMServiceRole","path":"aws-glue-job-scalasparkflex-etl/IAMServiceRole/ImportIAMServiceRole","constructInfo":{"fqn":"aws-cdk-lib.Resource","version":"0.0.0","metadata":["*"]}},"Resource":{"id":"Resource","path":"aws-glue-job-scalasparkflex-etl/IAMServiceRole/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Role","aws:cdk:cloudformation:props":{"assumeRolePolicyDocument":{"Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":"glue.amazonaws.com"}}],"Version":"2012-10-17"},"managedPolicyArns":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":iam::aws:policy/service-role/AWSGlueServiceRole"]]}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnRole","version":"0.0.0"}},"DefaultPolicy":{"id":"DefaultPolicy","path":"aws-glue-job-scalasparkflex-etl/IAMServiceRole/DefaultPolicy","children":{"Resource":{"id":"Resource","path":"aws-glue-job-scalasparkflex-etl/IAMServiceRole/DefaultPolicy/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Policy","aws:cdk:cloudformation:props":{"policyDocument":{"Statement":[{"Action":["s3:GetBucket*","s3:GetObject*","s3:List*"],"Effect":"Allow","Resource":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/*"]]},{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}]]}]}],"Version":"2012-10-17"},"policyName":"IAMServiceRoleDefaultPolicy379D1A0E","roles":[{"Ref":"IAMServiceRole61C662C4"}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnPolicy","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Policy","version":"0.0.0","metadata":["*",{"attachToRole":["*"]},{"attachToRole":["*"]},{"addStatements":[{}]},{"addStatements":[{}]}]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Role","version":"0.0.0","metadata":[{"assumedBy":{"principalAccount":"*","assumeRoleAction":"*"},"managedPolicies":[{"managedPolicyArn":"*"}]},{"addToPrincipalPolicy":[{}]},{"attachInlinePolicy":["*"]},{"attachInlinePolicy":["*"]},{"addToPrincipalPolicy":[{}]}]}},"BasicScalaSparkFlexEtlJob":{"id":"BasicScalaSparkFlexEtlJob","path":"aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob","children":{"Codeb1ad5b8a47fabefcad19ad781568f6a7":{"id":"Codeb1ad5b8a47fabefcad19ad781568f6a7","path":"aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Codeb1ad5b8a47fabefcad19ad781568f6a7","children":{"Stage":{"id":"Stage","path":"aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Codeb1ad5b8a47fabefcad19ad781568f6a7/Stage","constructInfo":{"fqn":"aws-cdk-lib.AssetStaging","version":"0.0.0"}},"AssetBucket":{"id":"AssetBucket","path":"aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Codeb1ad5b8a47fabefcad19ad781568f6a7/AssetBucket","constructInfo":{"fqn":"aws-cdk-lib.aws_s3.BucketBase","version":"0.0.0","metadata":[]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_s3_assets.Asset","version":"0.0.0"}},"Resource":{"id":"Resource","path":"aws-glue-job-scalasparkflex-etl/BasicScalaSparkFlexEtlJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"glueetl","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar"]]}},"defaultArguments":{"--job-language":"scala","--class":"com.example.HelloWorld","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true"},"executionClass":"FLEX","glueVersion":"3.0","jobRunQueuingEnabled":false,"numberOfWorkers":10,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.ScalaSparkFlexEtlJob","version":"0.0.0","metadata":["*","*"]}},"OverrideScalaSparkFlexEtlJob":{"id":"OverrideScalaSparkFlexEtlJob","path":"aws-glue-job-scalasparkflex-etl/OverrideScalaSparkFlexEtlJob","children":{"Resource":{"id":"Resource","path":"aws-glue-job-scalasparkflex-etl/OverrideScalaSparkFlexEtlJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"glueetl","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar"]]}},"defaultArguments":{"--job-language":"scala","--class":"com.example.HelloWorld","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true","arg1":"value1","arg2":"value2"},"description":"Optional Override ScalaSpark Flex Etl Job","executionClass":"FLEX","glueVersion":"3.0","jobRunQueuingEnabled":false,"name":"Optional Override ScalaSpark Flex Etl Job","notificationProperty":{"notifyDelayAfter":5},"numberOfWorkers":20,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"tags":{"key":"value"},"timeout":15,"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.ScalaSparkFlexEtlJob","version":"0.0.0","metadata":["*","*"]}},"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-scalasparkflex-etl/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-scalasparkflex-etl/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}},"aws-glue-job-scalasparkflex-etl-integ-test":{"id":"aws-glue-job-scalasparkflex-etl-integ-test","path":"aws-glue-job-scalasparkflex-etl-integ-test","children":{"DefaultTest":{"id":"DefaultTest","path":"aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest","children":{"Default":{"id":"Default","path":"aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest/Default","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}},"DeployAssert":{"id":"DeployAssert","path":"aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest/DeployAssert","children":{"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest/DeployAssert/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-scalasparkflex-etl-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTestCase","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTest","version":"0.0.0"}},"Tree":{"id":"Tree","path":"Tree","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}}},"constructInfo":{"fqn":"aws-cdk-lib.App","version":"0.0.0"}}} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.ts index 6ecb66ded6352..377f271e4bf90 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-flex-etl.ts @@ -54,6 +54,7 @@ new glue.ScalaSparkFlexEtlJob(stack, 'OverrideScalaSparkFlexEtlJob', { tags: { key: 'value', }, + notifyDelayAfter: cdk.Duration.minutes(5), }); new integ.IntegTest(app, 'aws-glue-job-scalasparkflex-etl-integ-test', { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/aws-glue-job-scalaspark-streaming.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/aws-glue-job-scalaspark-streaming.assets.json index 8d1a11329b205..686bbe761d239 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/aws-glue-job-scalaspark-streaming.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/aws-glue-job-scalaspark-streaming.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602": { "source": { @@ -14,7 +14,7 @@ } } }, - "98deadb899c73aece21dfd4bd6d75bb5b1df86ac6ba8e388e75564817f530423": { + "d9e912fee96913cd13f055f96dda70ae332b40be266ced809b9d4542b7fa639d": { "source": { "path": "aws-glue-job-scalaspark-streaming.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "98deadb899c73aece21dfd4bd6d75bb5b1df86ac6ba8e388e75564817f530423.json", + "objectKey": "d9e912fee96913cd13f055f96dda70ae332b40be266ced809b9d4542b7fa639d.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/aws-glue-job-scalaspark-streaming.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/aws-glue-job-scalaspark-streaming.template.json index b852a48bba566..7f247c0c0f44b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/aws-glue-job-scalaspark-streaming.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/aws-glue-job-scalaspark-streaming.template.json @@ -157,6 +157,9 @@ "JobRunQueuingEnabled": true, "MaxRetries": 0, "Name": "Optional Override ScalaSpark Streaming Job", + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, "NumberOfWorkers": 20, "Role": { "Fn::GetAtt": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/awsgluejobscalasparkstreamingintegtestDefaultTestDeployAssertCD3F6A81.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/awsgluejobscalasparkstreamingintegtestDefaultTestDeployAssertCD3F6A81.assets.json index af0fb3c80173b..5acbc1079f40e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/awsgluejobscalasparkstreamingintegtestDefaultTestDeployAssertCD3F6A81.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/awsgluejobscalasparkstreamingintegtestDefaultTestDeployAssertCD3F6A81.assets.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/cdk.out index c6e612584e352..1e02a2deb191b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"38.0.1"} \ No newline at end of file +{"version":"40.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/integ.json index 1cd0ca1422206..52effac7c7f91 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "testCases": { "aws-glue-job-scalaspark-streaming-integ-test/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/manifest.json index 8c24245f1e397..359bc69a24754 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "38.0.1", + "version": "40.0.0", "artifacts": { "aws-glue-job-scalaspark-streaming.assets": { "type": "cdk:asset-manifest", @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/98deadb899c73aece21dfd4bd6d75bb5b1df86ac6ba8e388e75564817f530423.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/d9e912fee96913cd13f055f96dda70ae332b40be266ced809b9d4542b7fa639d.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -34,24 +34,136 @@ "aws-glue-job-scalaspark-streaming.assets" ], "metadata": { + "/aws-glue-job-scalaspark-streaming/IAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": { + "assumedBy": { + "principalAccount": "*", + "assumeRoleAction": "*" + }, + "managedPolicies": [ + { + "managedPolicyArn": "*" + } + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachInlinePolicy": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addToPrincipalPolicy": [ + {} + ] + } + } + ], + "/aws-glue-job-scalaspark-streaming/IAMServiceRole/ImportIAMServiceRole": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalaspark-streaming/IAMServiceRole/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRole61C662C4" } ], + "/aws-glue-job-scalaspark-streaming/IAMServiceRole/DefaultPolicy": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "attachToRole": [ + "*" + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + }, + { + "type": "aws:cdk:analytics:method", + "data": { + "addStatements": [ + {} + ] + } + } + ], "/aws-glue-job-scalaspark-streaming/IAMServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", "data": "IAMServiceRoleDefaultPolicy379D1A0E" } ], + "/aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Resource": [ { "type": "aws:cdk:logicalId", "data": "BasicScalaSparkStreamingJob03E183FE" } ], + "/aws-glue-job-scalaspark-streaming/OverrideScalaSparkStreamingJob": [ + { + "type": "aws:cdk:analytics:construct", + "data": "*" + }, + { + "type": "aws:cdk:analytics:construct", + "data": "*" + } + ], "/aws-glue-job-scalaspark-streaming/OverrideScalaSparkStreamingJob/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/tree.json index 9c61ad2ceb369..82acbd6bb7d91 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.js.snapshot/tree.json @@ -1,378 +1 @@ -{ - "version": "tree-0.1", - "tree": { - "id": "App", - "path": "", - "children": { - "aws-glue-job-scalaspark-streaming": { - "id": "aws-glue-job-scalaspark-streaming", - "path": "aws-glue-job-scalaspark-streaming", - "children": { - "IAMServiceRole": { - "id": "IAMServiceRole", - "path": "aws-glue-job-scalaspark-streaming/IAMServiceRole", - "children": { - "ImportIAMServiceRole": { - "id": "ImportIAMServiceRole", - "path": "aws-glue-job-scalaspark-streaming/IAMServiceRole/ImportIAMServiceRole", - "constructInfo": { - "fqn": "aws-cdk-lib.Resource", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalaspark-streaming/IAMServiceRole/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Role", - "aws:cdk:cloudformation:props": { - "assumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "managedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSGlueServiceRole" - ] - ] - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnRole", - "version": "0.0.0" - } - }, - "DefaultPolicy": { - "id": "DefaultPolicy", - "path": "aws-glue-job-scalaspark-streaming/IAMServiceRole/DefaultPolicy", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalaspark-streaming/IAMServiceRole/DefaultPolicy/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::IAM::Policy", - "aws:cdk:cloudformation:props": { - "policyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":s3:::", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - } - ] - ] - } - ] - } - ], - "Version": "2012-10-17" - }, - "policyName": "IAMServiceRoleDefaultPolicy379D1A0E", - "roles": [ - { - "Ref": "IAMServiceRole61C662C4" - } - ] - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Policy", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_iam.Role", - "version": "0.0.0" - } - }, - "BasicScalaSparkStreamingJob": { - "id": "BasicScalaSparkStreamingJob", - "path": "aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob", - "children": { - "Codeb58a68516710fd95a65c427a7e567405": { - "id": "Codeb58a68516710fd95a65c427a7e567405", - "path": "aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Codeb58a68516710fd95a65c427a7e567405", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Codeb58a68516710fd95a65c427a7e567405/Stage", - "constructInfo": { - "fqn": "aws-cdk-lib.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Codeb58a68516710fd95a65c427a7e567405/AssetBucket", - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_s3_assets.Asset", - "version": "0.0.0" - } - }, - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "gluestreaming", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar" - ] - ] - } - }, - "defaultArguments": { - "--job-language": "scala", - "--class": "com.example.HelloWorld", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true" - }, - "glueVersion": "4.0", - "jobRunQueuingEnabled": false, - "numberOfWorkers": 10, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkStreamingJob", - "version": "0.0.0" - } - }, - "OverrideScalaSparkStreamingJob": { - "id": "OverrideScalaSparkStreamingJob", - "path": "aws-glue-job-scalaspark-streaming/OverrideScalaSparkStreamingJob", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-glue-job-scalaspark-streaming/OverrideScalaSparkStreamingJob/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::Glue::Job", - "aws:cdk:cloudformation:props": { - "command": { - "name": "gluestreaming", - "scriptLocation": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" - }, - "/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar" - ] - ] - } - }, - "defaultArguments": { - "--job-language": "scala", - "--class": "com.example.HelloWorld", - "--enable-continuous-cloudwatch-log": "true", - "--enable-metrics": "", - "--enable-observability-metrics": "true", - "arg1": "value1", - "arg2": "value2" - }, - "description": "Optional Override ScalaSpark Streaming Job", - "glueVersion": "3.0", - "jobRunQueuingEnabled": true, - "maxRetries": 0, - "name": "Optional Override ScalaSpark Streaming Job", - "numberOfWorkers": 20, - "role": { - "Fn::GetAtt": [ - "IAMServiceRole61C662C4", - "Arn" - ] - }, - "tags": { - "key": "value" - }, - "timeout": 15, - "workerType": "G.1X" - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.aws_glue.CfnJob", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkStreamingJob", - "version": "0.0.0" - } - }, - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-scalaspark-streaming/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-scalaspark-streaming/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - }, - "aws-glue-job-scalaspark-streaming-integ-test": { - "id": "aws-glue-job-scalaspark-streaming-integ-test", - "path": "aws-glue-job-scalaspark-streaming-integ-test", - "children": { - "DefaultTest": { - "id": "DefaultTest", - "path": "aws-glue-job-scalaspark-streaming-integ-test/DefaultTest", - "children": { - "Default": { - "id": "Default", - "path": "aws-glue-job-scalaspark-streaming-integ-test/DefaultTest/Default", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - }, - "DeployAssert": { - "id": "DeployAssert", - "path": "aws-glue-job-scalaspark-streaming-integ-test/DefaultTest/DeployAssert", - "children": { - "BootstrapVersion": { - "id": "BootstrapVersion", - "path": "aws-glue-job-scalaspark-streaming-integ-test/DefaultTest/DeployAssert/BootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnParameter", - "version": "0.0.0" - } - }, - "CheckBootstrapVersion": { - "id": "CheckBootstrapVersion", - "path": "aws-glue-job-scalaspark-streaming-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion", - "constructInfo": { - "fqn": "aws-cdk-lib.CfnRule", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.Stack", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", - "version": "0.0.0" - } - }, - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.4.2" - } - } - }, - "constructInfo": { - "fqn": "aws-cdk-lib.App", - "version": "0.0.0" - } - } -} \ No newline at end of file +{"version":"tree-0.1","tree":{"id":"App","path":"","children":{"aws-glue-job-scalaspark-streaming":{"id":"aws-glue-job-scalaspark-streaming","path":"aws-glue-job-scalaspark-streaming","children":{"IAMServiceRole":{"id":"IAMServiceRole","path":"aws-glue-job-scalaspark-streaming/IAMServiceRole","children":{"ImportIAMServiceRole":{"id":"ImportIAMServiceRole","path":"aws-glue-job-scalaspark-streaming/IAMServiceRole/ImportIAMServiceRole","constructInfo":{"fqn":"aws-cdk-lib.Resource","version":"0.0.0","metadata":["*"]}},"Resource":{"id":"Resource","path":"aws-glue-job-scalaspark-streaming/IAMServiceRole/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Role","aws:cdk:cloudformation:props":{"assumeRolePolicyDocument":{"Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":"glue.amazonaws.com"}}],"Version":"2012-10-17"},"managedPolicyArns":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":iam::aws:policy/service-role/AWSGlueServiceRole"]]}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnRole","version":"0.0.0"}},"DefaultPolicy":{"id":"DefaultPolicy","path":"aws-glue-job-scalaspark-streaming/IAMServiceRole/DefaultPolicy","children":{"Resource":{"id":"Resource","path":"aws-glue-job-scalaspark-streaming/IAMServiceRole/DefaultPolicy/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::IAM::Policy","aws:cdk:cloudformation:props":{"policyDocument":{"Statement":[{"Action":["s3:GetBucket*","s3:GetObject*","s3:List*"],"Effect":"Allow","Resource":[{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/*"]]},{"Fn::Join":["",["arn:",{"Ref":"AWS::Partition"},":s3:::",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"}]]}]}],"Version":"2012-10-17"},"policyName":"IAMServiceRoleDefaultPolicy379D1A0E","roles":[{"Ref":"IAMServiceRole61C662C4"}]}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.CfnPolicy","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Policy","version":"0.0.0","metadata":["*",{"attachToRole":["*"]},{"attachToRole":["*"]},{"addStatements":[{}]},{"addStatements":[{}]}]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_iam.Role","version":"0.0.0","metadata":[{"assumedBy":{"principalAccount":"*","assumeRoleAction":"*"},"managedPolicies":[{"managedPolicyArn":"*"}]},{"addToPrincipalPolicy":[{}]},{"attachInlinePolicy":["*"]},{"attachInlinePolicy":["*"]},{"addToPrincipalPolicy":[{}]}]}},"BasicScalaSparkStreamingJob":{"id":"BasicScalaSparkStreamingJob","path":"aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob","children":{"Codeb1ad5b8a47fabefcad19ad781568f6a7":{"id":"Codeb1ad5b8a47fabefcad19ad781568f6a7","path":"aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Codeb1ad5b8a47fabefcad19ad781568f6a7","children":{"Stage":{"id":"Stage","path":"aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Codeb1ad5b8a47fabefcad19ad781568f6a7/Stage","constructInfo":{"fqn":"aws-cdk-lib.AssetStaging","version":"0.0.0"}},"AssetBucket":{"id":"AssetBucket","path":"aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Codeb1ad5b8a47fabefcad19ad781568f6a7/AssetBucket","constructInfo":{"fqn":"aws-cdk-lib.aws_s3.BucketBase","version":"0.0.0","metadata":[]}}},"constructInfo":{"fqn":"aws-cdk-lib.aws_s3_assets.Asset","version":"0.0.0"}},"Resource":{"id":"Resource","path":"aws-glue-job-scalaspark-streaming/BasicScalaSparkStreamingJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"gluestreaming","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar"]]}},"defaultArguments":{"--job-language":"scala","--class":"com.example.HelloWorld","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true"},"glueVersion":"4.0","jobRunQueuingEnabled":false,"numberOfWorkers":10,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.ScalaSparkStreamingJob","version":"0.0.0","metadata":["*","*"]}},"OverrideScalaSparkStreamingJob":{"id":"OverrideScalaSparkStreamingJob","path":"aws-glue-job-scalaspark-streaming/OverrideScalaSparkStreamingJob","children":{"Resource":{"id":"Resource","path":"aws-glue-job-scalaspark-streaming/OverrideScalaSparkStreamingJob/Resource","attributes":{"aws:cdk:cloudformation:type":"AWS::Glue::Job","aws:cdk:cloudformation:props":{"command":{"name":"gluestreaming","scriptLocation":{"Fn::Join":["",["s3://",{"Fn::Sub":"cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"},"/e305655b966b957f91fcec580e3f8703573eb6b69528c5d52190d72579c91602.jar"]]}},"defaultArguments":{"--job-language":"scala","--class":"com.example.HelloWorld","--enable-continuous-cloudwatch-log":"true","--enable-metrics":"","--enable-observability-metrics":"true","arg1":"value1","arg2":"value2"},"description":"Optional Override ScalaSpark Streaming Job","glueVersion":"3.0","jobRunQueuingEnabled":true,"maxRetries":0,"name":"Optional Override ScalaSpark Streaming Job","notificationProperty":{"notifyDelayAfter":5},"numberOfWorkers":20,"role":{"Fn::GetAtt":["IAMServiceRole61C662C4","Arn"]},"tags":{"key":"value"},"timeout":15,"workerType":"G.1X"}},"constructInfo":{"fqn":"aws-cdk-lib.aws_glue.CfnJob","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/aws-glue-alpha.ScalaSparkStreamingJob","version":"0.0.0","metadata":["*","*"]}},"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-scalaspark-streaming/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-scalaspark-streaming/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}},"aws-glue-job-scalaspark-streaming-integ-test":{"id":"aws-glue-job-scalaspark-streaming-integ-test","path":"aws-glue-job-scalaspark-streaming-integ-test","children":{"DefaultTest":{"id":"DefaultTest","path":"aws-glue-job-scalaspark-streaming-integ-test/DefaultTest","children":{"Default":{"id":"Default","path":"aws-glue-job-scalaspark-streaming-integ-test/DefaultTest/Default","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}},"DeployAssert":{"id":"DeployAssert","path":"aws-glue-job-scalaspark-streaming-integ-test/DefaultTest/DeployAssert","children":{"BootstrapVersion":{"id":"BootstrapVersion","path":"aws-glue-job-scalaspark-streaming-integ-test/DefaultTest/DeployAssert/BootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnParameter","version":"0.0.0"}},"CheckBootstrapVersion":{"id":"CheckBootstrapVersion","path":"aws-glue-job-scalaspark-streaming-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion","constructInfo":{"fqn":"aws-cdk-lib.CfnRule","version":"0.0.0"}}},"constructInfo":{"fqn":"aws-cdk-lib.Stack","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTestCase","version":"0.0.0"}}},"constructInfo":{"fqn":"@aws-cdk/integ-tests-alpha.IntegTest","version":"0.0.0"}},"Tree":{"id":"Tree","path":"Tree","constructInfo":{"fqn":"constructs.Construct","version":"10.4.2"}}},"constructInfo":{"fqn":"aws-cdk-lib.App","version":"0.0.0"}}} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.ts index 9dfb5b450dc61..72731bf5a1360 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-scalaspark-streaming.ts @@ -55,6 +55,7 @@ new glue.ScalaSparkStreamingJob(stack, 'OverrideScalaSparkStreamingJob', { key: 'value', }, jobRunQueuingEnabled: true, + notifyDelayAfter: cdk.Duration.minutes(5), }); new integ.IntegTest(app, 'aws-glue-job-scalaspark-streaming-integ-test', { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/pyspark-etl-jobs.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/pyspark-etl-jobs.test.ts index 7e9d5c0dd3f85..aa8d84edd7b0c 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/pyspark-etl-jobs.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/pyspark-etl-jobs.test.ts @@ -454,6 +454,7 @@ describe('Job', () => { }, numberOfWorkers: 2, maxRetries: 2, + notifyDelayAfter: cdk.Duration.minutes(5), }); }); @@ -544,6 +545,14 @@ describe('Job', () => { }, }); }); + + test('Should set notify delay', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }); + }); }); describe('Create PySpark ETL Job with overridden job run queueing', () => { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/pyspark-flex-etl-jobs.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/pyspark-flex-etl-jobs.test.ts index 1a5c635c3430f..a3eb1b240fc13 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/pyspark-flex-etl-jobs.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/pyspark-flex-etl-jobs.test.ts @@ -155,6 +155,7 @@ describe('Job', () => { }, numberOfWorkers: 2, maxRetries: 2, + notifyDelayAfter: cdk.Duration.minutes(5), }); }); @@ -245,5 +246,13 @@ describe('Job', () => { }, }); }); + + test('Should set notify delay', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }); + }); }); }); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/pyspark-streaming-jobs.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/pyspark-streaming-jobs.test.ts index 6d65fa819fa67..a1dcb2f762419 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/pyspark-streaming-jobs.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/pyspark-streaming-jobs.test.ts @@ -466,6 +466,7 @@ describe('Job', () => { }, numberOfWorkers: 2, maxRetries: 2, + notifyDelayAfter: cdk.Duration.minutes(5), }); }); @@ -566,6 +567,14 @@ describe('Job', () => { }, }); }); + + test('Should set notify delay', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }); + }); }); describe('Create PySpark Streaming Job with job run queuing enabled', () => { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/python-shell-job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/python-shell-job.test.ts index 5a2fc35197ab2..9c41fb888fd00 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/python-shell-job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/python-shell-job.test.ts @@ -199,6 +199,7 @@ describe('Job', () => { }, numberOfWorkers: 2, maxRetries: 2, + notifyDelayAfter: cdk.Duration.minutes(5), }); }); @@ -293,6 +294,14 @@ describe('Job', () => { MaxCapacity: 1, }); }); + + test('Should set notify delay', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }); + }); }); describe('Create Python Shell Job with job run queuing enabled', () => { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-etl-jobs.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-etl-jobs.test.ts index c70cb3338f198..8a02ad0ce8b18 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-etl-jobs.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-etl-jobs.test.ts @@ -161,6 +161,7 @@ describe('Job', () => { }, numberOfWorkers: 2, maxRetries: 2, + notifyDelayAfter: cdk.Duration.minutes(5), }); }); @@ -251,6 +252,14 @@ describe('Job', () => { }, }); }); + + test('Should set notify delay', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }); + }); }); describe('Create ScalaSpark ETL Job with job run queuing enabled', () => { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-flex-etl-jobs.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-flex-etl-jobs.test.ts index dbf9b94c55210..f511cf5d49ec2 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-flex-etl-jobs.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-flex-etl-jobs.test.ts @@ -161,6 +161,7 @@ describe('Job', () => { }, numberOfWorkers: 2, maxRetries: 2, + notifyDelayAfter: cdk.Duration.minutes(5), }); }); @@ -251,6 +252,14 @@ describe('Job', () => { }, }); }); + + test('Should set notify delay', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }); + }); }); describe('Create ScalaSpark Flex ETL Job with extraFiles, extraJars and extraJarsFirst', () => { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-streaming-jobs.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-streaming-jobs.test.ts index 5ce1440f5e094..4968c7234dbd1 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-streaming-jobs.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/scalaspark-streaming-jobs.test.ts @@ -276,6 +276,7 @@ describe('Job', () => { numberOfWorkers: 2, maxRetries: 2, jobRunQueuingEnabled: true, + notifyDelayAfter: cdk.Duration.minutes(5), }); }); @@ -372,6 +373,14 @@ describe('Job', () => { }, }); }); + + test('Should set notify delay', () => { + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Job', { + NotificationProperty: { + NotifyDelayAfter: 5, + }, + }); + }); }); describe('Override SparkUI properties for ScalaSpark Streaming ETL Job', () => {