Skip to content

Update RFS default worker config and add initial-lease-duration #1878

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Update RFS default worker config and add initial-lease-duration #1878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions ...oyment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore {
command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--lucene-dir", `"${storagePath}/lucene"`)
command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--target-host", osClusterEndpoint)
command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--max-shard-size-bytes", `${Math.ceil(maxShardSizeBytes)}`)
command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--max-connections", props.reindexFromSnapshotWorkerSize === "maximum" ? "100" : "10")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is the current minimum too low, shouldn't we advise customers with larger shards to set the size to maximum and avoid tweaking the min values?

command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--max-connections", props.reindexFromSnapshotWorkerSize === "maximum" ? "100" : "20")
command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--initial-lease-duration", "PT60M")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we update the default in the java code rather than cloud formation, this won't be the default on the EKS deployments of RFS

if (props.clusterAuthDetails.sigv4) {
command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--target-aws-service-signing-name", props.clusterAuthDetails.sigv4.serviceSigningName)
command = appendArgIfNotInExtraArgs(command, extraArgsDict, "--target-aws-region", props.clusterAuthDetails.sigv4.region)
Expand Down Expand Up @@ -162,7 +163,7 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore {
size: Size.gibibytes(shardVolumeSizeGiB),
volumeType: EbsDeviceVolumeType.GP3,
fileSystemType: FileSystemType.XFS,
throughput: props.reindexFromSnapshotWorkerSize === "maximum" ? 450 : 125,
throughput: props.reindexFromSnapshotWorkerSize === "maximum" ? 450 : 250,
tagSpecifications: [{
tags: {
Name: `rfs-snapshot-volume-${props.stage}`,
Expand Down Expand Up @@ -191,8 +192,8 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore {
mountPoints: mountPoints,
taskRolePolicies: servicePolicies,
cpuArchitecture: props.fargateCpuArch,
taskCpuUnits: props.reindexFromSnapshotWorkerSize === "maximum" ? 16 * 1024 : 2 * 1024,
taskMemoryLimitMiB: props.reindexFromSnapshotWorkerSize === "maximum" ? 32 * 1024 : 4 * 1024,
taskCpuUnits: props.reindexFromSnapshotWorkerSize === "maximum" ? 16 * 1024 : 4 * 1024,
taskMemoryLimitMiB: props.reindexFromSnapshotWorkerSize === "maximum" ? 32 * 1024 : 8 * 1024,
ephemeralStorageGiB: ephemeralStorageGiB,
environment: {
"RFS_COMMAND": command,
Expand Down
20 changes: 10 additions & 10 deletions deployment/cdk/opensearch-service-migration/test/reindex-from-snapshot-stack.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ describe('ReindexFromSnapshotStack Tests', () => {
});
// Assert CPU configuration
template.hasResourceProperties('AWS::ECS::TaskDefinition', {
Cpu: "2048",
Memory: "4096",
Cpu: "4096",
Memory: "8192",
});
});

Expand Down Expand Up @@ -123,7 +123,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
{
"Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
},
" --max-shard-size-bytes 94489280512 --max-connections 10 --source-version \"ES_7.10\""
" --max-shard-size-bytes 94489280512 --max-connections 20 --initial-lease-duration PT60M --source-version \"ES_7.10\""
],
],
}
Expand Down Expand Up @@ -181,7 +181,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
{
"Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
},
" --max-shard-size-bytes 94489280512 --max-connections 10 --target-aws-service-signing-name aoss --target-aws-region eu-west-1 --source-version \"ES_7.10\"",
" --max-shard-size-bytes 94489280512 --max-connections 20 --initial-lease-duration PT60M --target-aws-service-signing-name aoss --target-aws-region eu-west-1 --source-version \"ES_7.10\"",
],
],
}
Expand Down Expand Up @@ -243,7 +243,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
{
"Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
},
" --max-shard-size-bytes 94489280512 --max-connections 10 --source-version \"ES_7.10\""
" --max-shard-size-bytes 94489280512 --max-connections 20 --initial-lease-duration PT60M --source-version \"ES_7.10\""
],
],
}
Expand Down Expand Up @@ -318,7 +318,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
{
"Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
},
" --max-shard-size-bytes 94489280512 --max-connections 10 --source-version \"ES_7.10\" --custom-arg value --flag --snapshot-name \"custom-snapshot\""
" --max-shard-size-bytes 94489280512 --max-connections 20 --initial-lease-duration PT60M --source-version \"ES_7.10\" --custom-arg value --flag --snapshot-name \"custom-snapshot\""
]
]
}
Expand Down Expand Up @@ -372,7 +372,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
{
"Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
},
" --max-shard-size-bytes 94489280512 --max-connections 100 --source-version \"ES_7.10\"",
" --max-shard-size-bytes 94489280512 --max-connections 100 --initial-lease-duration PT60M --source-version \"ES_7.10\"",
],
],
}
Expand Down Expand Up @@ -501,7 +501,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
{
"Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
},
" --max-shard-size-bytes 1181117188 --max-connections 10 --target-aws-service-signing-name aoss --target-aws-region eu-west-1 --source-version \"ES_7.10\"",
" --max-shard-size-bytes 1181117188 --max-connections 20 --initial-lease-duration PT60M --target-aws-service-signing-name aoss --target-aws-region eu-west-1 --source-version \"ES_7.10\"",
],
],
}
Expand Down Expand Up @@ -576,7 +576,7 @@ describe('ReindexFromSnapshotStack Tests', () => {
{
"Ref": "SsmParameterValuemigrationunittestdefaultosClusterEndpointC96584B6F00A464EAD1953AFF4B05118Parameter",
},
" --max-shard-size-bytes 94489280512 --max-connections 10 --source-version \"ES_7.9\""
" --max-shard-size-bytes 94489280512 --max-connections 20 --initial-lease-duration PT60M --source-version \"ES_7.9\""
],
],
}
Expand Down Expand Up @@ -649,4 +649,4 @@ describe('ReindexFromSnapshotStack Tests', () => {
// This demonstrates that RFS will fail at runtime with:
// "The following option is required: [--source-version, --sourceVersion]"
});
});
});