Skip to content

Commit ee77c8c

Browse files
xschildwxschildw
authored
Merge pull request #783 from marcomarasca/PLFM-9172
PLFM-9172: Move the synapse help docs bedrock KB to global resources
2 parents 298c70a + bee6db4 commit ee77c8c

20 files changed

+779
-284
lines changed

‎src/main/java/org/sagebionetworks/template/CloudFormationClientWrapper.java‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package org.sagebionetworks.template;
22

33
import java.util.Optional;
4-
import java.util.Set;
54
import java.util.stream.Stream;
65

76
import software.amazon.awssdk.services.cloudformation.model.CloudFormationException;
@@ -65,19 +64,6 @@ public interface CloudFormationClientWrapper {
6564
* @throws RuntimeException if the stack operation fails or times out
6665
*/
6766
Optional<Stack> waitForStackToComplete(String stackName) throws InterruptedException;
68-
69-
/**
70-
* Waits for a stack operation to complete while handling CloudFormation wait conditions.
71-
* Each wait condition handler in the provided set will be invoked when its corresponding
72-
* wait condition is triggered during stack creation.
73-
*
74-
* @param stackName the name of the stack to wait for
75-
* @param waitConditionHandlers a set of handlers for processing CloudFormation wait conditions
76-
* @return an Optional containing the final Stack if it exists, or empty if it doesn't
77-
* @throws InterruptedException if the waiting thread is interrupted
78-
* @throws RuntimeException if the stack operation fails, a wait condition fails, or the operation times out
79-
*/
80-
Optional<Stack> waitForStackToComplete(String stackName, Set<WaitConditionHandler> waitConditionHandlers) throws InterruptedException;
8167

8268
/**
8369
* Retrieves the value of a specific output from a CloudFormation stack.

‎src/main/java/org/sagebionetworks/template/CloudFormationClientWrapperImpl.java‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -57,21 +57,24 @@ public class CloudFormationClientWrapperImpl implements CloudFormationClientWrap
5757

5858
public static final int SLEEP_TIME = 10 * 1000;
5959
public static final String NO_UPDATES_ARE_TO_BE_PERFORMED = "No updates are to be performed";
60-
CloudFormationClient cloudFormationClient;
61-
AmazonS3 s3Client;
62-
Configuration configuration;
63-
Logger logger;
64-
ThreadProvider threadProvider;
65-
60+
61+
private final CloudFormationClient cloudFormationClient;
62+
private final AmazonS3 s3Client;
63+
private final Configuration configuration;
64+
private final Logger logger;
65+
private final ThreadProvider threadProvider;
66+
private final Map<String, WaitConditionHandler> waitConditionHandlerMap;
67+
6668
@Inject
6769
public CloudFormationClientWrapperImpl(CloudFormationClient cloudFormationClient, AmazonS3 s3Client,
68-
Configuration configuration, LoggerFactory loggerFactory, ThreadProvider threadProvider) {
70+
Configuration configuration, LoggerFactory loggerFactory, ThreadProvider threadProvider, Set<WaitConditionHandler> waitConditionHandlers) {
6971
super();
7072
this.cloudFormationClient = cloudFormationClient;
7173
this.s3Client = s3Client;
7274
this.configuration = configuration;
7375
this.logger = loggerFactory.getLogger(CloudFormationClientWrapperImpl.class);
7476
this.threadProvider = threadProvider;
77+
this.waitConditionHandlerMap = waitConditionHandlers.stream().collect(Collectors.toMap(WaitConditionHandler::getWaitConditionId, Function.identity()));
7578
}
7679

7780
@Override
@@ -251,16 +254,8 @@ public boolean isStartedInUpdateRollbackComplete(String stackName) {
251254

252255
@Override
253256
public Optional<Stack> waitForStackToComplete(String stackName) throws InterruptedException {
254-
return waitForStackToComplete(stackName, Collections.emptySet());
255-
}
256-
257-
@Override
258-
public Optional<Stack> waitForStackToComplete(String stackName, Set<WaitConditionHandler> waitConditionHandlers) throws InterruptedException {
259257
boolean startedInUpdateRollbackComplete = isStartedInUpdateRollbackComplete(stackName); // Initial state
260258

261-
Map<String, WaitConditionHandler> waitConditionHandlerMap = waitConditionHandlers.stream()
262-
.collect(Collectors.toMap(WaitConditionHandler::getWaitConditionId, Function.identity()));
263-
264259
// To avoid re-processing the same wait condition multiple times we need to keep track of them
265260
Set<String> processedWaitConditionSet = new HashSet<>();
266261

‎src/main/java/org/sagebionetworks/template/Constants.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ public class Constants {
7070
public static final String PROPERTY_KEY_VPC_ENDPOINTS_AZ = "org.sagebionetworks.vpc.endpoints.availability.zones";
7171
public static final String PROPERTY_KEY_VPC_PEERING_ACCEPT_ROLE_ARN = "org.sagebionetworks.vpc.peering.accept.role.arn";
7272
public static final String PROPERTY_KEY_OLD_VPC_CIDR = "org.sagebionetworks.vpc.old.vpc.cidr";
73-
public static final String PROPERTY_KEY_OPS_VPC_EXPORT_PREFIX = "org.sagebionetworks.vpc.ops.export.prefix";
7473
// repo
7574
public static final String PROPERTY_KEY_STACK = "org.sagebionetworks.stack";
7675
public static final String PROPERTY_KEY_INSTANCE = "org.sagebionetworks.instance";

‎src/main/java/org/sagebionetworks/template/TemplateGuiceModule.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
import org.sagebionetworks.template.docs.SynapseDocsBuilderImpl;
4545
import org.sagebionetworks.template.global.GlobalResourcesBuilder;
4646
import org.sagebionetworks.template.global.GlobalResourcesBuilderImpl;
47+
import org.sagebionetworks.template.global.waitconditions.SynapseHelpCollectionIndexCreation;
48+
import org.sagebionetworks.template.global.waitconditions.SynapseHelpKnowledgeBaseDataSourceSync;
4749
import org.sagebionetworks.template.ip.address.IpAddressPoolBuilder;
4850
import org.sagebionetworks.template.ip.address.IpAddressPoolBuilderImpl;
4951
import org.sagebionetworks.template.jobs.AsynchAdminJobExecutor;
@@ -81,8 +83,6 @@
8183
import org.sagebionetworks.template.repo.beanstalk.ssl.CertificateBuilderImpl;
8284
import org.sagebionetworks.template.repo.beanstalk.ssl.ElasticBeanstalkExtentionBuilder;
8385
import org.sagebionetworks.template.repo.beanstalk.ssl.ElasticBeanstalkExtentionBuilderImpl;
84-
import org.sagebionetworks.template.repo.bedrock.SynapseHelpCollectionIndexCreation;
85-
import org.sagebionetworks.template.repo.bedrock.SynapseHelpKnowledgeBaseDataSourceSync;
8686
import org.sagebionetworks.template.repo.cloudwatchlogs.CloudwatchLogsConfig;
8787
import org.sagebionetworks.template.repo.cloudwatchlogs.CloudwatchLogsConfigValidator;
8888
import org.sagebionetworks.template.repo.cloudwatchlogs.CloudwatchLogsVelocityContextProvider;

‎src/main/java/org/sagebionetworks/template/global/GlobalResourcesBuilderImpl.java‎

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,59 @@
11
package org.sagebionetworks.template.global;
22

3-
import com.google.inject.Inject;
4-
import org.apache.logging.log4j.Logger;
3+
import static org.sagebionetworks.template.Constants.DELETION_POLICY;
4+
import static org.sagebionetworks.template.Constants.GLOBAL_CFSTACK_OUTPUT_KEY_SES_BOUNCE_TOPIC;
5+
import static org.sagebionetworks.template.Constants.GLOBAL_CFSTACK_OUTPUT_KEY_SES_COMPLAINT_TOPIC;
6+
import static org.sagebionetworks.template.Constants.GLOBAL_RESOURCES_STACK_NAME_FORMAT;
7+
import static org.sagebionetworks.template.Constants.IDENTITY_ARN;
8+
import static org.sagebionetworks.template.Constants.JSON_INDENT;
9+
import static org.sagebionetworks.template.Constants.OPS_VPC_EXPORT_PREFIX;
10+
import static org.sagebionetworks.template.Constants.PROPERTY_KEY_STACK;
11+
import static org.sagebionetworks.template.Constants.SES_SYNAPSE_DOMAIN;
12+
import static org.sagebionetworks.template.Constants.STACK;
13+
import static org.sagebionetworks.template.Constants.TEMPLATE_GLOBAL_RESOURCES;
14+
import static org.sagebionetworks.template.Constants.VPC_EXPORT_PREFIX;
15+
16+
import java.io.StringWriter;
17+
518
import org.apache.velocity.Template;
619
import org.apache.velocity.VelocityContext;
720
import org.apache.velocity.app.VelocityEngine;
821
import org.json.JSONObject;
922
import org.sagebionetworks.template.CloudFormationClientWrapper;
1023
import org.sagebionetworks.template.Constants;
1124
import org.sagebionetworks.template.CreateOrUpdateStackRequest;
12-
import org.sagebionetworks.template.LoggerFactory;
1325
import org.sagebionetworks.template.SesClientWrapper;
1426
import org.sagebionetworks.template.StackTagsProvider;
1527
import org.sagebionetworks.template.config.Configuration;
1628
import org.sagebionetworks.template.repo.DeletionPolicy;
17-
import software.amazon.awssdk.services.cloudformation.model.Capability;
1829

19-
import java.io.StringWriter;
30+
import com.google.inject.Inject;
2031

21-
import static org.sagebionetworks.template.Constants.DELETION_POLICY;
22-
import static org.sagebionetworks.template.Constants.GLOBAL_RESOURCES_STACK_NAME_FORMAT;
23-
import static org.sagebionetworks.template.Constants.JSON_INDENT;
24-
import static org.sagebionetworks.template.Constants.PROPERTY_KEY_STACK;
25-
import static org.sagebionetworks.template.Constants.SES_SYNAPSE_DOMAIN;
26-
import static org.sagebionetworks.template.Constants.STACK;
27-
import static org.sagebionetworks.template.Constants.TEMPLATE_GLOBAL_RESOURCES;
28-
import static org.sagebionetworks.template.Constants.CAPABILITY_NAMED_IAM;
29-
import static org.sagebionetworks.template.Constants.GLOBAL_CFSTACK_OUTPUT_KEY_SES_BOUNCE_TOPIC;
30-
import static org.sagebionetworks.template.Constants.GLOBAL_CFSTACK_OUTPUT_KEY_SES_COMPLAINT_TOPIC;
32+
import software.amazon.awssdk.services.cloudformation.model.Capability;
33+
import software.amazon.awssdk.services.sts.StsClient;
3134

3235
public class GlobalResourcesBuilderImpl implements GlobalResourcesBuilder {
3336

34-
CloudFormationClientWrapper cloudFormationClientWrapper;
35-
VelocityEngine velocityEngine;
36-
Configuration config;
37-
Logger logger;
38-
StackTagsProvider stackTagsProvider;
39-
SesClientWrapper sesClientWrapper;
37+
private final CloudFormationClientWrapper cloudFormationClientWrapper;
38+
private final VelocityEngine velocityEngine;
39+
private final Configuration config;
40+
private final StackTagsProvider stackTagsProvider;
41+
private final SesClientWrapper sesClientWrapper;
42+
private final StsClient stsClient;
4043

4144
@Inject
4245
public GlobalResourcesBuilderImpl(CloudFormationClientWrapper cloudFormationClientWrapper,
4346
VelocityEngine velocityEngine,
4447
Configuration config,
45-
LoggerFactory loggerFactory,
4648
StackTagsProvider stackTagsProvider,
47-
SesClientWrapper sesClientWrapper) {
49+
SesClientWrapper sesClientWrapper,
50+
StsClient stsClient) {
4851
this.cloudFormationClientWrapper = cloudFormationClientWrapper;
4952
this.velocityEngine = velocityEngine;
5053
this.config = config;
51-
this.logger = loggerFactory.getLogger(GlobalResourcesBuilderImpl.class);
5254
this.stackTagsProvider = stackTagsProvider;
5355
this.sesClientWrapper = sesClientWrapper;
56+
this.stsClient = stsClient;
5457
}
5558

5659
@Override
@@ -83,15 +86,23 @@ public String createStackName() {
8386

8487
public VelocityContext createContext() {
8588
VelocityContext context = new VelocityContext();
86-
context.put(STACK, config.getProperty(PROPERTY_KEY_STACK));
87-
context.put(DELETION_POLICY,
88-
Constants.isProd(config.getProperty(PROPERTY_KEY_STACK)) ? DeletionPolicy.Retain.name() : DeletionPolicy.Delete.name());
89+
90+
String stack = config.getProperty(PROPERTY_KEY_STACK);
91+
92+
context.put(STACK, stack);
93+
context.put(DELETION_POLICY, Constants.isProd(config.getProperty(PROPERTY_KEY_STACK)) ? DeletionPolicy.Retain.name() : DeletionPolicy.Delete.name());
94+
95+
context.put(VPC_EXPORT_PREFIX, Constants.createVpcExportPrefix(stack));
96+
context.put(OPS_VPC_EXPORT_PREFIX, "us-east-1-vpc");
97+
context.put(IDENTITY_ARN, stsClient.getCallerIdentity().arn());
98+
8999
return context;
90100
}
91101

92102
public void setupSesTopics(String stackName) {
93103
String sesComplaintSnsTopic = this.cloudFormationClientWrapper.getOutput(stackName, GLOBAL_CFSTACK_OUTPUT_KEY_SES_COMPLAINT_TOPIC);
94104
String sesBounceSnsTopic = this.cloudFormationClientWrapper.getOutput(stackName, GLOBAL_CFSTACK_OUTPUT_KEY_SES_BOUNCE_TOPIC);
105+
95106
sesClientWrapper.setComplaintNotificationTopic(SES_SYNAPSE_DOMAIN, sesComplaintSnsTopic);
96107
sesClientWrapper.setBounceNotificationTopic(SES_SYNAPSE_DOMAIN, sesBounceSnsTopic);
97108
}

src/main/java/org/sagebionetworks/template/repo/bedrock/SynapseHelpCollectionIndexCreation.java renamed to src/main/java/org/sagebionetworks/template/global/waitconditions/SynapseHelpCollectionIndexCreation.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
1-
package org.sagebionetworks.template.repo.bedrock;
1+
package org.sagebionetworks.template.global.waitconditions;
22

33
import java.io.IOException;
44
import java.util.Optional;
55

66
import org.apache.logging.log4j.Logger;
77
import org.opensearch.client.opensearch._types.OpenSearchException;
8+
import org.opensearch.client.opensearch._types.WaitForActiveShardOptions;
89
import org.opensearch.client.opensearch.indices.OpenSearchIndicesClient;
910
import org.sagebionetworks.template.Constants;
1011
import org.sagebionetworks.template.LoggerFactory;
1112
import org.sagebionetworks.template.OpenSearchClientFactory;
1213
import org.sagebionetworks.template.WaitConditionHandler;
1314
import org.sagebionetworks.template.config.RepoConfiguration;
1415

15-
import software.amazon.awssdk.services.cloudformation.model.StackEvent;
1616
import com.google.inject.Inject;
1717

18+
import software.amazon.awssdk.services.cloudformation.model.StackEvent;
1819
import software.amazon.awssdk.services.opensearchserverless.OpenSearchServerlessClient;
1920
import software.amazon.awssdk.services.opensearchserverless.model.CollectionDetail;
2021
import software.amazon.awssdk.services.opensearchserverless.model.CollectionStatus;
@@ -56,7 +57,7 @@ public String getWaitConditionId() {
5657

5758
@Override
5859
public Optional<String> handle(StackEvent stackEvent) {
59-
String collectionName = config.getProperty(Constants.PROPERTY_KEY_STACK) + "-" + config.getProperty(Constants.PROPERTY_KEY_INSTANCE) + "-synhelp";
60+
String collectionName = config.getProperty(Constants.PROPERTY_KEY_STACK) + "-synhelp";
6061

6162
CollectionDetail collection = ossManagementClient.batchGetCollection(req -> req
6263
.names(collectionName)
@@ -79,6 +80,7 @@ public Optional<String> handle(StackEvent stackEvent) {
7980
logger.info("Index {} does not exist, creating...", IDX_NAME);
8081

8182
client.create(req -> req
83+
.waitForActiveShards( opt -> opt.option(WaitForActiveShardOptions.All))
8284
.index(IDX_NAME)
8385
.settings(settings -> settings.knn(true).knnAlgoParamEfSearch(512))
8486
.mappings(mappings -> mappings

src/main/java/org/sagebionetworks/template/repo/bedrock/SynapseHelpKnowledgeBaseDataSourceSync.java renamed to src/main/java/org/sagebionetworks/template/global/waitconditions/SynapseHelpKnowledgeBaseDataSourceSync.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package org.sagebionetworks.template.repo.bedrock;
1+
package org.sagebionetworks.template.global.waitconditions;
22

33
import java.util.Optional;
44

@@ -48,17 +48,17 @@ public String getWaitConditionId() {
4848

4949
@Override
5050
public Optional<String> handle(StackEvent stackEvent) throws InterruptedException {
51-
String stackPrefix = config.getProperty(Constants.PROPERTY_KEY_STACK) + "-" + config.getProperty(Constants.PROPERTY_KEY_INSTANCE);
51+
String stack = config.getProperty(Constants.PROPERTY_KEY_STACK);
5252

53-
String knowledgeBaseName = stackPrefix + "-synhelp-knowledge-base";
53+
String knowledgeBaseName = stack + "-synhelp-knowledge-base";
5454
String knowledgeBaseId = bedrockAgentClient.listKnowledgeBasesPaginator(req -> {})
5555
.knowledgeBaseSummaries().stream()
5656
.filter(kb -> kb.name().equals(knowledgeBaseName))
5757
.findFirst()
5858
.map(KnowledgeBaseSummary::knowledgeBaseId)
5959
.orElseThrow();
6060

61-
String dataSourceName = stackPrefix + "-synhelp-datasource";
61+
String dataSourceName = stack + "-synhelp-datasource";
6262
String dataSourceId = bedrockAgentClient.listDataSourcesPaginator(req -> req.knowledgeBaseId(knowledgeBaseId))
6363
.dataSourceSummaries().stream()
6464
.filter(dataSource -> dataSource.name().equals(dataSourceName))

0 commit comments

Comments
 (0)