Skip to content

Commit 11baf35

Browse files
aws-kevinrickardaws-kevinrickard
authored
feat: add deploymentConfigurationTimeSource configuration (#1706)
* feat: add deploymentConfigurationTimeSource configuration * chore: add debug logs for deployment processing structures
1 parent 8e6e91b commit 11baf35

File tree

17 files changed

+191
-100
lines changed

17 files changed

+191
-100
lines changed

‎src/integrationtests/java/com/aws/greengrass/integrationtests/deployment/DeploymentConfigMergingTest.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ void GIVEN_kernel_running_with_some_config_WHEN_merge_simple_yaml_file_THEN_conf
158158
((Map<String, Object>)newConfig.get(SERVICES_NAMESPACE_TOPIC)).put(DEFAULT_NUCLEUS_COMPONENT_NAME,
159159
getNucleusConfig());
160160

161-
deploymentConfigMerger.mergeInNewConfig(testDeployment(), newConfig).get(60, TimeUnit.SECONDS);
161+
deploymentConfigMerger.mergeInNewConfig(testDeployment(), newConfig, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
162162

163163
t = kernel.findServiceTopic(FleetStatusService.FLEET_STATUS_SERVICE_TOPICS);
164164
assertNotNull(t, "FSS Topics should not be null after merging");
@@ -210,7 +210,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_changes_service_THEN_service
210210
}});
211211
put(DEFAULT_NUCLEUS_COMPONENT_NAME, getNucleusConfig());
212212
}});
213-
}}).get(60, TimeUnit.SECONDS);
213+
}}, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
214214

215215
// THEN
216216
assertTrue(mainRestarted.await(10, TimeUnit.SECONDS), "main restarted");
@@ -268,7 +268,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_change_adding_dependency_THE
268268

269269
put(DEFAULT_NUCLEUS_COMPONENT_NAME, getNucleusConfig());
270270
}});
271-
}}).get(60, TimeUnit.SECONDS);
271+
}}, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
272272
// THEN
273273
assertTrue(newServiceStarted.get(), "new service started");
274274
}
@@ -338,7 +338,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_change_adding_nested_depende
338338

339339
put(DEFAULT_NUCLEUS_COMPONENT_NAME, getNucleusConfig());
340340
}});
341-
}}).get(60, TimeUnit.SECONDS);
341+
}}, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
342342

343343
// THEN
344344
assertTrue(newService2Started.get());
@@ -411,7 +411,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_same_doc_happens_twice_THEN_
411411
kernel.getContext().addGlobalStateChangeListener(listener);
412412

413413
GreengrassService main = kernel.locate("main");
414-
deploymentConfigMerger.mergeInNewConfig(testDeployment(), newConfig).get(60, TimeUnit.SECONDS);
414+
deploymentConfigMerger.mergeInNewConfig(testDeployment(), newConfig, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
415415
kernel.getContext().waitForPublishQueueToClear();
416416

417417
// Verify that first merge succeeded.
@@ -437,7 +437,7 @@ void GIVEN_kernel_running_single_service_WHEN_merge_same_doc_happens_twice_THEN_
437437
// THEN
438438
// merge in the same config the second time
439439
// merge shouldn't block
440-
deploymentConfigMerger.mergeInNewConfig(testDeployment(), newConfig).get(60, TimeUnit.SECONDS);
440+
deploymentConfigMerger.mergeInNewConfig(testDeployment(), newConfig, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
441441
kernel.getContext().waitForPublishQueueToClear();
442442

443443
// main should be finished
@@ -484,7 +484,7 @@ void GIVEN_kernel_running_services_WHEN_merge_removes_service_THEN_removed_servi
484484
lifecycle.put(LIFECYCLE_RUN_NAMESPACE_TOPIC,
485485
((String) lifecycle.get(LIFECYCLE_RUN_NAMESPACE_TOPIC)).replace("5", "10"));
486486

487-
Future<DeploymentResult> deploymentFuture = deploymentConfigMerger.mergeInNewConfig(testDeployment(), currentConfig);
487+
Future<DeploymentResult> deploymentFuture = deploymentConfigMerger.mergeInNewConfig(testDeployment(), currentConfig, System.currentTimeMillis());
488488

489489
DeploymentResult deploymentResult = deploymentFuture.get(30, TimeUnit.SECONDS);
490490

@@ -588,7 +588,7 @@ public void onStreamClosed() {
588588
Map<String, Object> currentConfig = new HashMap<>(kernel.getConfig().toPOJO());
589589
Deployment testDeployment = testDeployment();
590590
Future<DeploymentResult> future =
591-
deploymentConfigMerger.mergeInNewConfig(testDeployment, currentConfig);
591+
deploymentConfigMerger.mergeInNewConfig(testDeployment, currentConfig, System.currentTimeMillis());
592592

593593
// update should be deferred for 5 seconds
594594
assertThrows(TimeoutException.class, () -> future.get(5, TimeUnit.SECONDS),
@@ -632,7 +632,7 @@ void GIVEN_kernel_running_single_service_WHEN_deployment_with_skip_safety_check_
632632
}});
633633
put(DEFAULT_NUCLEUS_COMPONENT_NAME, getNucleusConfig());
634634
}});
635-
}}).get(60, TimeUnit.SECONDS);
635+
}}, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
636636

637637
// THEN
638638
mainRestarted.run();
@@ -696,7 +696,7 @@ void GIVEN_kernel_running_service_WHEN_run_with_change_THEN_service_restarts() t
696696

697697
put(DEFAULT_NUCLEUS_COMPONENT_NAME, getNucleusConfig());
698698
}});
699-
}}).get(60, TimeUnit.SECONDS);
699+
}}, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
700700

701701
// THEN
702702
serviceRestarts.run();

‎src/integrationtests/java/com/aws/greengrass/integrationtests/deployment/DeploymentTaskIntegrationTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import com.aws.greengrass.deployment.DeploymentDirectoryManager;
2121
import com.aws.greengrass.deployment.DeploymentDocumentDownloader;
2222
import com.aws.greengrass.deployment.DeploymentService;
23+
import com.aws.greengrass.deployment.DeviceConfiguration;
2324
import com.aws.greengrass.deployment.ThingGroupHelper;
2425
import com.aws.greengrass.deployment.exceptions.ServiceUpdateException;
2526
import com.aws.greengrass.deployment.model.Deployment;
@@ -1306,7 +1307,7 @@ private Future<DeploymentResult> submitSampleJobDocument(URI uri, Long timestamp
13061307
new DefaultDeploymentTask(dependencyResolver, componentManager, kernelConfigResolver,
13071308
deploymentConfigMerger, logger,
13081309
new Deployment(sampleJobDocument, Deployment.DeploymentType.IOT_JOBS, "jobId", DEFAULT),
1309-
deploymentServiceTopics, kernel.getContext().get(ExecutorService.class), deploymentDocumentDownloader, thingGroupHelper);
1310+
deploymentServiceTopics, kernel.getContext().get(ExecutorService.class), deploymentDocumentDownloader, thingGroupHelper, kernel.getContext().get(DeviceConfiguration.class));
13101311
return executorService.submit(deploymentTask);
13111312
}
13121313
}

‎src/integrationtests/java/com/aws/greengrass/integrationtests/deployment/DynamicComponentConfigurationValidationTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ void before(ExtensionContext context) throws Exception {
130130
}});
131131
}});
132132
}};
133-
deploymentConfigMerger.mergeInNewConfig(createTestDeployment(), newConfig).get(60, TimeUnit.SECONDS);
133+
deploymentConfigMerger.mergeInNewConfig(createTestDeployment(), newConfig, System.currentTimeMillis()).get(60, TimeUnit.SECONDS);
134134

135135
assertTrue(mainRestarted.get());
136136
assertTrue(serviceStarted.get());
@@ -217,7 +217,7 @@ public void onStreamClosed() {
217217
put(DEFAULT_NUCLEUS_COMPONENT_NAME, getNucleusConfig(kernel));
218218
}});
219219
}};
220-
DeploymentResult result = deploymentConfigMerger.mergeInNewConfig(createTestDeployment(), newConfig)
220+
DeploymentResult result = deploymentConfigMerger.mergeInNewConfig(createTestDeployment(), newConfig, System.currentTimeMillis())
221221
.get(60, TimeUnit.SECONDS);
222222
assertEquals(DeploymentResult.DeploymentStatus.SUCCESSFUL, result.getDeploymentStatus());
223223
assertTrue(eventReceivedByClient.await(20, TimeUnit.SECONDS));
@@ -289,7 +289,7 @@ public void onStreamClosed() {
289289
put(DEFAULT_NUCLEUS_COMPONENT_NAME, getNucleusConfig(kernel));
290290
}});
291291
}};
292-
DeploymentResult result = deploymentConfigMerger.mergeInNewConfig(createTestDeployment(), newConfig)
292+
DeploymentResult result = deploymentConfigMerger.mergeInNewConfig(createTestDeployment(), newConfig, System.currentTimeMillis())
293293
.get(60, TimeUnit.SECONDS);
294294
assertEquals(DeploymentResult.DeploymentStatus.FAILED_NO_STATE_CHANGE, result.getDeploymentStatus());
295295
assertTrue(result.getFailureCause() instanceof ComponentConfigurationValidationException);

‎src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/PluginComponentTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import com.aws.greengrass.deployment.DeploymentDirectoryManager;
2121
import com.aws.greengrass.deployment.DeploymentDocumentDownloader;
2222
import com.aws.greengrass.deployment.DeploymentService;
23+
import com.aws.greengrass.deployment.DeviceConfiguration;
2324
import com.aws.greengrass.deployment.ThingGroupHelper;
2425
import com.aws.greengrass.deployment.activator.KernelUpdateActivator;
2526
import com.aws.greengrass.deployment.bootstrap.BootstrapManager;
@@ -116,7 +117,7 @@ private static Future<DeploymentResult> submitSampleJobDocument(DeploymentDocume
116117
deploymentConfigMerger, LogManager.getLogger("Deployer"),
117118
new Deployment(sampleJobDocument, Deployment.DeploymentType.IOT_JOBS, "jobId", DEFAULT),
118119
Topics.of(kernel.getContext(), DeploymentService.DEPLOYMENT_SERVICE_TOPICS, null),
119-
kernel.getContext().get(ExecutorService.class), deploymentDocumentDownloader, thingGroupHelper);
120+
kernel.getContext().get(ExecutorService.class), deploymentDocumentDownloader, thingGroupHelper, kernel.getContext().get(DeviceConfiguration.class));
120121
return kernel.getContext().get(ExecutorService.class).submit(deploymentTask);
121122
}
122123

‎src/integrationtests/java/com/aws/greengrass/integrationtests/lifecyclemanager/ServiceDependencyLifecycleTest.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ void GIVEN_hard_dependency_WHEN_dependency_goes_through_lifecycle_events_THEN_cu
201201
when(doc1.getFailureHandlingPolicy()).thenReturn(FailureHandlingPolicy.DO_NOTHING);
202202

203203
testRoutine(TEST_ROUTINE_LONG_TIMEOUT, kernel,
204-
() -> configMerger.mergeInNewConfig(createMockDeployment(doc1), configRemoveDep).get(60,
204+
() -> configMerger.mergeInNewConfig(createMockDeployment(doc1), configRemoveDep, System.currentTimeMillis()).get(60,
205205
TimeUnit.SECONDS),
206206
"dependency removed", expectedDepRemoved, unexpectedDepRemoved);
207207

@@ -219,7 +219,7 @@ void GIVEN_hard_dependency_WHEN_dependency_goes_through_lifecycle_events_THEN_cu
219219
when(doc2.getFailureHandlingPolicy()).thenReturn(FailureHandlingPolicy.DO_NOTHING);
220220

221221
testRoutine(60, kernel,
222-
() -> configMerger.mergeInNewConfig(createMockDeployment(doc2), configAddDep).get(10, TimeUnit.SECONDS),
222+
() -> configMerger.mergeInNewConfig(createMockDeployment(doc2), configAddDep, System.currentTimeMillis()).get(10, TimeUnit.SECONDS),
223223
"dependency added", expectedDepAdded, Collections.emptySet());
224224

225225

@@ -314,7 +314,7 @@ void GIVEN_soft_dependency_WHEN_dependency_goes_through_lifecycle_events_THEN_cu
314314
when(doc1.getFailureHandlingPolicy()).thenReturn(FailureHandlingPolicy.DO_NOTHING);
315315

316316
testRoutine(TEST_ROUTINE_MEDIUM_TIMEOUT, kernel,
317-
() -> configMerger.mergeInNewConfig(createMockDeployment(doc1), configRemoveDep).get(30,
317+
() -> configMerger.mergeInNewConfig(createMockDeployment(doc1), configRemoveDep, System.currentTimeMillis()).get(30,
318318
TimeUnit.SECONDS),
319319
"dependency removed", expectedDepRemoved, unexpectedDuringAllSoftDepChange);
320320

@@ -333,7 +333,7 @@ void GIVEN_soft_dependency_WHEN_dependency_goes_through_lifecycle_events_THEN_cu
333333
when(doc2.getFailureHandlingPolicy()).thenReturn(FailureHandlingPolicy.DO_NOTHING);
334334

335335
testRoutine(TEST_ROUTINE_LONG_TIMEOUT, kernel,
336-
() -> configMerger.mergeInNewConfig(createMockDeployment(doc2), configAddDep).get(60, TimeUnit.SECONDS),
336+
() -> configMerger.mergeInNewConfig(createMockDeployment(doc2), configAddDep, System.currentTimeMillis()).get(60, TimeUnit.SECONDS),
337337
"dependency added", expectedDepAdded, Collections.emptySet());
338338

339339

@@ -402,7 +402,7 @@ void WHEN_dependency_type_changes_with_no_other_updates_THEN_customer_app_should
402402
when(doc2.getFailureHandlingPolicy()).thenReturn(FailureHandlingPolicy.DO_NOTHING);
403403

404404
testRoutine(5, kernel,
405-
() -> configMerger.mergeInNewConfig(createMockDeployment(doc2), depTypeSoftToHard).get(10, TimeUnit.SECONDS),
405+
() -> configMerger.mergeInNewConfig(createMockDeployment(doc2), depTypeSoftToHard, System.currentTimeMillis()).get(10, TimeUnit.SECONDS),
406406
"dependency type changes from soft to hard", new LinkedList<>(), new HashSet<>(stateTransitions));
407407

408408

@@ -416,7 +416,7 @@ void WHEN_dependency_type_changes_with_no_other_updates_THEN_customer_app_should
416416
when(doc1.getFailureHandlingPolicy()).thenReturn(FailureHandlingPolicy.DO_NOTHING);
417417

418418
testRoutine(5, kernel,
419-
() -> configMerger.mergeInNewConfig(createMockDeployment(doc1), depTypeHardToSoft).get(10, TimeUnit.SECONDS),
419+
() -> configMerger.mergeInNewConfig(createMockDeployment(doc1), depTypeHardToSoft, System.currentTimeMillis()).get(10, TimeUnit.SECONDS),
420420
"dependency type changes from hard to soft", new LinkedList<>(), new HashSet<>(stateTransitions));
421421
}
422422

‎src/main/java/com/aws/greengrass/componentmanager/KernelConfigResolver.java‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -143,16 +143,20 @@ public KernelConfigResolver(ComponentStore componentStore, Kernel kernel, Nucleu
143143
* @param componentsToDeploy package identifiers for resolved packages of complete dependency graph across groups
144144
* @param document deployment document
145145
* @param rootPackages root level packages
146+
* @param configMergeTimestamp timestamp to use for configuration merge
146147
* @return a kernel config map
147148
* @throws PackageLoadingException if any service package was unable to be loaded
148149
* @throws IOException for directory issues
149150
*/
150151
public Map<String, Object> resolve(List<ComponentIdentifier> componentsToDeploy, DeploymentDocument document,
151-
List<String> rootPackages) throws PackageLoadingException, IOException {
152+
List<String> rootPackages, long configMergeTimestamp) throws PackageLoadingException, IOException {
152153
Map<String, Object> servicesConfig = new HashMap<>();
154+
LOGGER.atDebug().kv("Components to deploy", componentsToDeploy).kv("Root packages", rootPackages)
155+
.log("Resolving services configuration");
153156
// resolve configuration
154157
for (ComponentIdentifier componentToDeploy : componentsToDeploy) {
155-
servicesConfig.put(componentToDeploy.getName(), getServiceConfig(componentToDeploy, document));
158+
servicesConfig.put(componentToDeploy.getName(), getServiceConfig(componentToDeploy, document,
159+
configMergeTimestamp));
156160
}
157161

158162
// Interpolate configurations
@@ -185,6 +189,7 @@ public Map<String, Object> resolve(List<ComponentIdentifier> componentsToDeploy,
185189
servicesConfig.putIfAbsent(nucleusComponentName, getNucleusComponentConfig(nucleusComponentName));
186190
servicesConfig.put(kernel.getMain().getName(), getMainConfig(rootPackages, nucleusComponentName));
187191

192+
LOGGER.atDebug().kv("Services Configuration", servicesConfig).log("Resolved services configuration.");
188193
// Services need to be under the services namespace in kernel config
189194
return Collections.singletonMap(SERVICES_NAMESPACE_TOPIC, servicesConfig);
190195
}
@@ -217,10 +222,12 @@ private boolean shouldInterpolateConfiguration(Map<String, Object> servicesConfi
217222
*
218223
* @param componentIdentifier target component id
219224
* @param document deployment doc for the current deployment
225+
* @param configMergeTimestamp timestamp to use during configuration merge
220226
* @return a built map representing the kernel config under "services" key for a particular component
221227
* @throws PackageLoadingException if any service package was unable to be loaded
222228
*/
223-
private Map<String, Object> getServiceConfig(ComponentIdentifier componentIdentifier, DeploymentDocument document)
229+
private Map<String, Object> getServiceConfig(ComponentIdentifier componentIdentifier, DeploymentDocument document,
230+
long configMergeTimestamp)
224231
throws PackageLoadingException {
225232

226233
ComponentRecipe componentRecipe = componentStore.getPackageRecipe(componentIdentifier);
@@ -262,7 +269,7 @@ private Map<String, Object> getServiceConfig(ComponentIdentifier componentIdenti
262269
}
263270

264271
Map<String, Object> resolvedConfiguration = resolveConfigurationToApply(optionalConfigUpdate.orElse(null),
265-
componentRecipe, document);
272+
componentRecipe, configMergeTimestamp);
266273

267274
// merge resolved param and resolved configuration for backward compatibility
268275
resolvedServiceConfig
@@ -333,13 +340,13 @@ private void updateRunWith(RunWith runWith, Map<String, Object> resolvedServiceC
333340
*
334341
* @param configurationUpdateOperation nullable component configuration update operation.
335342
* @param componentRecipe component recipe containing default configuration.
336-
* @param document deployment document
343+
* @param configMergeTimestamp timestamp to use during configuration merge
337344
* @return resolved configuration for this component. non null.
338345
*/
339346
@SuppressWarnings("PMD.ConfusingTernary")
340347
private Map<String, Object> resolveConfigurationToApply(
341348
@Nullable ConfigurationUpdateOperation configurationUpdateOperation, ComponentRecipe componentRecipe,
342-
DeploymentDocument document) {
349+
long configMergeTimestamp) {
343350

344351
// try read the running service config
345352
try (Context context = new Context()) {
@@ -379,7 +386,7 @@ private Map<String, Object> resolveConfigurationToApply(
379386

380387
// Merge in the requested config updates
381388
if (configurationUpdateOperation != null && configurationUpdateOperation.getValueToMerge() != null) {
382-
currentRunningConfig.mergeMap(document.getTimestamp(), configurationUpdateOperation.getValueToMerge());
389+
currentRunningConfig.mergeMap(configMergeTimestamp, configurationUpdateOperation.getValueToMerge());
383390
}
384391

385392
return currentRunningConfig.toPOJO();

0 commit comments

Comments
 (0)