Skip to content

feat:[NEXT-287] Direct aws, azure & gcp to the asset state service #2346

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kevin-paladin wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat:[NEXT-287] Direct aws, azure & gcp to the asset state service #2346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
08f17fc
Direct aws, azure & gcp to the asset state service
kevin-paladin Nov 22, 2024
14e14b9
Use the same event & format for the policy engine & asset state service
kevin-paladin Dec 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions commons/pac-batch-commons/src/main/java/com/tmobile/pacman/commons/dto/JobDoneMessage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,22 @@
******************************************************************************/
package com.tmobile.pacman.commons.dto;

import com.fasterxml.jackson.annotation.JsonProperty;

public class JobDoneMessage {
private String jobName;
// NOTE: 'paladinCloudTenantId' is currently sent by the legacy shipper while 'tenantId' is sent
// by the delta engine; it's not clear if different recipients will use what's available so
// both are sent here.
private String paladinCloudTenantId;
@JsonProperty("tenant_id")
private String tenantId;
private String source;
private String enricherSource;
@JsonProperty("tenant_name")
private String tenantName;
private String[] assetTypes;


public String getJobName() {
return jobName;
Expand Down Expand Up @@ -53,10 +64,20 @@ public void setEnricherSource(String enricherSource) {
this.enricherSource = enricherSource;
}

public String getTenantName() { return tenantName; }
public String[] getAssetTypes() { return assetTypes; }

public JobDoneMessage (String jobName, String paladinCloudTenantId, String source, String enricherSource) {
this(jobName, paladinCloudTenantId, source, enricherSource, null, null);
}

public JobDoneMessage (String jobName, String paladinCloudTenantId, String source, String enricherSource, String tenantName, String[] assetTypes) {
this.jobName = jobName;
this.paladinCloudTenantId = paladinCloudTenantId;
this.tenantId = paladinCloudTenantId;
this.tenantName = tenantName;
this.source = source;
this.enricherSource = enricherSource;
this.assetTypes = assetTypes;
}
Comment on lines +74 to 82
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot Dec 2, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Implement defensive copying for array parameter

The assetTypes array should be defensively copied to prevent external modification of the internal state.

     public JobDoneMessage (String jobName, String paladinCloudTenantId, String source, String enricherSource, String tenantName, String[] assetTypes) {
         this.jobName = jobName;
         this.paladinCloudTenantId = paladinCloudTenantId;
         this.tenantId = paladinCloudTenantId;
         this.tenantName = tenantName;
         this.source = source;
         this.enricherSource = enricherSource;
-        this.assetTypes = assetTypes;
+        this.assetTypes = assetTypes != null ? assetTypes.clone() : null;
     }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
public JobDoneMessage (String jobName, String paladinCloudTenantId, String source, String enricherSource, String tenantName, String[] assetTypes) {
this.jobName = jobName;
this.paladinCloudTenantId = paladinCloudTenantId;
this.tenantId = paladinCloudTenantId;
this.tenantName = tenantName;
this.source = source;
this.enricherSource = enricherSource;
this.assetTypes = assetTypes;
}
public JobDoneMessage (String jobName, String paladinCloudTenantId, String source, String enricherSource, String tenantName, String[] assetTypes) {
this.jobName = jobName;
this.paladinCloudTenantId = paladinCloudTenantId;
this.tenantId = paladinCloudTenantId;
this.tenantName = tenantName;
this.source = source;
this.enricherSource = enricherSource;
this.assetTypes = assetTypes != null ? assetTypes.clone() : null;
}

}
28 changes: 23 additions & 5 deletions jobs/pacman-data-shipper/src/main/java/com/tmobile/cso/pacman/datashipper/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
******************************************************************************/
package com.tmobile.cso.pacman.datashipper;

import com.tmobile.cso.pacman.datashipper.config.ConfigManager;
import com.tmobile.cso.pacman.datashipper.dto.DatasourceData;
import com.tmobile.cso.pacman.datashipper.entity.*;
import com.tmobile.cso.pacman.datashipper.es.ESManager;
Expand All @@ -23,11 +24,10 @@
import com.tmobile.pacman.commons.aws.sqs.SQSManager;
import com.tmobile.pacman.commons.dto.JobDoneMessage;
import com.tmobile.pacman.commons.jobs.PacmanJob;
import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

@PacmanJob(methodToexecute = "shipData", jobName = "data-shipper", desc = "Job to load data from s3 to OP", priority = 5)
public class Main implements Constants {
private static final Logger LOGGER = LoggerFactory.getLogger(Main.class);
Expand Down Expand Up @@ -107,10 +107,28 @@ public static Map<String, Object> shipData(Map<String, String> params) {
errorList.add(errorMap);
LOGGER.error("Error while updating stats", e);
}

// Route all assets through the asset state service, which will route to the policy engine
// once it's done processing.
SQSManager sqsManager = SQSManager.getInstance();
JobDoneMessage jobDoneMessage = new JobDoneMessage(ds + "-Shipper-Job", tenantId, ds, null);
String sqsMessageID = sqsManager.sendSQSMessage(jobDoneMessage, System.getenv("SHIPPER_SQS_QUEUE_URL"));
LOGGER.debug("Shipper done SQS message ID: {}", sqsMessageID);
boolean isAsset = ds.equalsIgnoreCase("azure")
|| ds.equalsIgnoreCase("aws")
|| ds.equalsIgnoreCase("gcp");
if (isAsset) {
List<String> assetTypes = new ArrayList<>(ConfigManager.getTypesWithDisplayName(ds).keySet());
Collections.sort(assetTypes);
JobDoneMessage jobDoneMessage = new JobDoneMessage(ds + "-Shipper-Job", tenantId,
ds, null, null, assetTypes.toArray(new String[0]));
String sqsMessageID =
sqsManager.sendSQSMessage(jobDoneMessage, System.getenv("ASSET_STATE_QUEUE_URL"));
LOGGER.debug("Done SQS message ID: {}", sqsMessageID);
} else {
JobDoneMessage jobDoneMessage = new JobDoneMessage(ds + "-Shipper-Job", tenantId, ds, null);
String sqsMessageID =
sqsManager.sendSQSMessage(jobDoneMessage, System.getenv("SHIPPER_SQS_QUEUE_URL"));
LOGGER.debug("Shipper done SQS message ID: {}", sqsMessageID);
}

Map<String, Object> status = ErrorManageUtil.formErrorCode(jobName, errorList);
LOGGER.info("Job Return Status {} ", status);
return status;
Expand Down