Merge pull request #3772 from harshithb3304/archival-disable-retention

Archival disable retention

Author: ayushaga14

apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/threat_configuration/ArchivalConfigComponent.jsx

@@ -5,8 +5,8 @@ import api from "../../../pages/threat_detection/api.js";
 import Dropdown from "../../../components/layouts/Dropdown.jsx";
 
 const ArchivalConfigComponent = ({ title, description }) => {
-    const [archivalDays, setArchivalDays] = useState(60);
-    const [archivalEnabled, setArchivalEnabled] = useState(false);
+    const [deletionDays, setDeletionDays] = useState(60);
+    const [deletionEnabled, setDeletionEnabled] = useState(false);
     const [isSaveDisabled, setIsSaveDisabled] = useState(true);
     const [isToggleChanged, setIsToggleChanged] = useState(false);
     const [isDaysChanged, setIsDaysChanged] = useState(false);
@@ -15,10 +15,10 @@ const ArchivalConfigComponent = ({ title, description }) => {
         const response = await api.fetchThreatConfiguration();
         const days = response?.threatConfiguration?.archivalDays;
         const value = days === 30 || days === 60 || days === 90 ? days : 60;
-        setArchivalDays(value);
+        setDeletionDays(value);
 
         const enabled = response?.threatConfiguration?.archivalEnabled || false;
-        setArchivalEnabled(enabled);
+        setDeletionEnabled(enabled);
 
         setIsSaveDisabled(true);
         setIsToggleChanged(false);
@@ -27,23 +27,23 @@ const ArchivalConfigComponent = ({ title, description }) => {
 
     const onSave = async () => {
         try {
-            // Save archival days if changed
+            // Save deletion days if changed
             if (isDaysChanged) {
                 const payload = {
-                    archivalDays: archivalDays
+                    archivalDays: deletionDays
                 };
                 await api.modifyThreatConfiguration(payload);
             }
 
-            // Toggle archival enabled if changed
+            // Toggle deletion enabled if changed
             if (isToggleChanged) {
-                await api.toggleArchivalEnabled(archivalEnabled);
+                await api.toggleArchivalEnabled(deletionEnabled);
             }
 
-            func.setToast(true, false, "Archival configuration saved successfully");
+            func.setToast(true, false, "Deletion configuration saved successfully");
             fetchData();
         } catch (error) {
-            func.setToast(true, true, "Error saving archival configuration");
+            func.setToast(true, true, "Error saving deletion configuration");
         }
     };
 
@@ -69,13 +69,13 @@ const ArchivalConfigComponent = ({ title, description }) => {
     }
 
     const onChange = (val) => {
-        setArchivalDays(val);
+        setDeletionDays(val);
         setIsDaysChanged(true);
         setIsSaveDisabled(false);
     };
 
     const onToggleEnabled = (val) => {
-        setArchivalEnabled(val);
+        setDeletionEnabled(val);
         setIsToggleChanged(true);
         setIsSaveDisabled(false);
     };
@@ -93,17 +93,17 @@ const ArchivalConfigComponent = ({ title, description }) => {
             <LegacyCard.Section>
                 <VerticalStack gap="4">
                     <Checkbox
-                        label="Enable archival cron"
-                        checked={archivalEnabled}
+                        label="Enable deletion cron"
+                        checked={deletionEnabled}
                         onChange={onToggleEnabled}
-                        helpText="When enabled, malicious events older than the configured archival time will be automatically archived."
+                        helpText="When enabled, malicious events older than the configured retention time will be automatically deleted."
                     />
                     <Box width="200px">
                         <Dropdown
                             menuItems={options}
                             selected={(val) => onChange(val)}
-                            label="Archival Time"
-                            initial={() => archivalDays}
+                            label="Retention Time"
+                            initial={() => `${deletionDays} days`}
                         />
                     </Box>
                 </VerticalStack>

apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/threat_configuration/ThreatConfiguration.jsx

@@ -17,9 +17,9 @@ function ThreatConfiguration() {
             key={"ratelimitConfig"}
         />,
         <ArchivalConfigComponent
-            title={"Archival Configuration"}
-            description={"Choose how long to retain malicious events before archival."}
-            key={"archivalConfig"}
+            title={"Deletion Configuration"}
+            description={"Choose how long to retain malicious events before deletion."}
+            key={"deletionConfig"}
         />
     ];
 

apps/threat-detection-backend/src/main/java/com/akto/threat/backend/cron/ArchiveOldMaliciousEventsCron.java

@@ -2,7 +2,6 @@
 
 import com.akto.log.LoggerMaker;
 import com.akto.dao.context.Context;
-import com.akto.threat.backend.dao.ArchivedMaliciousEventDao;
 import com.akto.threat.backend.dao.MaliciousEventDao;
 import com.akto.threat.backend.dao.ThreatConfigurationDao;
 import com.mongodb.client.MongoClient;
@@ -29,8 +28,8 @@ public class ArchiveOldMaliciousEventsCron implements Runnable {
     private static final long DEFAULT_RETENTION_DAYS = 60L; // default, can be overridden from DB
     private static final long MIN_RETENTION_DAYS = 30L;
     private static final long MAX_RETENTION_DAYS = 90L;
-    private static final long MAX_SOURCE_DOCS = 400_000L; // cap size
     private static final long MAX_DELETES_PER_ITERATION = 100_000L; // cap per cron iteration
+    private static final long MAX_SOURCE_DOCS = 1000_000L;
 
     private final MongoClient mongoClient;
     private final ScheduledExecutorService scheduler;
@@ -97,19 +96,18 @@ private boolean shouldSkipDatabase(String dbName) {
     private void archiveOldMaliciousEvents(String dbName, long nowSeconds) {
         String accountId = dbName;
 
-        // Check if archival is enabled for this account
-        if (!isArchivalEnabled(accountId)) {
-            logger.infoAndAddToDb("Archival is disabled for account " + accountId + ", skipping", LoggerMaker.LogDb.RUNTIME);
+        // Check if deletion is enabled for this account
+        if (!isDeletionEnabled(accountId)) {
+            logger.infoAndAddToDb("Deletion is disabled for account " + accountId + ", skipping", LoggerMaker.LogDb.RUNTIME);
             return;
         }
 
         long retentionDays = fetchRetentionDays(accountId);
         long threshold = nowSeconds - (retentionDays * 24 * 60 * 60);
 
         MongoCollection<Document> source = MaliciousEventDao.instance.getDocumentCollection(accountId);
-        MongoCollection<Document> dest = ArchivedMaliciousEventDao.instance.getCollection(accountId);
 
-        int totalMoved = 0;
+        int totalDeleted = 0;
         long deletesThisIteration = 0L;
 
         while (true) {
@@ -132,14 +130,12 @@ private void archiveOldMaliciousEvents(String dbName, long nowSeconds) {
                 ids.add(doc.get("_id"));
             }
 
-            asyncUpsertToArchive(batch, accountId);
-
             long deleted = deleteByIds(source, ids, accountId);
-            totalMoved += (int) deleted;
+            totalDeleted += (int) deleted;
             deletesThisIteration += deleted;
 
             long iterationElapsedMs = (System.nanoTime() - iterationStartNanos) / 1_000_000L;
-            logger.infoAndAddToDb("Archive loop iteration in db " + dbName + ": batch=" + batch.size() + ", deleted=" + deleted + ", tookMs=" + iterationElapsedMs, LoggerMaker.LogDb.RUNTIME);
+            logger.infoAndAddToDb("Delete loop iteration in db " + dbName + ": batch=" + batch.size() + ", deleted=" + deleted + ", tookMs=" + iterationElapsedMs, LoggerMaker.LogDb.RUNTIME);
 
             if (batch.size() < BATCH_SIZE) {
                 break;
@@ -151,14 +147,13 @@ private void archiveOldMaliciousEvents(String dbName, long nowSeconds) {
             }
         }
 
-        if (totalMoved > 0) {
-            logger.infoAndAddToDb("Completed archiving for db " + dbName + ", total moved: " + totalMoved, LoggerMaker.LogDb.RUNTIME);
+        if (totalDeleted > 0) {
+            logger.infoAndAddToDb("Completed deletion for db " + dbName + ", total deleted: " + totalDeleted, LoggerMaker.LogDb.RUNTIME);
         }
 
-        // Enforce collection size cap by trimming oldest docs beyond 400k.
         try {
             if (deletesThisIteration < MAX_DELETES_PER_ITERATION) {
-                trimCollectionIfExceedsCap(accountId, source, dest);
+                trimCollectionIfExceedsCap(accountId, source);
             } else {
                 logger.infoAndAddToDb("Skipping trim step as delete cap reached in db " + dbName, LoggerMaker.LogDb.RUNTIME);
             }
@@ -167,7 +162,7 @@ private void archiveOldMaliciousEvents(String dbName, long nowSeconds) {
         }
     }
 
-    private boolean isArchivalEnabled(String accountId) {
+    private boolean isDeletionEnabled(String accountId) {
         try {
             Document doc = ThreatConfigurationDao.instance.getCollection(accountId).find().first();
             if (doc == null) return false; // disabled by default
@@ -199,7 +194,7 @@ private long fetchRetentionDays(String accountId) {
         return DEFAULT_RETENTION_DAYS;
     }
 
-    private void trimCollectionIfExceedsCap(String accountId, MongoCollection<Document> source, MongoCollection<Document> dest) {
+    private void trimCollectionIfExceedsCap(String accountId, MongoCollection<Document> source) {
         long approxCount = source.countDocuments();
 
         if (approxCount <= MAX_SOURCE_DOCS) return;
@@ -223,8 +218,6 @@ private void trimCollectionIfExceedsCap(String accountId, MongoCollection<Docume
 
             if (oldestDocs.isEmpty()) break;
 
-            asyncUpsertToArchive(oldestDocs, accountId);
-
             Set<Object> ids = new HashSet<>();
             for (Document d : oldestDocs) {
                 ids.add(d.get("_id"));
@@ -242,19 +235,6 @@ private void trimCollectionIfExceedsCap(String accountId, MongoCollection<Docume
         }
     }
 
-    private void asyncUpsertToArchive(List<Document> docs, String accountId) {
-        if (docs == null || docs.isEmpty()) return;
-        final List<Document> docsSnapshot = new ArrayList<>(docs);
-        final String accountIdFinal = accountId;
-        this.scheduler.submit(() -> {
-            try {
-                ArchivedMaliciousEventDao.instance.bulkInsert(accountIdFinal, docsSnapshot);
-            } catch (Exception e) {
-                logger.errorAndAddToDb("Async error writing archive batch in account " + accountIdFinal + ": " + e.getMessage(), LoggerMaker.LogDb.RUNTIME);
-            }
-        });
-    }
-
     private long deleteByIds(MongoCollection<Document> source, Set<Object> ids, String accountId) {
         if (ids == null || ids.isEmpty()) return 0L;
         try {