Skip to content

Commit 8ed5602

Browse files
ayushaga14ayushaga14
authored
Merge pull request #3759 from akto-api-security/feature/cyborg-release-ignore-template-urls-fix
removing merged urls from the sti while fetching
2 parents 695de32 + f660c9f commit 8ed5602

File tree

5 files changed

+265
-17
lines changed

5 files changed

+265
-17
lines changed

‎apps/database-abstractor/src/main/java/com/akto/action/DbAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -621,7 +621,7 @@ public String bulkWriteSti() {
621621
}
622622

623623
// Filter for account 1759386565: ignore template URLs
624-
if (accId == 1759386565 && url != null && APICatalog.isTemplateUrl(url)) {
624+
if (accId == Constants.MERGED_URLS_FILTER_ACCOUNT_ID && url != null && APICatalog.isTemplateUrl(url)) {
625625
ignore = true;
626626
}
627627

@@ -717,7 +717,7 @@ public String bulkWriteSti() {
717717
}
718718

719719
// Filter for account 1759386565: ignore template URLs
720-
if (accId == 1759386565 && url != null && APICatalog.isTemplateUrl(url)) {
720+
if (accId == Constants.MERGED_URLS_FILTER_ACCOUNT_ID && url != null && APICatalog.isTemplateUrl(url)) {
721721
ignore = true;
722722
}
723723

‎apps/database-abstractor/src/test/java/com/akto/action/DbActionTest.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.akto.action;
22

33
import com.akto.dto.type.APICatalog;
4+
import com.akto.util.Constants;
45
import org.junit.Test;
56

67
import static org.junit.Assert.*;
@@ -131,33 +132,33 @@ public void testMixedUrlPatterns() {
131132
*/
132133
@Test
133134
public void testAccount1759386565FilteringLogic() {
134-
int accountId = 1759386565;
135+
int accountId = Constants.MERGED_URLS_FILTER_ACCOUNT_ID;
135136

136137
// Scenario 1: Template URL for account 1759386565 - should be ignored
137138
String templateUrl = "api/users/INTEGER";
138-
boolean shouldIgnore = (accountId == 1759386565 &&
139+
boolean shouldIgnore = (accountId == Constants.MERGED_URLS_FILTER_ACCOUNT_ID &&
139140
templateUrl != null &&
140141
APICatalog.isTemplateUrl(templateUrl));
141142
assertTrue("Template URL for account 1759386565 should be ignored", shouldIgnore);
142143

143144
// Scenario 2: Non-template URL for account 1759386565 - should NOT be ignored
144145
String regularUrl = "api/users/12345";
145-
shouldIgnore = (accountId == 1759386565 &&
146+
shouldIgnore = (accountId == Constants.MERGED_URLS_FILTER_ACCOUNT_ID &&
146147
regularUrl != null &&
147148
APICatalog.isTemplateUrl(regularUrl));
148149
assertFalse("Regular URL for account 1759386565 should NOT be ignored", shouldIgnore);
149150

150151
// Scenario 3: Template URL for different account - should NOT be ignored by this rule
151-
int differentAccountId = 1000000;
152-
shouldIgnore = (differentAccountId == 1759386565 &&
152+
int differentAccountId = 1000001;
153+
shouldIgnore = (differentAccountId == Constants.MERGED_URLS_FILTER_ACCOUNT_ID &&
153154
templateUrl != null &&
154155
APICatalog.isTemplateUrl(templateUrl));
155156
assertFalse("Template URL for different account should NOT be ignored by this rule",
156157
shouldIgnore);
157158

158159
// Scenario 4: Null URL for account 1759386565 - should NOT cause errors
159160
String nullUrl = null;
160-
shouldIgnore = (accountId == 1759386565 &&
161+
shouldIgnore = (accountId == Constants.MERGED_URLS_FILTER_ACCOUNT_ID &&
161162
nullUrl != null &&
162163
APICatalog.isTemplateUrl(nullUrl));
163164
assertFalse("Null URL should be handled gracefully", shouldIgnore);

‎apps/database-abstractor/src/test/java/com/akto/action/MergedUrlsFilterTest.java‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
package com.akto.action;
2+
3+
import com.akto.dto.filter.MergedUrls;
4+
import com.akto.util.Constants;
5+
import org.junit.Test;
6+
7+
import java.util.HashSet;
8+
import java.util.Set;
9+
10+
import static org.junit.Assert.*;
11+
12+
/**
13+
* Unit tests for merged URLs filtering logic in DbLayer.fetchStiBasedOnHostHeaders()
14+
* Tests the filtering that applies to account ID 1759386565 (Constants.MERGED_URLS_FILTER_ACCOUNT_ID)
15+
*
16+
* This test verifies that:
17+
* 1. MergedUrls equality works correctly based on (url, method, apiCollectionId)
18+
* 2. HashSet contains() method works properly with MergedUrls
19+
* 3. The filtering logic correctly identifies STIs that should be filtered out
20+
*/
21+
public class MergedUrlsFilterTest {
22+
23+
/**
24+
* Test that MergedUrls equality works correctly
25+
* This is critical for the Set.contains() check in the filtering logic
26+
*/
27+
@Test
28+
public void testMergedUrlsEquality() {
29+
MergedUrls url1 = new MergedUrls("api/users/123", "GET", 1000);
30+
MergedUrls url2 = new MergedUrls("api/users/123", "GET", 1000);
31+
MergedUrls url3 = new MergedUrls("api/users/456", "GET", 1000);
32+
MergedUrls url4 = new MergedUrls("api/users/123", "POST", 1000);
33+
MergedUrls url5 = new MergedUrls("api/users/123", "GET", 2000);
34+
35+
// Same url, method, and apiCollectionId should be equal
36+
assertEquals("Same parameters should be equal", url1, url2);
37+
assertEquals("Hash codes should match", url1.hashCode(), url2.hashCode());
38+
39+
// Different url should not be equal
40+
assertNotEquals("Different URL should not be equal", url1, url3);
41+
42+
// Different method should not be equal
43+
assertNotEquals("Different method should not be equal", url1, url4);
44+
45+
// Different apiCollectionId should not be equal
46+
assertNotEquals("Different apiCollectionId should not be equal", url1, url5);
47+
}
48+
49+
/**
50+
* Test that HashSet contains() works with MergedUrls
51+
* This simulates the core filtering logic: mergedUrlsSet.contains(stiAsUrl)
52+
*/
53+
@Test
54+
public void testMergedUrlsSetContains() {
55+
Set<MergedUrls> mergedUrlsSet = new HashSet<>();
56+
57+
// Add some merged URLs to the set
58+
mergedUrlsSet.add(new MergedUrls("api/users/INTEGER", "GET", 1000));
59+
mergedUrlsSet.add(new MergedUrls("api/products/STRING", "POST", 1000));
60+
mergedUrlsSet.add(new MergedUrls("api/orders/OBJECT_ID", "GET", 2000));
61+
62+
// Test that contains() works for exact matches
63+
assertTrue("Should find exact match",
64+
mergedUrlsSet.contains(new MergedUrls("api/users/INTEGER", "GET", 1000)));
65+
66+
assertTrue("Should find exact match with different URL",
67+
mergedUrlsSet.contains(new MergedUrls("api/products/STRING", "POST", 1000)));
68+
69+
// Test that contains() returns false for non-matches
70+
assertFalse("Should not find with different URL",
71+
mergedUrlsSet.contains(new MergedUrls("api/users/123", "GET", 1000)));
72+
73+
assertFalse("Should not find with different method",
74+
mergedUrlsSet.contains(new MergedUrls("api/users/INTEGER", "POST", 1000)));
75+
76+
assertFalse("Should not find with different apiCollectionId",
77+
mergedUrlsSet.contains(new MergedUrls("api/users/INTEGER", "GET", 3000)));
78+
79+
assertFalse("Should not find URL not in set",
80+
mergedUrlsSet.contains(new MergedUrls("api/not/exists", "GET", 1000)));
81+
}
82+
83+
/**
84+
* Test filtering logic - determines which STIs should be kept vs filtered out
85+
*/
86+
@Test
87+
public void testMergedUrlsFilteringLogic() {
88+
// Create merged URLs set (simulates what's cached in DbLayer)
89+
Set<MergedUrls> mergedUrlsSet = new HashSet<>();
90+
mergedUrlsSet.add(new MergedUrls("api/users/INTEGER", "GET", 100));
91+
mergedUrlsSet.add(new MergedUrls("api/products/STRING", "POST", 100));
92+
mergedUrlsSet.add(new MergedUrls("api/orders/INTEGER", "GET", 200));
93+
94+
// Test Case 1: STI matching merged URL - should be FILTERED OUT
95+
MergedUrls sti1 = new MergedUrls("api/users/INTEGER", "GET", 100);
96+
boolean shouldFilter1 = mergedUrlsSet.contains(sti1);
97+
assertTrue("STI matching merged URL should be filtered", shouldFilter1);
98+
99+
// Test Case 2: STI with concrete URL - should be KEPT
100+
MergedUrls sti2 = new MergedUrls("api/users/123", "GET", 100);
101+
boolean shouldFilter2 = mergedUrlsSet.contains(sti2);
102+
assertFalse("STI with concrete URL should not be filtered", shouldFilter2);
103+
104+
// Test Case 3: STI matching merged URL - should be FILTERED OUT
105+
MergedUrls sti3 = new MergedUrls("api/products/STRING", "POST", 100);
106+
boolean shouldFilter3 = mergedUrlsSet.contains(sti3);
107+
assertTrue("STI matching merged URL should be filtered", shouldFilter3);
108+
109+
// Test Case 4: STI with different method - should be KEPT
110+
MergedUrls sti4 = new MergedUrls("api/users/INTEGER", "POST", 100);
111+
boolean shouldFilter4 = mergedUrlsSet.contains(sti4);
112+
assertFalse("STI with different method should not be filtered", shouldFilter4);
113+
114+
// Test Case 5: STI with different collection - should be KEPT
115+
MergedUrls sti5 = new MergedUrls("api/users/INTEGER", "GET", 300);
116+
boolean shouldFilter5 = mergedUrlsSet.contains(sti5);
117+
assertFalse("STI with different apiCollectionId should not be filtered", shouldFilter5);
118+
119+
// Test Case 6: STI matching merged URL in different collection - should be FILTERED OUT
120+
MergedUrls sti6 = new MergedUrls("api/orders/INTEGER", "GET", 200);
121+
boolean shouldFilter6 = mergedUrlsSet.contains(sti6);
122+
assertTrue("STI matching merged URL in collection 200 should be filtered", shouldFilter6);
123+
}
124+
125+
/**
126+
* Test realistic scenario with template URLs
127+
*/
128+
@Test
129+
public void testRealisticTemplateUrlScenario() {
130+
Set<MergedUrls> mergedUrlsSet = new HashSet<>();
131+
132+
// Add merged template URLs (these would come from the merged_urls collection)
133+
mergedUrlsSet.add(new MergedUrls("api/v1/users/INTEGER", "GET", Constants.MERGED_URLS_FILTER_ACCOUNT_ID));
134+
mergedUrlsSet.add(new MergedUrls("api/v1/users/INTEGER/posts", "GET", Constants.MERGED_URLS_FILTER_ACCOUNT_ID));
135+
mergedUrlsSet.add(new MergedUrls("api/v1/products/STRING", "POST", Constants.MERGED_URLS_FILTER_ACCOUNT_ID));
136+
137+
// Template URLs matching merged entries should be filtered
138+
assertTrue("Template URL matching merged entry should be filtered",
139+
mergedUrlsSet.contains(new MergedUrls("api/v1/users/INTEGER", "GET", Constants.MERGED_URLS_FILTER_ACCOUNT_ID)));
140+
141+
assertTrue("Template URL with path matching merged entry should be filtered",
142+
mergedUrlsSet.contains(new MergedUrls("api/v1/users/INTEGER/posts", "GET", Constants.MERGED_URLS_FILTER_ACCOUNT_ID)));
143+
144+
// Concrete URLs should NOT be filtered
145+
assertFalse("Concrete URL should not be filtered",
146+
mergedUrlsSet.contains(new MergedUrls("api/v1/users/123", "GET", Constants.MERGED_URLS_FILTER_ACCOUNT_ID)));
147+
148+
assertFalse("Concrete URL with path should not be filtered",
149+
mergedUrlsSet.contains(new MergedUrls("api/v1/users/123/posts", "GET", Constants.MERGED_URLS_FILTER_ACCOUNT_ID)));
150+
151+
// Same URL with different method should NOT be filtered
152+
assertFalse("Same URL with different method should not be filtered",
153+
mergedUrlsSet.contains(new MergedUrls("api/v1/users/INTEGER", "POST", Constants.MERGED_URLS_FILTER_ACCOUNT_ID)));
154+
}
155+
156+
/**
157+
* Test edge cases
158+
*/
159+
@Test
160+
public void testEdgeCases() {
161+
Set<MergedUrls> mergedUrlsSet = new HashSet<>();
162+
163+
// Empty set - nothing should be filtered
164+
assertFalse("Empty set should not contain any URL",
165+
mergedUrlsSet.contains(new MergedUrls("api/test", "GET", 100)));
166+
167+
// Add a URL and test
168+
mergedUrlsSet.add(new MergedUrls("api/test", "GET", 100));
169+
assertTrue("Should find the URL after adding",
170+
mergedUrlsSet.contains(new MergedUrls("api/test", "GET", 100)));
171+
172+
// Test with null URL
173+
mergedUrlsSet.add(new MergedUrls(null, "GET", 100));
174+
assertTrue("Should handle null URL",
175+
mergedUrlsSet.contains(new MergedUrls(null, "GET", 100)));
176+
}
177+
178+
/**
179+
* Test account ID constant value
180+
*/
181+
@Test
182+
public void testAccountIdConstant() {
183+
// Verify the expected account ID
184+
int expectedAccountId = Constants.MERGED_URLS_FILTER_ACCOUNT_ID;
185+
186+
// This test documents the account ID that triggers merged URLs filtering
187+
assertEquals("Expected account ID for merged URLs filtering",
188+
Constants.MERGED_URLS_FILTER_ACCOUNT_ID, expectedAccountId);
189+
}
190+
}

‎libs/dao/src/main/java/com/akto/util/Constants.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,7 @@ private Constants() {}
3939
public static final List<String> MERGING_ALLOWED_TAG_VALUES = Arrays.asList("agoda-routing");
4040
public static final List<String> MERGING_ALLOWED_TAG_SUBSTRINGS = Arrays.asList("proxy", "gateway");
4141
public static final String AKTO_GEN_AI_TAG = "gen-ai";
42+
43+
// Account ID for filtering merged URLs in STI fetch
44+
public static final int MERGED_URLS_FILTER_ACCOUNT_ID = 1759386565;
4245
}

‎libs/utils/src/main/java/com/akto/data_actor/DbLayer.java‎

Lines changed: 63 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,11 @@ public class DbLayer {
132132
private static volatile long routingCollectionIdsCacheTimestamp = 0;
133133
private static final long CACHE_TTL_MS = 60 * 60 * 1000; // 60 minutes
134134

135+
// Cache for merged URLs filtering
136+
private static volatile Set<MergedUrls> mergedUrlsCache = null;
137+
private static volatile long mergedUrlsCacheTimestamp = 0;
138+
private static final long MERGED_URLS_CACHE_TTL_MS = 15 * 60 * 1000; // 15 minutes
139+
135140
private static int getLastUpdatedTsForAccount(int accountId) {
136141
return lastUpdatedTsMap.computeIfAbsent(accountId, k -> 0);
137142
}
@@ -341,11 +346,16 @@ public static List<SingleTypeInfo> fetchStiBasedOnHostHeaders(ObjectId objectId)
341346
List<SingleTypeInfo> results = new ArrayList<>();
342347
ObjectId currentObjectId = objectId;
343348
Set<Integer> routingCollectionIds = null;
349+
Set<MergedUrls> mergedUrlsSet = null;
344350

345351
if (Context.accountId.get() == Constants.ROUTING_SKIP_ACCOUNT_ID) {
346352
routingCollectionIds = getRoutingCollectionIds();
347353
}
348354

355+
if (Context.accountId.get() == Constants.MERGED_URLS_FILTER_ACCOUNT_ID) {
356+
mergedUrlsSet = getMergedUrlsSet();
357+
}
358+
349359
// Loop and discard template URLs from collections belonging to routing tags
350360
// Keep fetching until we get a non-empty list or exhaust documents
351361
while (results.isEmpty()) {
@@ -364,16 +374,32 @@ public static List<SingleTypeInfo> fetchStiBasedOnHostHeaders(ObjectId objectId)
364374
currentObjectId = batch.get(batch.size() - 1).getId();
365375

366376
// Filter out STRING template URLs from collections that should skip merging
367-
if (routingCollectionIds != null) {
377+
// Also filter out STIs belonging to merged URLs for specific account
378+
if (routingCollectionIds != null || mergedUrlsSet != null) {
368379
for (SingleTypeInfo sti : batch) {
369-
if (!routingCollectionIds.contains(sti.getApiCollectionId())) {
370-
// Not from skip-merging collection - keep everything
371-
results.add(sti);
372-
} else if (!APICatalog.isTemplateUrl(sti.getUrl())) {
373-
// From skip-merging collection but static URL - keep it
374-
results.add(sti);
375-
} else if (!APICatalog.isStringTemplateUrl(sti.getUrl())) {
376-
// From skip-merging collection but non-STRING template URL (INTEGER, LOCALE, etc.) - keep it
380+
boolean shouldKeep = true;
381+
382+
// Check routing collection filter
383+
if (routingCollectionIds != null) {
384+
if (routingCollectionIds.contains(sti.getApiCollectionId())) {
385+
// From skip-merging collection
386+
if (APICatalog.isTemplateUrl(sti.getUrl()) && APICatalog.isStringTemplateUrl(sti.getUrl())) {
387+
// STRING template URL from routing collection - discard
388+
shouldKeep = false;
389+
}
390+
}
391+
}
392+
393+
// Check merged URLs filter (only if not already filtered)
394+
if (shouldKeep && mergedUrlsSet != null && !mergedUrlsSet.isEmpty()) {
395+
MergedUrls stiAsUrl = new MergedUrls(sti.getUrl(), sti.getMethod(), sti.getApiCollectionId());
396+
if (mergedUrlsSet.contains(stiAsUrl)) {
397+
// STI belongs to merged URL - discard
398+
shouldKeep = false;
399+
}
400+
}
401+
402+
if (shouldKeep) {
377403
results.add(sti);
378404
}
379405
// From skip-merging collection AND STRING template URL - discard
@@ -425,6 +451,34 @@ private static Set<Integer> getRoutingCollectionIds() {
425451
}
426452
}
427453

454+
private static Set<MergedUrls> getMergedUrlsSet() {
455+
long currentTime = System.currentTimeMillis();
456+
457+
// Check if cache is valid
458+
if (mergedUrlsCache != null &&
459+
(currentTime - mergedUrlsCacheTimestamp) < MERGED_URLS_CACHE_TTL_MS) {
460+
return mergedUrlsCache;
461+
}
462+
463+
// Cache miss or expired - rebuild cache
464+
synchronized (DbLayer.class) {
465+
// Double-check after acquiring lock
466+
if (mergedUrlsCache != null &&
467+
(currentTime - mergedUrlsCacheTimestamp) < MERGED_URLS_CACHE_TTL_MS) {
468+
return mergedUrlsCache;
469+
}
470+
471+
// Fetch merged URLs from database
472+
Set<MergedUrls> mergedUrls = MergedUrlsDao.instance.getMergedUrls();
473+
474+
// Update cache
475+
mergedUrlsCache = mergedUrls;
476+
mergedUrlsCacheTimestamp = currentTime;
477+
478+
return mergedUrls;
479+
}
480+
}
481+
428482
public static List<Integer> fetchApiCollectionIds() {
429483
List<Integer> apiCollectionIds = new ArrayList<>();
430484
List<ApiCollection> apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject(),

0 commit comments

Comments
 (0)