Skip to content

Commit 1401fe4

Browse files
tareq89tareq89
authored
Update elasticsearch location storage format (#11160)
* added getEventIndexWithoutLocationHierarchy before es query returning search result * index event with location hierarchy * Test full location-hierarchy indexing and hierarchy-stripping in event search results * Ensure minimum location hierarchy in tests and update reindex/indexing tests to validate full location hierarchy in Elasticsearch * refactor * update changelog.md
1 parent 6a9c23d commit 1401fe4

File tree

8 files changed

+463
-16
lines changed

8 files changed

+463
-16
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88

99
HTTP input now accepts `field('..')` references in the HTTP body definition.
1010

11+
#### Jurisdiction
12+
13+
- Elasticsearch now stores location IDs as a full administrative hierarchy, with the leaf representing the actual event location. This enables searching events by any jurisdiction level (district, province, office, health facility etc.).
14+
1115
## 1.9.1
1216

1317
### Breaking changes

‎packages/events/src/router/event/__snapshots__/event.search.test.ts.snap‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -874,6 +874,7 @@ exports[`User with my-jurisdiction scope can see events from other offices based
874874
exports[`User with my-jurisdiction scope only sees events created by system user to their primary office 1`] = `
875875
{
876876
"createdAt": "[sanitized]",
877+
"createdAtLocation": "[sanitized]",
877878
"createdBy": "[sanitized]",
878879
"createdByUserType": "[sanitized]",
879880
"dateOfEvent": "[sanitized]",

‎packages/events/src/router/event/event.search.test.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1976,3 +1976,67 @@ test('Returns events using nested AND/OR query combinations', async () => {
19761976
expect(matchingFirstnames).toContain('Bob')
19771977
expect(matchingEmails).toContain('[email protected]')
19781978
})
1979+
1980+
test('Search response contains single UUIDs for location fields, not arrays', async () => {
1981+
const { user, generator } = await setupTestCase()
1982+
1983+
const client = createTestClient(user, [
1984+
'search[event=tennis-club-membership,access=all]',
1985+
'record.create[event=birth|death|tennis-club-membership]',
1986+
'record.declare[event=birth|death|tennis-club-membership]'
1987+
])
1988+
const initialData = {
1989+
'applicant.name': {
1990+
firstname: 'John',
1991+
surname: 'Doe'
1992+
},
1993+
'applicant.dob': '2000-01-01',
1994+
'recommender.none': true,
1995+
'applicant.address': {
1996+
country: 'FAR',
1997+
addressType: AddressType.DOMESTIC,
1998+
administrativeArea: '27160bbd-32d1-4625-812f-860226bfb92a',
1999+
streetLevelDetails: {
2000+
state: 'state',
2001+
district2: 'district2'
2002+
}
2003+
}
2004+
} satisfies ActionUpdate
2005+
2006+
const event = await client.event.create(generator.event.create())
2007+
2008+
await client.event.actions.declare.request(
2009+
generator.event.actions.declare(event.id, {
2010+
declaration: initialData
2011+
})
2012+
)
2013+
2014+
const { results } = await client.event.search({
2015+
query: {
2016+
type: 'and',
2017+
clauses: [
2018+
{
2019+
eventType: TENNIS_CLUB_MEMBERSHIP,
2020+
data: {
2021+
'applicant.name': { type: 'fuzzy', term: 'John' }
2022+
}
2023+
}
2024+
]
2025+
}
2026+
})
2027+
2028+
expect(results).toHaveLength(1)
2029+
const result = results[0]
2030+
2031+
// --- ASSERTIONS ---
2032+
2033+
// createdAtLocation must be a string UUID
2034+
expect(typeof result.createdAtLocation).toBe('string')
2035+
expect(result.createdAtLocation).toMatch(/^[0-9a-fA-F-]{36}$/)
2036+
2037+
// updatedAtLocation must be a string UUID (or null if nothing updated)
2038+
if (result.updatedAtLocation) {
2039+
expect(typeof result.updatedAtLocation).toBe('string')
2040+
expect(result.updatedAtLocation).toMatch(/^[0-9a-fA-F-]{36}$/)
2041+
}
2042+
})

‎packages/events/src/service/events/reindex.test.ts‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,15 @@ import { http, HttpResponse } from 'msw'
1414
import {
1515
ActionStatus,
1616
ActionType,
17-
createPrng,
1817
EventDocument,
1918
EventIndex,
2019
generateEventDocument,
2120
getUUID,
21+
LocationType,
2222
SCOPES,
23-
TENNIS_CLUB_MEMBERSHIP
23+
TENNIS_CLUB_MEMBERSHIP,
24+
generateUuid,
25+
createPrng
2426
} from '@opencrvs/commons'
2527
import { tennisClubMembershipEvent } from '@opencrvs/commons/fixtures'
2628
import { createSystemTestClient, setupTestCase } from '@events/tests/utils'
@@ -30,6 +32,7 @@ import {
3032
} from '@events/storage/elasticsearch'
3133
import { mswServer } from '@events/tests/msw'
3234
import { env } from '@events/environment'
35+
import { getLocations } from '../locations/locations'
3336

3437
const spy = vi.fn()
3538

@@ -49,8 +52,38 @@ let rngNumber = 949
4952
beforeEach(async () => {
5053
mswServer.use(postHandler)
5154
spy.mockReset()
52-
const { user, eventsDb } = await setupTestCase()
55+
const { user, eventsDb, seed } = await setupTestCase()
56+
5357
const rng = createPrng(rngNumber)
58+
59+
const adminLevel1Id = generateUuid(rng)
60+
const adminLevel2Id = generateUuid(rng)
61+
const crvsOfficeId = generateUuid(rng)
62+
63+
await seed.locations([
64+
{
65+
name: 'Adming level 1',
66+
parentId: null,
67+
locationType: LocationType.enum.ADMIN_STRUCTURE,
68+
id: adminLevel1Id,
69+
validUntil: null
70+
},
71+
{
72+
name: 'Admin level 2',
73+
parentId: adminLevel1Id,
74+
locationType: LocationType.enum.ADMIN_STRUCTURE,
75+
id: adminLevel2Id,
76+
validUntil: null
77+
},
78+
{
79+
name: 'Admin level 2',
80+
parentId: adminLevel2Id,
81+
locationType: LocationType.enum.CRVS_OFFICE,
82+
id: crvsOfficeId,
83+
validUntil: null
84+
}
85+
])
86+
5487
rngNumber++
5588

5689
event = generateEventDocument({
@@ -62,6 +95,9 @@ beforeEach(async () => {
6295
]
6396
})
6497

98+
const locations = await getLocations()
99+
const crvsOffice = locations.find((x) => x.id === crvsOfficeId)
100+
65101
const draftDocument = generateEventDocument({
66102
configuration: tennisClubMembershipEvent,
67103
rng,
@@ -89,6 +125,7 @@ beforeEach(async () => {
89125
actionType: ActionType.CREATE,
90126
createdBy: user.id,
91127
createdAt: event.createdAt,
128+
createdAtLocation: crvsOffice?.id,
92129
status: ActionStatus.Accepted,
93130
createdByRole: user.role,
94131
createdByUserType: 'user',
@@ -105,6 +142,7 @@ beforeEach(async () => {
105142
actionType: ActionType.DECLARE,
106143
createdBy: user.id,
107144
createdAt: event.createdAt,
145+
createdAtLocation: crvsOffice?.id,
108146
status: ActionStatus.Accepted,
109147
createdByRole: user.role,
110148
createdByUserType: 'user',
@@ -132,6 +170,7 @@ beforeEach(async () => {
132170
eventId: drafteventInDb!.id,
133171
actionType: ActionType.CREATE,
134172
createdBy: user.id,
173+
createdAtLocation: crvsOffice?.id,
135174
createdAt: event.createdAt,
136175
status: ActionStatus.Accepted,
137176
createdByRole: user.role,
@@ -180,6 +219,12 @@ test('reindexing indexes all events into Elasticsearch', async () => {
180219
expect(spy.mock.calls[0]).toHaveLength(1)
181220
// Does not reindex draftDocument - thus only one record is indexed
182221
expect(body.hits.hits).toHaveLength(1)
222+
223+
// EventIndex locations contains locations hierarchy
224+
const eventIndex = body.hits.hits[0]._source as EventIndex
225+
expect(eventIndex.legalStatuses.DECLARED?.createdAtLocation).toHaveLength(3)
226+
expect(eventIndex.createdAtLocation).toHaveLength(3)
227+
expect(eventIndex.updatedAtLocation).toHaveLength(3)
183228
})
184229

185230
test('reindexing twice', async () => {
@@ -208,7 +253,11 @@ test('reindexing twice', async () => {
208253

209254
// Does not reindex draftDocument - thus only one record is indexed
210255
expect(body.hits.hits).toHaveLength(1)
211-
expect((body.hits.hits[0]._source as EventIndex).trackingId).toEqual(
212-
event.trackingId
213-
)
256+
257+
// EventIndex locations contains locations hierarchy
258+
const eventIndex = body.hits.hits[0]._source as EventIndex
259+
expect(eventIndex.legalStatuses.DECLARED?.createdAtLocation).toHaveLength(3)
260+
expect(eventIndex.createdAtLocation).toHaveLength(3)
261+
expect(eventIndex.updatedAtLocation).toHaveLength(3)
262+
expect(eventIndex.trackingId).toEqual(event.trackingId)
214263
})

‎packages/events/src/service/indexing/indexing.test.ts‎

Lines changed: 127 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,17 @@
1111
*/
1212

1313
import { tennisClubMembershipEvent } from '@opencrvs/commons/fixtures'
14-
import { QueryType, TENNIS_CLUB_MEMBERSHIP } from '@opencrvs/commons/events'
15-
import { createTestClient, setupTestCase } from '@events/tests/utils'
14+
import {
15+
createPrng,
16+
LocationType,
17+
QueryType,
18+
TENNIS_CLUB_MEMBERSHIP
19+
} from '@opencrvs/commons/events'
20+
import {
21+
createTestClient,
22+
setupTestCase,
23+
TEST_USER_DEFAULT_SCOPES
24+
} from '@events/tests/utils'
1625
import {
1726
getEventIndexName,
1827
getOrCreateClient
@@ -41,6 +50,122 @@ test('records are not indexed when they are created', async () => {
4150
expect(body.hits.hits).toHaveLength(0)
4251
})
4352

53+
test('records are indexed with full location hierarchy', async () => {
54+
const { user, generator, seed } = await setupTestCase()
55+
56+
const client = createTestClient(user, [
57+
...TEST_USER_DEFAULT_SCOPES,
58+
`search[event=${TENNIS_CLUB_MEMBERSHIP},access=all]`
59+
])
60+
const esClient = getOrCreateClient()
61+
62+
// --- Setup locations -------------------------------------------------------
63+
const locationRng = createPrng(842)
64+
65+
const parentLocation = {
66+
...generator.locations.set(1, locationRng)[0],
67+
locationType: LocationType.enum.ADMIN_STRUCTURE,
68+
name: 'Parent location'
69+
}
70+
71+
const childLocation = {
72+
...generator.locations.set(1, locationRng)[0],
73+
id: user.primaryOfficeId,
74+
parentId: parentLocation.id,
75+
name: 'Child location',
76+
locationType: LocationType.enum.CRVS_OFFICE
77+
}
78+
79+
await seed.locations([parentLocation, childLocation])
80+
81+
// --- Create & move event through lifecycle --------------------------------
82+
const createdEvent = await client.event.create(
83+
generator.event.create({ type: TENNIS_CLUB_MEMBERSHIP })
84+
)
85+
86+
const event = generator.event.actions.declare(createdEvent.id, {
87+
keepAssignment: true
88+
})
89+
90+
await client.event.actions.declare.request({
91+
...generator.event.actions.declare(createdEvent.id, {
92+
declaration: event.declaration,
93+
keepAssignment: true
94+
})
95+
})
96+
97+
await client.event.actions.validate.request(
98+
generator.event.actions.validate(createdEvent.id, {
99+
declaration: event.declaration,
100+
keepAssignment: true
101+
})
102+
)
103+
104+
// --- Verify indexed ES document contains full hierarchy -------------------
105+
const searchResponse = await esClient.search({
106+
index: getEventIndexName(TENNIS_CLUB_MEMBERSHIP),
107+
body: { query: { match_all: {} } }
108+
})
109+
110+
expect(searchResponse.hits.hits).toHaveLength(1)
111+
112+
expect(searchResponse.hits.hits[0]._source).toMatchObject({
113+
id: createdEvent.id,
114+
type: TENNIS_CLUB_MEMBERSHIP,
115+
status: 'VALIDATED',
116+
createdAtLocation: [parentLocation.id, childLocation.id],
117+
updatedAtLocation: [parentLocation.id, childLocation.id],
118+
legalStatuses: {
119+
DECLARED: {
120+
createdAtLocation: [parentLocation.id, childLocation.id]
121+
}
122+
},
123+
declaration: {
124+
applicant____address: {
125+
administrativeArea: [parentLocation.id, childLocation.id]
126+
}
127+
}
128+
})
129+
130+
// --- Search API result must NOT contain hierarchy -------------------------
131+
const { results } = await client.event.search({
132+
query: { type: 'and', clauses: [{ eventType: TENNIS_CLUB_MEMBERSHIP }] }
133+
})
134+
135+
expect(results).toHaveLength(1)
136+
expect(results[0].createdAtLocation).toEqual(childLocation.id)
137+
expect(results[0].updatedAtLocation).toEqual(childLocation.id)
138+
expect(results[0].legalStatuses.DECLARED?.createdAtLocation).toEqual(
139+
childLocation.id
140+
)
141+
expect(results[0].declaration.applicant____address).toEqual(undefined) // address is stripped of from api response
142+
143+
// --- Modify & re-search, hierarchy must STILL be stripped -----------------
144+
await client.event.actions.register.request({
145+
...generator.event.actions.register(createdEvent.id, {
146+
declaration: {
147+
...event.declaration,
148+
'applicant.name': {
149+
firstname: 'John',
150+
surname: 'Doe'
151+
}
152+
}
153+
})
154+
})
155+
156+
const { results: results2 } = await client.event.search({
157+
query: { type: 'and', clauses: [{ eventType: TENNIS_CLUB_MEMBERSHIP }] }
158+
})
159+
160+
expect(results2).toHaveLength(1)
161+
expect(results2[0].createdAtLocation).toEqual(childLocation.id)
162+
expect(results2[0].updatedAtLocation).toEqual(childLocation.id)
163+
expect(results2[0].legalStatuses.DECLARED?.createdAtLocation).toEqual(
164+
childLocation.id
165+
)
166+
expect(results2[0].declaration.applicant____address).toEqual(undefined) // address is stripped of from api response
167+
})
168+
44169
const RANDOM_UUID = '650a711b-a725-48f9-a92f-794b4a04fea6'
45170
const exactStatusPayload: QueryType = {
46171
type: 'and',

0 commit comments

Comments
 (0)