Skip to content

Commit ccef31b

Browse files
haiodohaiodo
authored
UBERF-9560: Filter query fixes (#8161)
Signed-off-by: Andrey Sobolev <[email protected]>
1 parent 67f776d commit ccef31b

File tree

3 files changed

+131
-32
lines changed

3 files changed

+131
-32
lines changed

Diff for: ‎server/postgres/src/__tests__/conversion.spec.ts

+45-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { convertArrayParams, decodeArray } from '../utils'
1+
import { convertArrayParams, decodeArray, filterProjection } from '../utils'
22

33
describe('array conversion', () => {
44
it('should handle undefined parameters', () => {
@@ -55,3 +55,47 @@ describe('array decoding', () => {
5555
expect(decodeArray('{"first \\"quote\\"","second \\"quote\\""}')).toEqual(['first "quote"', 'second "quote"'])
5656
})
5757
})
58+
59+
describe('projection', () => {
60+
it('mixin query projection', () => {
61+
const data = {
62+
'638611f18894c91979399ef3': {
63+
Источник_6386125d8894c91979399eff: 'Workable'
64+
},
65+
attachments: 1,
66+
avatar: null,
67+
avatarProps: null,
68+
avatarType: 'color',
69+
channels: 3,
70+
city: 'Poland',
71+
docUpdateMessages: 31,
72+
name: 'Mulkuha,Muklyi',
73+
'notification:mixin:Collaborators': {
74+
collaborators: []
75+
},
76+
'recruit:mixin:Candidate': {
77+
Title_63f38419efefd99805238bbd: 'Backend-RoR',
78+
Trash_64493626f9b50e77bf82d231: 'Нет',
79+
__mixin: 'true',
80+
applications: 1,
81+
onsite: null,
82+
remote: null,
83+
skills: 18,
84+
title: '',
85+
Опытработы_63860d5c8894c91979399e73: '2018',
86+
Уровеньанглийского_63860d038894c91979399e6f: 'UPPER'
87+
}
88+
}
89+
const projected = filterProjection<any>(data, {
90+
'recruit:mixin:Candidate.Уровеньанглийского_63860d038894c91979399e6f': 1,
91+
_class: 1,
92+
space: 1,
93+
modifiedOn: 1
94+
})
95+
expect(projected).toEqual({
96+
'recruit:mixin:Candidate': {
97+
Уровеньанглийского_63860d038894c91979399e6f: 'UPPER'
98+
}
99+
})
100+
})
101+
})

Diff for: ‎server/postgres/src/storage.ts

+59-24
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,12 @@ import core, {
6060
type WorkspaceId
6161
} from '@hcengineering/core'
6262
import {
63+
calcHashHash,
6364
type DbAdapter,
6465
type DbAdapterHandler,
6566
type DomainHelperOperations,
6667
type ServerFindOptions,
67-
type TxAdapter,
68-
calcHashHash
68+
type TxAdapter
6969
} from '@hcengineering/server-core'
7070
import type postgres from 'postgres'
7171
import { createDBClient, createGreenDBClient, type DBClient } from './client'
@@ -1304,7 +1304,11 @@ abstract class PostgresAdapterBase implements DbAdapter {
13041304
private translateQueryValue (vars: ValuesVariables, tkey: string, value: any, type: ValueType): string | undefined {
13051305
const tkeyData = tkey.includes('data') && (tkey.includes('->') || tkey.includes('#>>'))
13061306
if (tkeyData && (Array.isArray(value) || (typeof value !== 'object' && typeof value !== 'string'))) {
1307-
value = Array.isArray(value) ? value.map((it) => (it == null ? null : `${it}`)) : `${value}`
1307+
value = Array.isArray(value)
1308+
? value.map((it) => (it == null ? null : `${it}`))
1309+
: value == null
1310+
? null
1311+
: `${value}`
13081312
}
13091313

13101314
if (value === null) {
@@ -1315,76 +1319,87 @@ abstract class PostgresAdapterBase implements DbAdapter {
13151319
for (const operator in value) {
13161320
let val = value[operator]
13171321
if (tkeyData && (Array.isArray(val) || (typeof val !== 'object' && typeof val !== 'string'))) {
1318-
val = Array.isArray(val) ? val.map((it) => (it == null ? null : `${it}`)) : `${val}`
1322+
val = Array.isArray(val) ? val.map((it) => (it == null ? null : `${it}`)) : val == null ? null : `${val}`
13191323
}
1324+
1325+
let valType = inferType(val)
1326+
const { tlkey, arrowCount } = prepareJsonValue(tkey, valType)
1327+
if (arrowCount > 0 && valType === '::text') {
1328+
valType = ''
1329+
}
1330+
13201331
switch (operator) {
13211332
case '$ne':
1322-
if (val === null) {
1323-
res.push(`${tkey} IS NOT NULL`)
1333+
if (val == null) {
1334+
res.push(`${tlkey} IS NOT NULL`)
13241335
} else {
1325-
res.push(`${tkey} != ${vars.add(val, inferType(val))}`)
1336+
res.push(`${tlkey} != ${vars.add(val, valType)}`)
13261337
}
13271338
break
13281339
case '$gt':
1329-
res.push(`${tkey} > ${vars.add(val, inferType(val))}`)
1340+
res.push(`${tlkey} > ${vars.add(val, valType)}`)
13301341
break
13311342
case '$gte':
1332-
res.push(`${tkey} >= ${vars.add(val, inferType(val))}`)
1343+
res.push(`${tlkey} >= ${vars.add(val, valType)}`)
13331344
break
13341345
case '$lt':
1335-
res.push(`${tkey} < ${vars.add(val, inferType(val))}`)
1346+
res.push(`${tlkey} < ${vars.add(val, valType)}`)
13361347
break
13371348
case '$lte':
1338-
res.push(`${tkey} <= ${vars.add(val, inferType(val))}`)
1349+
res.push(`${tlkey} <= ${vars.add(val, valType)}`)
13391350
break
13401351
case '$in':
13411352
switch (type) {
13421353
case 'common':
13431354
if (Array.isArray(val) && val.includes(null)) {
1344-
const vv = vars.addArray(val, inferType(val))
1345-
res.push(`(${tkey} = ANY(${vv}) OR ${tkey} IS NULL)`)
1355+
const vv = vars.addArray(val, valType)
1356+
res.push(`(${tlkey} = ANY(${vv}) OR ${tkey} IS NULL)`)
13461357
} else {
13471358
if (val.length > 0) {
1348-
res.push(`${tkey} = ANY(${vars.addArray(val, inferType(val))})`)
1359+
res.push(`${tlkey} = ANY(${vars.addArray(val, valType)})`)
13491360
} else {
1350-
res.push(`${tkey} IN ('NULL')`)
1361+
res.push(`${tlkey} IN ('NULL')`)
13511362
}
13521363
}
13531364
break
13541365
case 'array':
13551366
{
1356-
const vv = vars.addArrayI(val, inferType(val))
1367+
const vv = vars.addArrayI(val, valType)
13571368
res.push(`${tkey} && ${vv}`)
13581369
}
13591370
break
13601371
case 'dataArray':
13611372
{
1362-
const vv = vars.addArrayI(val, inferType(val))
1373+
const vv = vars.addArrayI(val, valType)
13631374
res.push(`${tkey} ?| ${vv}`)
13641375
}
13651376
break
13661377
}
13671378
break
13681379
case '$nin':
13691380
if (Array.isArray(val) && val.includes(null)) {
1370-
res.push(`(${tkey} != ALL(${vars.addArray(val, inferType(val))}) AND ${tkey} IS NOT NULL)`)
1381+
res.push(`(${tlkey} != ALL(${vars.addArray(val, valType)}) AND ${tkey} IS NOT NULL)`)
13711382
} else if (Array.isArray(val) && val.length > 0) {
1372-
res.push(`${tkey} != ALL(${vars.addArray(val, inferType(val))})`)
1383+
res.push(`${tlkey} != ALL(${vars.addArray(val, valType)})`)
13731384
}
13741385
break
13751386
case '$like':
1376-
res.push(`${tkey} ILIKE ${vars.add(val, inferType(val))}`)
1387+
res.push(`${tlkey} ILIKE ${vars.add(val, valType)}`)
13771388
break
13781389
case '$exists':
1379-
res.push(`${tkey} IS ${val === true || val === 'true' ? 'NOT NULL' : 'NULL'}`)
1390+
res.push(`${tlkey} IS ${val === true || val === 'true' ? 'NOT NULL' : 'NULL'}`)
13801391
break
13811392
case '$regex':
1382-
res.push(`${tkey} SIMILAR TO ${vars.add(val, inferType(val))}`)
1393+
res.push(`${tlkey} SIMILAR TO ${vars.add(val, valType)}`)
13831394
break
13841395
case '$options':
13851396
break
13861397
case '$all':
1387-
res.push(`${tkey} @> ${vars.addArray(value, inferType(value))}`)
1398+
if (arrowCount > 0) {
1399+
res.push(`${tkey} @> '${JSON.stringify(val)}'::jsonb`)
1400+
} else {
1401+
res.push(`${tkey} @> ${vars.addArray(val, valType)}`)
1402+
}
13881403
break
13891404
default:
13901405
res.push(`${tkey} @> '[${JSON.stringify(value)}]'`)
@@ -1394,8 +1409,13 @@ abstract class PostgresAdapterBase implements DbAdapter {
13941409
return res.length === 0 ? undefined : res.join(' AND ')
13951410
}
13961411

1412+
let valType = inferType(value)
1413+
const { tlkey, arrowCount } = prepareJsonValue(tkey, valType)
1414+
if (arrowCount > 0 && valType === '::text') {
1415+
valType = ''
1416+
}
13971417
return type === 'common'
1398-
? `${tkey} = ${vars.add(value, inferType(value))}`
1418+
? `${tlkey} = ${vars.add(value, valType)}`
13991419
: type === 'array'
14001420
? `${tkey} @> '${typeof value === 'string' ? '{"' + value + '"}' : value}'`
14011421
: `${tkey} @> '${typeof value === 'string' ? '"' + value + '"' : value}'`
@@ -2093,6 +2113,21 @@ class PostgresTxAdapter extends PostgresAdapterBase implements TxAdapter {
20932113
return this.stripHash(systemTx.concat(userTx)) as Tx[]
20942114
}
20952115
}
2116+
function prepareJsonValue (tkey: string, valType: string): { tlkey: string, arrowCount: number } {
2117+
if (valType === '::string') {
2118+
valType = '' // No need to add a string conversion
2119+
}
2120+
const arrowCount = (tkey.match(/->/g) ?? []).length
2121+
// We need to convert to type without array if pressent
2122+
let tlkey = arrowCount > 0 ? `(${tkey})${valType.replace('[]', '')}` : tkey
2123+
2124+
if (arrowCount > 0) {
2125+
// We need to replace only the last -> to ->>
2126+
tlkey = arrowCount === 1 ? tlkey.replace('->', '->>') : tlkey.replace(/->(?!.*->)/, '->>')
2127+
}
2128+
return { tlkey, arrowCount }
2129+
}
2130+
20962131
/**
20972132
* @public
20982133
*/

Diff for: ‎server/postgres/src/utils.ts

+27-7
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,30 @@ export function convertArrayParams (parameters?: ParameterOrJSON<any>[]): any[]
556556
})
557557
}
558558

559+
export function filterProjection<T extends Doc> (data: any, projection: Projection<T> | undefined): any {
560+
for (const key in data) {
561+
if (!Object.prototype.hasOwnProperty.call(projection, key) || (projection as any)[key] === 0) {
562+
// check nested projections in case of object
563+
let value = data[key]
564+
if (typeof value === 'object' && !Array.isArray(value) && value != null) {
565+
// We need to filter projection for nested objects
566+
const innerP = Object.entries(projection as any)
567+
.filter((it) => it[0].startsWith(key))
568+
.map((it) => [it[0].substring(key.length + 1), it[1]])
569+
if (innerP.length > 0) {
570+
value = filterProjection(value, Object.fromEntries(innerP))
571+
data[key] = value
572+
continue
573+
}
574+
}
575+
576+
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
577+
delete data[key]
578+
}
579+
}
580+
return data
581+
}
582+
559583
export function parseDocWithProjection<T extends Doc> (
560584
doc: DBDoc,
561585
domain: string,
@@ -577,16 +601,12 @@ export function parseDocWithProjection<T extends Doc> (
577601
;(rest as any)[key] = decodeArray((rest as any)[key])
578602
}
579603
}
604+
let resultData = data
580605
if (projection !== undefined) {
581-
for (const key in data) {
582-
if (!Object.prototype.hasOwnProperty.call(projection, key) || (projection as any)[key] === 0) {
583-
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
584-
delete data[key]
585-
}
586-
}
606+
resultData = filterProjection(data, projection)
587607
}
588608
const res = {
589-
...data,
609+
...resultData,
590610
...rest
591611
} as any as T
592612

0 commit comments

Comments
 (0)