Skip to content

Commit aa3711a

Browse files
FelixKirschJWittmeyer
FelixKirsch
and
JWittmeyer
authored
Data browser filter extensions for datatypes (#79)
* fixes spelling mistakes * adds search operators between, greater, greater equal, less, less equal; and support for different data types * add support for LIKE, ILIKE and IN operator with wildcard support * Adds endpoint for recordComments & subquery search for comments * adds handling for empty values of filter_data * adds explicit handling for boolean values in is empty check * adds case sensitive operators * adds case sensitive operators to operator_has_quotes * enabels usage of * and ? wildcard operators * fixes typing, bug * Submodule change Co-authored-by: JWittmeyer <[email protected]>
1 parent 61ec871 commit aa3711a

File tree

6 files changed

+141
-40
lines changed

6 files changed

+141
-40
lines changed

‎controller/comment/manager.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,7 @@ def delete_comment(comment_id: str, user_id: str) -> CommentData:
122122
):
123123
raise ValueError(f"Can't delete comment")
124124
comments.remove(comment_id, with_commit=True)
125+
126+
127+
def get_record_comments(project_id: str, user_id: str, record_ids: List[str]):
128+
return comments.get_record_comments(project_id, user_id, record_ids)

‎graphql_api/query/record.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from controller.record import manager
1010
from controller.auth import manager as auth
1111
from controller.tokenization import manager as tokenization_manager
12+
from controller.comment import manager as comment_manager
1213

1314

1415
class RecordQuery(graphene.ObjectType):
@@ -52,6 +53,12 @@ class RecordQuery(graphene.ObjectType):
5253
record_id=graphene.ID(required=True),
5354
)
5455

56+
record_comments = graphene.Field(
57+
graphene.JSONString,
58+
project_id=graphene.ID(required=True),
59+
record_ids=graphene.List(graphene.ID, required=True),
60+
)
61+
5562
def resolve_all_records(self, info, project_id: str) -> List[Record]:
5663
auth.check_project_access(info, project_id)
5764
return manager.get_all_records(project_id)
@@ -112,3 +119,11 @@ def resolve_tokenize_record(self, info, record_id: str) -> TokenizedRecord:
112119
return tokenization_manager.get_tokenized_record(
113120
record_item.project_id, record_id
114121
)
122+
123+
def resolve_record_comments(
124+
self, info, project_id: str, record_ids: List[str]
125+
) -> str:
126+
auth.check_demo_access(info)
127+
auth.check_project_access(info, project_id)
128+
user_id = auth.get_user_id_by_info(info)
129+
return comment_manager.get_record_comments(project_id, user_id, record_ids)

‎service/search/search.py

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
import zlib
33
from typing import Tuple, Dict, List, Any, Optional, Union
44

5-
from exceptions.exceptions import TooManyRecordsForStaticSliceException
6-
from graphql_api import types
75
from graphql_api.types import ExtendedSearch
86
from submodules.model import UserSessions
97
from util.notification import create_notification
@@ -259,10 +257,10 @@ def __create_default_user_session_object(
259257
else:
260258
inner_sql = __build_inner_query(filter_data, project_id, 0, 0, False)
261259
id_sql_statement = __build_final_query(inner_sql, project_id, False, True)
262-
order_extention = __get_order_by(filter_data, project_id)
260+
order_extension = __get_order_by(filter_data, project_id)
263261

264-
if order_extention != "":
265-
id_sql_statement += order_extention
262+
if order_extension != "":
263+
id_sql_statement += order_extension
266264
else:
267265
id_sql_statement += "ORDER BY db_order"
268266
return UserSessionData(
@@ -328,10 +326,10 @@ def generate_select_sql(
328326
inner_sql = __build_inner_query(filter_data, project_id, limit, offset, False)
329327
final_sql = __build_final_query(inner_sql, project_id, False, for_id)
330328

331-
order_extention = __get_order_by(filter_data, project_id)
329+
order_extension = __get_order_by(filter_data, project_id)
332330

333-
if order_extention != "":
334-
final_sql += order_extention
331+
if order_extension != "":
332+
final_sql += order_extension
335333
else:
336334
final_sql += "ORDER BY db_order"
337335
return final_sql
@@ -360,7 +358,7 @@ def __build_base_query(
360358
else:
361359
order_by_add = __get_order_by(filter_data, project_id)
362360

363-
where_add = __build_where_add(filter_data)
361+
where_add = __build_where_add(project_id, filter_data)
364362

365363
tmp_selection_add, tmp_from_add = __build_subquery_data(
366364
filter_data, project_id, "WHITELIST"
@@ -454,17 +452,17 @@ def __build_subquery(
454452

455453

456454
def __build_where_add(
457-
filter_data: List[Dict[str, Any]], outer: Optional[bool] = True
455+
project_id: str, filter_data: List[Dict[str, Any]], outer: Optional[bool] = True
458456
) -> str:
459457
current_condition = ""
460458
for filter_element in filter_data:
461459
ret = ""
462460
if FilterDataDictKeys.OPERATOR.value in filter_element:
463-
ret = build_search_condition(filter_element)
461+
ret = build_search_condition(project_id, filter_element)
464462

465463
if FilterDataDictKeys.FILTER.value in filter_element:
466464
ret = __build_where_add(
467-
filter_element[FilterDataDictKeys.FILTER.value], False
465+
project_id, filter_element[FilterDataDictKeys.FILTER.value], False
468466
)
469467
if ret != "" and ret[0] != "(":
470468
ret = f"( {ret} )"

‎service/search/search_enum.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,21 @@ class SearchColumn(Enum):
2222
class SearchOperators(Enum):
2323
EQUAL = "EQUAL"
2424
BEGINS_WITH = "BEGINS_WITH"
25+
BEGINS_WITH_CS = "BEGINS_WITH_CS"
2526
ENDS_WITH = "ENDS_WITH"
27+
ENDS_WITH_CS = "ENDS_WITH_CS"
2628
CONTAINS = "CONTAINS"
29+
CONTAINS_CS = "CONTAINS_CS"
2730
IN = "IN"
31+
IN_WC = "IN_WC"
32+
IN_WC_CS = "IN_WC_CS"
33+
BETWEEN = "BETWEEN"
34+
GREATER = "GREATER"
35+
GREATER_EQUAL = "GREATER_EQUAL"
36+
LESS = "LESS"
37+
LESS_EQUAL = "LESS_EQUAL"
38+
LIKE = "LIKE"
39+
ILIKE = "ILIKE"
2840

2941

3042
class FilterDataDictKeys(Enum):
@@ -73,4 +85,5 @@ class SearchQueryTemplate(Enum):
7385
"SUBQUERY_RLA_DIFFERENT_IS_CLASSIFICATION"
7486
)
7587
SUBQUERY_RLA_DIFFERENT_IS_EXTRACTION = "SUBQUERY_RLA_DIFFERENT_IS_EXTRACTION"
76-
ORDER_RLA = "ORDER_RLA"
88+
ORDER_RLA = ("ORDER_RLA",)
89+
SUBQUERY_HAS_COMMENTS = "SUBQUERY_HAS_COMMENTS"

‎service/search/search_helper.py

Lines changed: 97 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -9,59 +9,94 @@
99
SearchTargetTables,
1010
)
1111
from submodules.model.enums import LabelSource
12+
from submodules.model.business_objects import attribute
13+
from submodules.model.enums import DataTypes
1214

1315

1416
def build_search_condition_value(target: SearchOperators, value) -> str:
1517
if target in __lookup_operator:
1618
operator = __lookup_operator[target]
1719
if target == SearchOperators.IN:
1820
all_in_values = ""
19-
for v in value:
20-
part = v
21-
if isinstance(v, str):
22-
part = "'" + part + "'"
23-
if all_in_values != "":
24-
all_in_values += ", " + part
25-
else:
26-
all_in_values = part
21+
if value:
22+
all_in_values = ", ".join(
23+
[f"'{v}'" if isinstance(v, str) else str(v) for v in value]
24+
)
2725
return operator.replace("@@VALUES@@", all_in_values)
26+
elif target == SearchOperators.BETWEEN:
27+
value1 = f"'{value[0]}'" if isinstance(value[0], str) else str(value[0])
28+
value2 = f"'{value[1]}'" if isinstance(value[1], str) else str(value[1])
29+
return operator.replace("@@VALUE1@@", value1).replace("@@VALUE2@@", value2)
2830
else:
2931
if target not in __lookup_operator_has_quotes and isinstance(value, str):
3032
value = "'" + value + "'"
31-
return operator.replace("@@VALUE@@", value)
33+
return operator.replace("@@VALUE@@", str(value))
3234
else:
3335
raise ValueError(target.value + " no operator info")
3436

3537

36-
def build_search_condition(filter_element: Dict[str, str]) -> str:
38+
def build_search_condition(project_id: str, filter_element: Dict[str, str]) -> str:
3739
table = SearchTargetTables[filter_element[FilterDataDictKeys.TARGET_TABLE.value]]
3840
column = SearchColumn[filter_element[FilterDataDictKeys.TARGET_COLUMN.value]]
39-
column_text = build_search_column_text(filter_element)
4041
operator = SearchOperators[filter_element[FilterDataDictKeys.OPERATOR.value]]
41-
42-
if operator == SearchOperators.IN:
43-
if table == SearchTargetTables.RECORD and column == SearchColumn.DATA:
44-
filter_values = filter_element[FilterDataDictKeys.VALUES.value][1:]
42+
attr_name = filter_element[FilterDataDictKeys.VALUES.value][0]
43+
filter_values = filter_element[FilterDataDictKeys.VALUES.value]
44+
search_column = build_search_column(project_id, table, column, operator, attr_name)
45+
46+
if operator in [SearchOperators.IN_WC, SearchOperators.IN_WC_CS]:
47+
conditions = []
48+
if operator == SearchOperators.IN_WC:
49+
used_operator = SearchOperators.ILIKE
4550
else:
46-
filter_values = filter_element[FilterDataDictKeys.VALUES.value]
47-
return column_text + build_search_condition_value(operator, filter_values)
51+
used_operator = SearchOperators.LIKE
52+
for value in filter_values[1:]:
53+
used_value = value.replace("*", "%").replace("?", "_")
54+
conditions.append(
55+
search_column + build_search_condition_value(used_operator, used_value)
56+
)
57+
return " OR ".join(conditions)
58+
elif operator == SearchOperators.IN:
59+
if table == SearchTargetTables.RECORD and column == SearchColumn.DATA:
60+
filter_values = filter_values[1:]
61+
if not filter_values:
62+
return ""
63+
return search_column + build_search_condition_value(operator, filter_values)
64+
elif operator == SearchOperators.BETWEEN:
65+
if table == SearchTargetTables.RECORD and column == SearchColumn.DATA:
66+
filter_values = filter_values[1:]
67+
if not filter_values[0] and not filter_values[1]:
68+
return ""
69+
elif not filter_values[0]:
70+
return search_column + build_search_condition_value(
71+
SearchOperators.LESS_EQUAL, filter_values[1]
72+
)
73+
elif not filter_values[1]:
74+
return search_column + build_search_condition_value(
75+
SearchOperators.GREATER_EQUAL, filter_values[0]
76+
)
77+
return search_column + build_search_condition_value(operator, filter_values)
4878
else:
4979
if table == SearchTargetTables.RECORD and column == SearchColumn.DATA:
50-
filter_value = filter_element[FilterDataDictKeys.VALUES.value][1]
80+
filter_value = filter_values[1]
81+
attribute_item = attribute.get_by_name(project_id, attr_name)
82+
if not filter_value and attribute_item.data_type != DataTypes.BOOLEAN.value:
83+
return ""
5184
else:
52-
filter_value = filter_element[FilterDataDictKeys.VALUES.value][0]
53-
54-
return column_text + build_search_condition_value(operator, filter_value)
85+
filter_value = filter_values[0]
5586

87+
return search_column + build_search_condition_value(operator, filter_value)
5688

57-
def build_search_column_text(filter_element: Dict[str, str]) -> str:
5889

59-
table = SearchTargetTables[filter_element[FilterDataDictKeys.TARGET_TABLE.value]]
90+
def build_search_column(
91+
project_id: str, table: str, column: str, operator: str, attr_name: str
92+
) -> str:
6093
table_alias = __lookup_table_alias[table]
61-
column = SearchColumn[filter_element[FilterDataDictKeys.TARGET_COLUMN.value]]
62-
6394
if table == SearchTargetTables.RECORD and column == SearchColumn.DATA:
64-
col_str = f"{table_alias}.\"data\" ->> '{filter_element[FilterDataDictKeys.VALUES.value][0]}'::TEXT"
95+
attr_data_type = "TEXT"
96+
if operator not in __lookup_operator_has_quotes:
97+
attr_data_type = attribute.get_data_type(project_id, attr_name)
98+
sql_cast = __lookup_sql_cast_data_type[attr_data_type]
99+
col_str = f"({table_alias}.\"data\" ->> '{attr_name}')::{sql_cast}"
65100
else:
66101
col_str = f"{table_alias}.{column.value}"
67102
return col_str
@@ -146,6 +181,9 @@ def build_query_template(
146181
upper = str(upper)
147182
template = template.replace("@@VALUE1@@", lower)
148183
template = template.replace("@@VALUE2@@", upper)
184+
elif target == SearchQueryTemplate.SUBQUERY_HAS_COMMENTS:
185+
template = template.replace("@@USER_ID@@", filter_values[0])
186+
149187
template = template.replace("@@PROJECT_ID@@", project_id)
150188
return template
151189

@@ -198,15 +236,40 @@ def build_order_by_table_select(
198236
__lookup_operator = {
199237
SearchOperators.EQUAL: " = @@VALUE@@",
200238
SearchOperators.CONTAINS: " ILIKE '%@@VALUE@@%'",
239+
SearchOperators.CONTAINS_CS: " LIKE '%@@VALUE@@%'",
201240
SearchOperators.BEGINS_WITH: " ILIKE '@@VALUE@@%'",
241+
SearchOperators.BEGINS_WITH_CS: " LIKE '@@VALUE@@%'",
202242
SearchOperators.ENDS_WITH: " ILIKE '%@@VALUE@@'",
243+
SearchOperators.ENDS_WITH_CS: " LIKE '%@@VALUE@@'",
203244
SearchOperators.IN: " IN (@@VALUES@@)",
245+
SearchOperators.BETWEEN: " BETWEEN @@VALUE1@@ AND @@VALUE2@@",
246+
SearchOperators.GREATER: " > @@VALUE@@",
247+
SearchOperators.GREATER_EQUAL: " >= @@VALUE@@",
248+
SearchOperators.LESS: " < @@VALUE@@",
249+
SearchOperators.LESS_EQUAL: " <= @@VALUE@@",
250+
SearchOperators.LIKE: " LIKE '@@VALUE@@'",
251+
SearchOperators.ILIKE: " ILIKE '@@VALUE@@'",
204252
}
205253

206254
__lookup_operator_has_quotes = {
207255
SearchOperators.CONTAINS: True,
256+
SearchOperators.CONTAINS_CS: True,
208257
SearchOperators.BEGINS_WITH: True,
258+
SearchOperators.BEGINS_WITH_CS: True,
209259
SearchOperators.ENDS_WITH: True,
260+
SearchOperators.ENDS_WITH_CS: True,
261+
SearchOperators.LIKE: True,
262+
SearchOperators.ILIKE: True,
263+
SearchOperators.IN_WC: True,
264+
SearchOperators.IN_WC_CS: True,
265+
}
266+
267+
__lookup_sql_cast_data_type = {
268+
DataTypes.INTEGER.value: "INTEGER",
269+
DataTypes.FLOAT.value: "FLOAT",
270+
DataTypes.BOOLEAN.value: "BOOLEAN",
271+
DataTypes.CATEGORY.value: "TEXT",
272+
DataTypes.TEXT.value: "TEXT",
210273
}
211274

212275

@@ -330,4 +393,12 @@ def build_order_by_table_select(
330393
HAVING COUNT(*) >1
331394
332395
""",
396+
SearchQueryTemplate.SUBQUERY_HAS_COMMENTS: """
397+
SELECT cd.project_id pID, cd.xfkey rID
398+
FROM comment_data cd
399+
WHERE cd.project_id = '@@PROJECT_ID@@'
400+
AND cd.xftype = 'RECORD'
401+
AND (cd.is_private = false OR cd.created_by = '@@USER_ID@@')
402+
GROUP BY cd.project_id, cd.xfkey
403+
""",
333404
}

‎submodules/model

0 commit comments

Comments
 (0)