Skip to content

Commit d325d09

Browse files
wmakwmak
authored
feat(validate): Add a new validate endpoint (#117474)
- This adds a new events-validate endpoint that will check all the parameters that could be sent to the events/ endpoint and validate them.
1 parent b731683 commit d325d09

4 files changed

Lines changed: 569 additions & 0 deletions

File tree

‎src/sentry/api/endpoints/organization_events_validate.py‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
from collections import defaultdict
2+
from dataclasses import asdict, dataclass
3+
from dataclasses import field as dataclass_field
4+
from typing import Any
5+
6+
from rest_framework.exceptions import ParseError
7+
from rest_framework.request import Request
8+
from rest_framework.response import Response
9+
from sentry_protos.snuba.v1.endpoint_trace_item_attributes_pb2 import TraceItemAttributeNamesRequest
10+
from sentry_protos.snuba.v1.request_common_pb2 import RequestMeta
11+
from sentry_protos.snuba.v1.trace_item_attribute_pb2 import AttributeKey
12+
13+
from sentry.api.api_publish_status import ApiPublishStatus
14+
from sentry.api.bases import NoProjects, OrganizationEventsEndpointBase
15+
from sentry.api.utils import handle_query_errors
16+
from sentry.exceptions import InvalidSearchQuery
17+
from sentry.models.organization import Organization
18+
from sentry.search.eap import constants
19+
from sentry.search.eap.columns import ResolvedAttribute
20+
from sentry.search.eap.resolver import SearchResolver
21+
from sentry.search.eap.types import SearchResolverConfig
22+
from sentry.search.events import fields
23+
from sentry.snuba.referrer import Referrer
24+
from sentry.snuba.utils import RPC_DATASETS
25+
from sentry.utils import snuba_rpc
26+
from sentry.utils.concurrent import ContextPropagatingThreadPoolExecutor
27+
28+
29+
@dataclass(kw_only=True)
30+
class Validation:
31+
valid: bool
32+
error: str | None
33+
34+
35+
@dataclass(kw_only=True)
36+
class AttributeValidation(Validation):
37+
name: str
38+
# None when its an error
39+
attrType: str | None
40+
41+
42+
@dataclass(kw_only=True)
43+
class ValidationResponse:
44+
valid: bool
45+
projects: list[Validation] = dataclass_field(default_factory=list)
46+
dataset: list[Validation] = dataclass_field(default_factory=list)
47+
field: list[AttributeValidation] = dataclass_field(default_factory=list)
48+
orderby: list[AttributeValidation] = dataclass_field(default_factory=list)
49+
50+
51+
def serialize_type(search_type: constants.SearchType) -> str:
52+
proto_type = constants.TYPE_MAP.get(search_type)
53+
if proto_type == constants.STRING:
54+
return "string"
55+
if proto_type == constants.BOOLEAN:
56+
return "boolean"
57+
# DOUBLE, INT, or anything else numeric
58+
return "number"
59+
60+
61+
MAX_ATTRIBUTE_VALIDATION_THREADS = 3
62+
63+
64+
def _check_attributes_by_type(
65+
meta: RequestMeta,
66+
attr_type: AttributeKey.Type.ValueType,
67+
attributes: list[ResolvedAttribute],
68+
) -> set[tuple[AttributeKey.Type.ValueType, str]]:
69+
"""Check which typed attribute names exist in storage for the active window."""
70+
if not attributes:
71+
return set()
72+
73+
requested_names = set(attribute.internal_name for attribute in attributes)
74+
# TODO(wmak): Need to update snuba here so we can pass the list of attributes, snuba currently does a hasAll if we
75+
# pass names in a OrFilter which means only rows with _all_ attributes will return
76+
attrs_request = TraceItemAttributeNamesRequest(
77+
meta=meta,
78+
limit=10_000,
79+
type=attr_type,
80+
)
81+
attrs_response = snuba_rpc.attribute_names_rpc(attrs_request)
82+
return {
83+
(attr_type, attribute.name)
84+
for attribute in attrs_response.attributes
85+
if attribute.name in requested_names
86+
}
87+
88+
89+
def check_attributes_exist(
90+
resolver: SearchResolver,
91+
dataset: Any,
92+
attrs_by_type: dict[AttributeKey.Type.ValueType, list[ResolvedAttribute]],
93+
) -> set[tuple[AttributeKey.Type.ValueType, str]]:
94+
"""Check which typed attribute internal names exist in storage."""
95+
if not attrs_by_type:
96+
return set()
97+
98+
meta = resolver.resolve_meta(referrer=Referrer.API_TRACE_ITEM_ATTRIBUTE_VALIDATE.value)
99+
100+
found: set[tuple[AttributeKey.Type.ValueType, str]] = set()
101+
with ContextPropagatingThreadPoolExecutor(
102+
thread_name_prefix="attr_validate",
103+
max_workers=MAX_ATTRIBUTE_VALIDATION_THREADS,
104+
) as pool:
105+
futures = [
106+
pool.submit(_check_attributes_by_type, meta, attr_type, names)
107+
for attr_type, names in attrs_by_type.items()
108+
]
109+
for future in futures:
110+
found.update(future.result())
111+
112+
return found
113+
114+
115+
class OrganizationEventsValidateEndpoint(OrganizationEventsEndpointBase):
116+
publish_status = {
117+
"GET": ApiPublishStatus.EXPERIMENTAL,
118+
}
119+
120+
def serialize_response(
121+
self,
122+
validity: ValidationResponse,
123+
) -> Response:
124+
return Response(
125+
status=200 if validity.valid else 400,
126+
data=asdict(validity),
127+
)
128+
129+
def get(self, request: Request, organization: Organization) -> Response:
130+
if not self.has_feature(organization, request):
131+
return Response(status=400)
132+
133+
response = ValidationResponse(valid=True)
134+
135+
try:
136+
snuba_params = self.get_snuba_params(
137+
request,
138+
organization,
139+
)
140+
except NoProjects:
141+
response.valid = False
142+
response.projects.append(
143+
Validation(valid=False, error="At least one valid project is required to query")
144+
)
145+
return self.serialize_response(response)
146+
147+
try:
148+
dataset = self.get_dataset(request, organization)
149+
except ParseError as error:
150+
response.valid = False
151+
response.dataset.append(Validation(valid=False, error=str(error)))
152+
return self.serialize_response(response)
153+
154+
if dataset not in RPC_DATASETS:
155+
response.dataset.append(
156+
Validation(
157+
valid=True,
158+
error="This dataset is not compatible with the validate endpoint, your request may still be valid",
159+
)
160+
)
161+
# Can't continue if this isn't a RPC dataset
162+
return self.serialize_response(response)
163+
164+
resolver = dataset.get_resolver(snuba_params, SearchResolverConfig())
165+
definitions = resolver.definitions
166+
167+
# Validate selected_columns
168+
selected_columns = self.get_field_list(organization, request)
169+
attributes_to_lookup = defaultdict(list)
170+
column_validity: list[AttributeValidation] = []
171+
for column in selected_columns:
172+
try:
173+
match = fields.is_function(column)
174+
if match:
175+
resolved, _ = resolver.resolve_function(column, match)
176+
column_validity.append(
177+
AttributeValidation(
178+
attrType=serialize_type(resolved.search_type),
179+
error=None,
180+
name=column,
181+
valid=True,
182+
)
183+
)
184+
else:
185+
resolved, _ = resolver.resolve_attribute(column)
186+
if column in definitions.contexts or column in definitions.columns:
187+
column_validity.append(
188+
AttributeValidation(
189+
attrType=serialize_type(resolved.search_type),
190+
error="",
191+
name=column,
192+
valid=True,
193+
)
194+
)
195+
else:
196+
attributes_to_lookup[resolved.proto_type].append(resolved)
197+
except InvalidSearchQuery as error:
198+
response.valid = False
199+
column_validity.append(
200+
AttributeValidation(
201+
attrType=None,
202+
error=str(error),
203+
name=column,
204+
valid=False,
205+
)
206+
)
207+
208+
if any(len(attributes) > 0 for attributes in attributes_to_lookup.values()):
209+
# Group by proto type because the storage check is keyed on
210+
# (proto_type, internal_name) — the same display name can exist
211+
# as both a string and a number attribute simultaneously.
212+
with handle_query_errors():
213+
existing = check_attributes_exist(resolver, dataset, attributes_to_lookup)
214+
for attribute_type, attributes in attributes_to_lookup.items():
215+
for resolved in attributes:
216+
if (resolved.proto_type, resolved.internal_name) in existing:
217+
column_validity.append(
218+
AttributeValidation(
219+
attrType=serialize_type(resolved.search_type),
220+
error="",
221+
name=resolved.public_alias,
222+
valid=True,
223+
)
224+
)
225+
else:
226+
response.valid = False
227+
column_validity.append(
228+
AttributeValidation(
229+
attrType=None,
230+
error="Unknown attribute",
231+
name=resolved.public_alias,
232+
valid=False,
233+
)
234+
)
235+
response.field = column_validity
236+
237+
# Validate orderby
238+
orderby_validity = []
239+
orderby_columns = self.get_orderby(request)
240+
if orderby_columns:
241+
for orderby in orderby_columns:
242+
stripped_orderby = orderby.lstrip("-")
243+
found = False
244+
for field in column_validity:
245+
if (
246+
field.name == stripped_orderby
247+
or fields.get_function_alias(field.name) == stripped_orderby
248+
):
249+
orderby_validity.append(
250+
AttributeValidation(
251+
attrType=field.attrType, error=None, name=orderby, valid=True
252+
)
253+
)
254+
found = True
255+
break
256+
if not found:
257+
response.valid = False
258+
orderby_validity.append(
259+
AttributeValidation(
260+
attrType=None,
261+
error="Orderby must also be a selected field",
262+
name=orderby,
263+
valid=False,
264+
)
265+
)
266+
response.orderby = orderby_validity
267+
268+
# TODO(wmak): Validate query
269+
270+
return self.serialize_response(response)

‎src/sentry/api/urls.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,6 +755,7 @@
755755
OrganizationEventsTrendsStatsEndpoint,
756756
)
757757
from .endpoints.organization_events_trends_v2 import OrganizationEventsNewTrendsStatsEndpoint
758+
from .endpoints.organization_events_validate import OrganizationEventsValidateEndpoint
758759
from .endpoints.organization_events_vitals import OrganizationEventsVitalsEndpoint
759760
from .endpoints.organization_measurements_meta import OrganizationMeasurementsMeta
760761
from .endpoints.organization_metrics_meta import (
@@ -1659,6 +1660,11 @@ def create_group_urls(name_prefix: str) -> list[URLPattern | URLResolver]:
16591660
OrganizationEventsEndpoint.as_view(),
16601661
name="sentry-api-0-organization-events",
16611662
),
1663+
re_path(
1664+
r"^(?P<organization_id_or_slug>[^/]+)/events/validate/$",
1665+
OrganizationEventsValidateEndpoint.as_view(),
1666+
name="sentry-api-0-organization-events-validate",
1667+
),
16621668
re_path(
16631669
r"^(?P<organization_id_or_slug>[^/]+)/events-sql/$",
16641670
OrganizationEventsSqlEndpoint.as_view(),

‎static/app/utils/api/knownSentryApiUrls.generated.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ export type KnownSentryApiUrls =
148148
| '/organizations/$organizationIdOrSlug/events/'
149149
| '/organizations/$organizationIdOrSlug/events/$projectIdOrSlug:$eventId/'
150150
| '/organizations/$organizationIdOrSlug/events/anomalies/'
151+
| '/organizations/$organizationIdOrSlug/events/validate/'
151152
| '/organizations/$organizationIdOrSlug/explore/saved/'
152153
| '/organizations/$organizationIdOrSlug/explore/saved/$id/'
153154
| '/organizations/$organizationIdOrSlug/explore/saved/$id/starred/'

0 commit comments

Comments
 (0)