Skip to content

Commit c96001e

Browse files
robot-ci-heartexmatt-bernsteinfern-api[bot]
authored
feat: ROOT-73: custom metrics in SDK (#541)
Co-authored-by: Matt Bernstein <[email protected]> Co-authored-by: fern-api <115122769+fern-api[bot]@users.noreply.github.com>
1 parent 052b9f6 commit c96001e

30 files changed

+2405
-53
lines changed

‎.mock/definition/__package__.yml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ errors:
1212
docs: KPI depends on other KPIs - not implemented
1313
InternalServerError:
1414
status-code: 500
15-
type: string
15+
type: unknown
1616
docs: Predicting error
1717
examples:
1818
- value: Server responded with an error.
@@ -4984,6 +4984,24 @@ types:
49844984
- CommentSerializerWithExpandedUser
49854985
source:
49864986
openapi: openapi/openapi.yaml
4987+
MetricParam:
4988+
properties:
4989+
agreement_threshold: string
4990+
allowed: string
4991+
max_additional_annotators_assignable:
4992+
type: optional<integer>
4993+
docs: >-
4994+
Maximum number of additional annotators that can be assigned to a low
4995+
agreement task
4996+
validation:
4997+
min: -2147483648
4998+
max: 2147483647
4999+
metric_name:
5000+
type: optional<string>
5001+
docs: Evaluation metric chosen for this project
5002+
metric_params: optional<unknown>
5003+
source:
5004+
openapi: openapi/openapi.yaml
49875005
ModeEnum:
49885006
enum:
49895007
- settings

‎.mock/definition/aws.yml‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎.mock/definition/projects/metrics.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
imports:
2+
root: ../__package__.yml
3+
service:
4+
auth: false
5+
base-path: ''
6+
endpoints:
7+
get:
8+
path: /api/projects/{id}/metricparam/
9+
method: GET
10+
auth: true
11+
docs: Get the current metrics configuration for a project.
12+
source:
13+
openapi: openapi/openapi.yaml
14+
path-parameters:
15+
id: integer
16+
display-name: Get project metrics configuration
17+
response:
18+
docs: Current metrics configuration
19+
type: root.MetricParam
20+
examples:
21+
- path-parameters:
22+
id: 1
23+
response:
24+
body:
25+
agreement_threshold: agreement_threshold
26+
allowed: allowed
27+
max_additional_annotators_assignable: 1
28+
metric_name: metric_name
29+
metric_params:
30+
key: value
31+
audiences:
32+
- public
33+
update:
34+
path: /api/projects/{id}/metricparam/
35+
method: POST
36+
auth: true
37+
docs: Update metrics strategy and parameters for a project.
38+
source:
39+
openapi: openapi/openapi.yaml
40+
path-parameters:
41+
id: integer
42+
display-name: Update project metrics configuration
43+
request:
44+
name: ProjectMetricParamUpdateRequest
45+
body:
46+
properties:
47+
additional_params: optional<map<string, unknown>>
48+
agreement_threshold:
49+
type: optional<integer>
50+
validation:
51+
min: 0
52+
max: 100
53+
max_additional_annotators_assignable:
54+
type: optional<integer>
55+
validation:
56+
min: 0
57+
metric_name:
58+
type: optional<string>
59+
validation:
60+
minLength: 1
61+
content-type: application/json
62+
response:
63+
docs: Updated metrics configuration
64+
type: root.MetricParam
65+
examples:
66+
- path-parameters:
67+
id: 1
68+
request: {}
69+
response:
70+
body:
71+
agreement_threshold: agreement_threshold
72+
allowed: allowed
73+
max_additional_annotators_assignable: 1
74+
metric_name: metric_name
75+
metric_params:
76+
key: value
77+
audiences:
78+
- public
79+
source:
80+
openapi: openapi/openapi.yaml

‎.mock/definition/projects/metrics/custom.yml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
types:
2+
CustomGetLambdaResponse:
3+
properties:
4+
code: string
5+
source:
6+
openapi: openapi/openapi.yaml
7+
service:
8+
auth: false
9+
base-path: ''
10+
endpoints:
11+
get_lambda:
12+
path: /api/projects/{id}/aws-custom-function
13+
method: GET
14+
auth: true
15+
docs: >-
16+
Get the AWS Lambda code for the custom metric configured for this
17+
project.
18+
source:
19+
openapi: openapi/openapi.yaml
20+
path-parameters:
21+
id: integer
22+
display-name: Get AWS custom metric code
23+
response:
24+
docs: Lambda code
25+
type: CustomGetLambdaResponse
26+
examples:
27+
- path-parameters:
28+
id: 1
29+
response:
30+
body:
31+
code: code
32+
audiences:
33+
- public
34+
update_lambda:
35+
path: /api/projects/{id}/aws-custom-function
36+
method: POST
37+
auth: true
38+
docs: >-
39+
Create or update the AWS Lambda function used for custom metrics in this
40+
project.
41+
source:
42+
openapi: openapi/openapi.yaml
43+
path-parameters:
44+
id: integer
45+
display-name: Update AWS custom metric Lambda
46+
request:
47+
name: AwsCustomFunctionUpdateRequest
48+
body:
49+
properties:
50+
code:
51+
type: string
52+
validation:
53+
minLength: 1
54+
region: optional<string>
55+
role: optional<string>
56+
content-type: application/json
57+
errors:
58+
- root.InternalServerError
59+
examples:
60+
- path-parameters:
61+
id: 1
62+
request:
63+
code: code
64+
audiences:
65+
- public
66+
logs:
67+
path: /api/projects/{id}/aws-custom-function-logs
68+
method: GET
69+
auth: true
70+
docs: >-
71+
Get AWS lambda logs for project, including filtering by start and end
72+
dates
73+
source:
74+
openapi: openapi/openapi.yaml
75+
path-parameters:
76+
id: integer
77+
display-name: Get AWS lambda logs for project
78+
request:
79+
name: CustomLogsRequest
80+
query-parameters:
81+
end_date:
82+
type: optional<string>
83+
docs: End date for AWS logs filtering in format %Y-%m-%d
84+
limit:
85+
type: optional<integer>
86+
default: 100
87+
docs: Limit the number of logs to return
88+
start_date:
89+
type: optional<string>
90+
docs: Start date for AWS logs filtering in format %Y-%m-%d
91+
response:
92+
docs: Successful response returns list of AWS lambda logs
93+
type: map<string, unknown>
94+
examples:
95+
- path-parameters:
96+
id: 1
97+
response:
98+
body:
99+
key: value
100+
audiences:
101+
- public
102+
check_function:
103+
path: /api/projects/{id}/check-function
104+
method: POST
105+
auth: true
106+
docs: Validate custom matching function code for the project.
107+
source:
108+
openapi: openapi/openapi.yaml
109+
path-parameters:
110+
id: integer
111+
display-name: Check custom matching function code
112+
request:
113+
name: CheckMatchingFunctionRequestRequest
114+
body:
115+
properties:
116+
code:
117+
type: string
118+
validation:
119+
minLength: 1
120+
content-type: application/json
121+
examples:
122+
- path-parameters:
123+
id: 1
124+
request:
125+
code: code
126+
audiences:
127+
- public
128+
source:
129+
openapi: openapi/openapi.yaml
130+
imports:
131+
root: ../../__package__.yml

‎.mock/definition/projects/stats.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,25 @@ types:
6161
docs: List of users in the matrix
6262
source:
6363
openapi: openapi/openapi.yaml
64+
StatsTotalAgreementResponseZero:
65+
properties:
66+
total_agreement: optional<double>
67+
source:
68+
openapi: openapi/openapi.yaml
69+
inline: true
70+
StatsTotalAgreementResponseOne:
71+
properties:
72+
total_agreement: optional<map<string, double>>
73+
source:
74+
openapi: openapi/openapi.yaml
75+
inline: true
76+
StatsTotalAgreementResponse:
77+
discriminated: false
78+
union:
79+
- StatsTotalAgreementResponseZero
80+
- StatsTotalAgreementResponseOne
81+
source:
82+
openapi: openapi/openapi.yaml
6483
service:
6584
auth: false
6685
base-path: ''
@@ -156,5 +175,38 @@ service:
156175
id: 2
157176
audiences:
158177
- public
178+
total_agreement:
179+
path: /api/projects/{id}/stats/total_agreement
180+
method: GET
181+
auth: true
182+
docs: >-
183+
Overall or per-label total agreement across the project.
184+
185+
186+
NOTE: due to an open issue in Fern, SDK clients will raise ApiError upon
187+
handling a 204 response. As a workaround, wrap call to this function in
188+
a try-except block.
189+
source:
190+
openapi: openapi/openapi.yaml
191+
path-parameters:
192+
id: integer
193+
display-name: Get total agreement for project
194+
request:
195+
name: StatsTotalAgreementRequest
196+
query-parameters:
197+
per_label:
198+
type: optional<boolean>
199+
docs: Return agreement per label
200+
response:
201+
docs: Total agreement
202+
type: StatsTotalAgreementResponse
203+
examples:
204+
- path-parameters:
205+
id: 1
206+
response:
207+
body:
208+
total_agreement: 1.1
209+
audiences:
210+
- public
159211
source:
160212
openapi: openapi/openapi.yaml

0 commit comments

Comments
 (0)