From da812804c6779e300e155f73c131a0539c50385a Mon Sep 17 00:00:00 2001 From: Dustin Ngo Date: Fri, 21 Feb 2025 12:13:12 -0500 Subject: [PATCH 01/53] Create annotation configs feature branch From 7c2894cfd24c7deacd2bb9fb53f148bc01c1f4b6 Mon Sep 17 00:00:00 2001 From: Dustin Ngo Date: Fri, 28 Feb 2025 13:54:10 -0500 Subject: [PATCH 02/53] feat: Allow multiple annotations per span with the same name (#6573) * Drop unique constraint migration * Update /span_annotations REST route * Update bulk inserters * Update data loader to aggregate label fractions per span first * Use correct kwarg name * Use sqlite compatible migration * Always use batch_alter_table * Ensure that spans with an annotation aggregate labels properly * Use auto recreate * Re-apply SpanFilter * Add migration to integration test * Update OpenAPI schema * Update traces router * Update helper test * Cast to floats before doing arithmetic * Add type annotations * Rebuild js client * Use more robust typing * Use a weighted avg for scores - optimize query to not use cartesian product - will scale very poorly as the number of labels increases * Use properly per-span aggregated scores * Cast to float before returning LabelFraction * Add tests * Remove score column from base stmt * Use column indexing * Improve clarity for type checker * Fix unit test type checks --- .../src/__generated__/api/v1.ts | 2 +- schemas/openapi.json | 2 +- .../db/insertion/document_annotation.py | 85 ++----- src/phoenix/db/insertion/span_annotation.py | 86 +------ src/phoenix/db/insertion/trace_annotation.py | 86 +------ ...f9d1a65945f_annotation_config_migration.py | 47 ++++ src/phoenix/db/models.py | 20 -- .../api/dataloaders/annotation_summaries.py | 216 +++++++++++++++-- src/phoenix/server/api/routers/v1/spans.py | 19 +- src/phoenix/server/api/routers/v1/traces.py | 15 +- .../server/api/types/AnnotationSummary.py | 20 +- .../test_up_and_down_migrations.py | 5 + tests/unit/db/insertion/test_helpers.py | 113 ++------- tests/unit/server/api/dataloaders/conftest.py | 225 ++++++++++++++++++ .../dataloaders/test_annotation_summaries.py | 87 +++++++ 15 files changed, 635 insertions(+), 393 deletions(-) create mode 100644 src/phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py diff --git a/js/packages/phoenix-client/src/__generated__/api/v1.ts b/js/packages/phoenix-client/src/__generated__/api/v1.ts index 981d9b4cee..32e42b2c78 100644 --- a/js/packages/phoenix-client/src/__generated__/api/v1.ts +++ b/js/packages/phoenix-client/src/__generated__/api/v1.ts @@ -219,7 +219,7 @@ export interface paths { }; get?: never; put?: never; - /** Create or update span annotations */ + /** Create span annotations */ post: operations["annotateSpans"]; delete?: never; options?: never; diff --git a/schemas/openapi.json b/schemas/openapi.json index 5c9653b511..dae8f472be 100644 --- a/schemas/openapi.json +++ b/schemas/openapi.json @@ -1078,7 +1078,7 @@ "tags": [ "spans" ], - "summary": "Create or update span annotations", + "summary": "Create span annotations", "operationId": "annotateSpans", "parameters": [ { diff --git a/src/phoenix/db/insertion/document_annotation.py b/src/phoenix/db/insertion/document_annotation.py index ee98221ede..b9b856c2ab 100644 --- a/src/phoenix/db/insertion/document_annotation.py +++ b/src/phoenix/db/insertion/document_annotation.py @@ -1,13 +1,13 @@ from collections.abc import Mapping from datetime import datetime -from typing import Any, NamedTuple, Optional +from typing import NamedTuple, Optional -from sqlalchemy import Row, Select, and_, select, tuple_ +from sqlalchemy import insert, select from sqlalchemy.ext.asyncio import AsyncSession from typing_extensions import TypeAlias from phoenix.db import models -from phoenix.db.helpers import dedup, num_docs_col +from phoenix.db.helpers import num_docs_col from phoenix.db.insertion.helpers import as_kv from phoenix.db.insertion.types import ( Insertables, @@ -46,7 +46,7 @@ class DocumentAnnotationQueueInserter( DocumentAnnotationDmlEvent, ], table=models.DocumentAnnotation, - unique_by=("name", "span_rowid", "document_position"), + unique_by=(), ): async def _events( self, @@ -54,7 +54,7 @@ async def _events( *insertions: Insertables.DocumentAnnotation, ) -> list[DocumentAnnotationDmlEvent]: records = [dict(as_kv(ins.row)) for ins in insertions] - stmt = self._insert_on_conflict(*records).returning(self.table.id) + stmt = insert(self.table).values(records).returning(self.table.id) ids = tuple([_ async for _ in await session.stream_scalars(stmt)]) return [DocumentAnnotationDmlEvent(ids)] @@ -71,35 +71,19 @@ async def _partition( to_postpone: list[Postponed[Precursors.DocumentAnnotation]] = [] to_discard: list[Received[Precursors.DocumentAnnotation]] = [] - stmt = self._select_existing(*map(_key, parcels)) - existing: list[Row[_Existing]] = [_ async for _ in await session.stream(stmt)] + span_ids = {p.item.span_id for p in parcels} + stmt = select(models.Span.id, models.Span.span_id, num_docs_col(self._db.dialect)).where( + models.Span.span_id.in_(span_ids) + ) + result = await session.execute(stmt) + spans = result.all() existing_spans: Mapping[str, _SpanAttr] = { - e.span_id: _SpanAttr(e.span_rowid, e.num_docs) for e in existing - } - existing_annos: Mapping[_Key, _AnnoAttr] = { - (e.name, e.span_id, e.document_position): _AnnoAttr(e.span_rowid, e.id, e.updated_at) - for e in existing - if e.id is not None - and e.name is not None - and e.document_position is not None - and e.updated_at is not None + row.span_id: _SpanAttr(row.id, row.num_docs) for row in spans } for p in parcels: - if (anno := existing_annos.get(_key(p))) is not None: - if p.received_at <= anno.updated_at: - to_discard.append(p) - else: - to_insert.append( - Received( - received_at=p.received_at, - item=p.item.as_insertable( - span_rowid=anno.span_rowid, - id_=anno.id_, - ), - ) - ) - elif (span := existing_spans.get(p.item.span_id)) is not None: + if p.item.span_id in existing_spans: + span = existing_spans[p.item.span_id] if 0 <= p.item.document_position < span.num_docs: to_insert.append( Received( @@ -122,50 +106,9 @@ async def _partition( to_discard.append(p) assert len(to_insert) + len(to_postpone) + len(to_discard) == len(parcels) - to_insert = dedup(sorted(to_insert, key=_time, reverse=True), _unique_by)[::-1] return to_insert, to_postpone, to_discard - def _select_existing(self, *keys: _Key) -> Select[_Existing]: - anno = self.table - span = ( - select(models.Span.id, models.Span.span_id, num_docs_col(self._db.dialect)) - .where(models.Span.span_id.in_({span_id for _, span_id, *_ in keys})) - .cte() - ) - onclause = and_( - span.c.id == anno.span_rowid, - anno.name.in_({name for name, *_ in keys}), - tuple_(anno.name, span.c.span_id, anno.document_position).in_(keys), - ) - return select( - span.c.id.label("span_rowid"), - span.c.span_id, - span.c.num_docs, - anno.id, - anno.name, - anno.document_position, - anno.updated_at, - ).outerjoin_from(span, anno, onclause) - class _SpanAttr(NamedTuple): span_rowid: _SpanRowId num_docs: _NumDocs - - -class _AnnoAttr(NamedTuple): - span_rowid: _SpanRowId - id_: _AnnoRowId - updated_at: datetime - - -def _key(p: Received[Precursors.DocumentAnnotation]) -> _Key: - return p.item.obj.name, p.item.span_id, p.item.document_position - - -def _unique_by(p: Received[Insertables.DocumentAnnotation]) -> _UniqueBy: - return p.item.obj.name, p.item.span_rowid, p.item.document_position - - -def _time(p: Received[Any]) -> datetime: - return p.received_at diff --git a/src/phoenix/db/insertion/span_annotation.py b/src/phoenix/db/insertion/span_annotation.py index 05a4121b16..688a0a6f09 100644 --- a/src/phoenix/db/insertion/span_annotation.py +++ b/src/phoenix/db/insertion/span_annotation.py @@ -1,13 +1,12 @@ from collections.abc import Mapping from datetime import datetime -from typing import Any, NamedTuple, Optional +from typing import Optional -from sqlalchemy import Row, Select, and_, select, tuple_ +from sqlalchemy import insert, select from sqlalchemy.ext.asyncio import AsyncSession from typing_extensions import TypeAlias from phoenix.db import models -from phoenix.db.helpers import dedup from phoenix.db.insertion.helpers import as_kv from phoenix.db.insertion.types import ( Insertables, @@ -42,7 +41,7 @@ class SpanAnnotationQueueInserter( SpanAnnotationDmlEvent, ], table=models.SpanAnnotation, - unique_by=("name", "span_rowid"), + unique_by=(), ): async def _events( self, @@ -50,7 +49,7 @@ async def _events( *insertions: Insertables.SpanAnnotation, ) -> list[SpanAnnotationDmlEvent]: records = [dict(as_kv(ins.row)) for ins in insertions] - stmt = self._insert_on_conflict(*records).returning(self.table.id) + stmt = insert(self.table).values(records).returning(self.table.id) ids = tuple([_ async for _ in await session.stream_scalars(stmt)]) return [SpanAnnotationDmlEvent(ids)] @@ -67,38 +66,18 @@ async def _partition( to_postpone: list[Postponed[Precursors.SpanAnnotation]] = [] to_discard: list[Received[Precursors.SpanAnnotation]] = [] - stmt = self._select_existing(*map(_key, parcels)) - existing: list[Row[_Existing]] = [_ async for _ in await session.stream(stmt)] - existing_spans: Mapping[str, _SpanAttr] = { - e.span_id: _SpanAttr(e.span_rowid) for e in existing - } - existing_annos: Mapping[_Key, _AnnoAttr] = { - (e.name, e.span_id): _AnnoAttr(e.span_rowid, e.id, e.updated_at) - for e in existing - if e.id is not None and e.name is not None and e.updated_at is not None - } + span_ids = {p.item.span_id for p in parcels} + stmt = select(models.Span.id, models.Span.span_id).where(models.Span.span_id.in_(span_ids)) + result = await session.execute(stmt) + spans = result.all() + existing_spans: Mapping[str, int] = {row.span_id: row.id for row in spans} for p in parcels: - if (anno := existing_annos.get(_key(p))) is not None: - if p.received_at <= anno.updated_at: - to_discard.append(p) - else: - to_insert.append( - Received( - received_at=p.received_at, - item=p.item.as_insertable( - span_rowid=anno.span_rowid, - id_=anno.id_, - ), - ) - ) - elif (span := existing_spans.get(p.item.span_id)) is not None: + if p.item.span_id in existing_spans: to_insert.append( Received( received_at=p.received_at, - item=p.item.as_insertable( - span_rowid=span.span_rowid, - ), + item=p.item.as_insertable(span_rowid=existing_spans[p.item.span_id]), ) ) elif isinstance(p, Postponed): @@ -112,47 +91,4 @@ async def _partition( to_discard.append(p) assert len(to_insert) + len(to_postpone) + len(to_discard) == len(parcels) - to_insert = dedup(sorted(to_insert, key=_time, reverse=True), _unique_by)[::-1] return to_insert, to_postpone, to_discard - - def _select_existing(self, *keys: _Key) -> Select[_Existing]: - anno = self.table - span = ( - select(models.Span.id, models.Span.span_id) - .where(models.Span.span_id.in_({span_id for _, span_id in keys})) - .cte() - ) - onclause = and_( - span.c.id == anno.span_rowid, - anno.name.in_({name for name, _ in keys}), - tuple_(anno.name, span.c.span_id).in_(keys), - ) - return select( - span.c.id.label("span_rowid"), - span.c.span_id, - anno.id, - anno.name, - anno.updated_at, - ).outerjoin_from(span, anno, onclause) - - -class _SpanAttr(NamedTuple): - span_rowid: _SpanRowId - - -class _AnnoAttr(NamedTuple): - span_rowid: _SpanRowId - id_: _AnnoRowId - updated_at: datetime - - -def _key(p: Received[Precursors.SpanAnnotation]) -> _Key: - return p.item.obj.name, p.item.span_id - - -def _unique_by(p: Received[Insertables.SpanAnnotation]) -> _UniqueBy: - return p.item.obj.name, p.item.span_rowid - - -def _time(p: Received[Any]) -> datetime: - return p.received_at diff --git a/src/phoenix/db/insertion/trace_annotation.py b/src/phoenix/db/insertion/trace_annotation.py index 3d83a6fc85..db0871aa5f 100644 --- a/src/phoenix/db/insertion/trace_annotation.py +++ b/src/phoenix/db/insertion/trace_annotation.py @@ -1,13 +1,11 @@ -from collections.abc import Mapping from datetime import datetime -from typing import Any, NamedTuple, Optional +from typing import Optional -from sqlalchemy import Row, Select, and_, select, tuple_ +from sqlalchemy import insert, select from sqlalchemy.ext.asyncio import AsyncSession from typing_extensions import TypeAlias from phoenix.db import models -from phoenix.db.helpers import dedup from phoenix.db.insertion.helpers import as_kv from phoenix.db.insertion.types import ( Insertables, @@ -42,7 +40,7 @@ class TraceAnnotationQueueInserter( TraceAnnotationDmlEvent, ], table=models.TraceAnnotation, - unique_by=("name", "trace_rowid"), + unique_by=(), ): async def _events( self, @@ -50,7 +48,7 @@ async def _events( *insertions: Insertables.TraceAnnotation, ) -> list[TraceAnnotationDmlEvent]: records = [dict(as_kv(ins.row)) for ins in insertions] - stmt = self._insert_on_conflict(*records).returning(self.table.id) + stmt = insert(self.table).values(records).returning(self.table.id) ids = tuple([_ async for _ in await session.stream_scalars(stmt)]) return [TraceAnnotationDmlEvent(ids)] @@ -67,37 +65,20 @@ async def _partition( to_postpone: list[Postponed[Precursors.TraceAnnotation]] = [] to_discard: list[Received[Precursors.TraceAnnotation]] = [] - stmt = self._select_existing(*map(_key, parcels)) - existing: list[Row[_Existing]] = [_ async for _ in await session.stream(stmt)] - existing_traces: Mapping[str, _TraceAttr] = { - e.trace_id: _TraceAttr(e.trace_rowid) for e in existing - } - existing_annos: Mapping[_Key, _AnnoAttr] = { - (e.name, e.trace_id): _AnnoAttr(e.trace_rowid, e.id, e.updated_at) - for e in existing - if e.id is not None and e.name is not None and e.updated_at is not None - } + stmt = select(models.Trace.id, models.Trace.trace_id).where( + models.Trace.trace_id.in_({p.item.trace_id for p in parcels}) + ) + result = await session.execute(stmt) + traces = result.all() + existing_traces = {row.trace_id: row.id for row in traces} for p in parcels: - if (anno := existing_annos.get(_key(p))) is not None: - if p.received_at <= anno.updated_at: - to_discard.append(p) - else: - to_insert.append( - Received( - received_at=p.received_at, - item=p.item.as_insertable( - trace_rowid=anno.trace_rowid, - id_=anno.id_, - ), - ) - ) - elif (trace := existing_traces.get(p.item.trace_id)) is not None: + if p.item.trace_id in existing_traces: to_insert.append( Received( received_at=p.received_at, item=p.item.as_insertable( - trace_rowid=trace.trace_rowid, + trace_rowid=existing_traces[p.item.trace_id], ), ) ) @@ -112,47 +93,4 @@ async def _partition( to_discard.append(p) assert len(to_insert) + len(to_postpone) + len(to_discard) == len(parcels) - to_insert = dedup(sorted(to_insert, key=_time, reverse=True), _unique_by)[::-1] return to_insert, to_postpone, to_discard - - def _select_existing(self, *keys: _Key) -> Select[_Existing]: - anno = self.table - trace = ( - select(models.Trace.id, models.Trace.trace_id) - .where(models.Trace.trace_id.in_({trace_id for _, trace_id in keys})) - .cte() - ) - onclause = and_( - trace.c.id == anno.trace_rowid, - anno.name.in_({name for name, _ in keys}), - tuple_(anno.name, trace.c.trace_id).in_(keys), - ) - return select( - trace.c.id.label("trace_rowid"), - trace.c.trace_id, - anno.id, - anno.name, - anno.updated_at, - ).outerjoin_from(trace, anno, onclause) - - -class _TraceAttr(NamedTuple): - trace_rowid: _TraceRowId - - -class _AnnoAttr(NamedTuple): - trace_rowid: _TraceRowId - id_: _AnnoRowId - updated_at: datetime - - -def _key(p: Received[Precursors.TraceAnnotation]) -> _Key: - return p.item.obj.name, p.item.trace_id - - -def _unique_by(p: Received[Insertables.TraceAnnotation]) -> _UniqueBy: - return p.item.obj.name, p.item.trace_rowid - - -def _time(p: Received[Any]) -> datetime: - return p.received_at diff --git a/src/phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py b/src/phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py new file mode 100644 index 0000000000..00c6e416cc --- /dev/null +++ b/src/phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py @@ -0,0 +1,47 @@ +"""Annotation config migrations + +Revision ID: 2f9d1a65945f +Revises: bc8fea3c2bc8 +Create Date: 2025-02-06 10:17:15.726197 + +""" + +from typing import Sequence, Union + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "2f9d1a65945f" +down_revision: Union[str, None] = "bc8fea3c2bc8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + with op.batch_alter_table("span_annotations", recreate="auto") as batch_op: + batch_op.drop_constraint("uq_span_annotations_name_span_rowid", type_="unique") + with op.batch_alter_table("trace_annotations", recreate="auto") as batch_op: + batch_op.drop_constraint("uq_trace_annotations_name_trace_rowid", type_="unique") + with op.batch_alter_table("document_annotations", recreate="auto") as batch_op: + batch_op.drop_constraint( + "uq_document_annotations_name_span_rowid_document_position", + type_="unique", + ) + + +def downgrade() -> None: + with op.batch_alter_table("span_annotations", recreate="auto") as batch_op: + batch_op.create_unique_constraint( + "uq_span_annotations_name_span_rowid", + ["name", "span_rowid"], + ) + with op.batch_alter_table("trace_annotations", recreate="auto") as batch_op: + batch_op.create_unique_constraint( + "uq_trace_annotations_name_trace_rowid", + ["name", "trace_rowid"], + ) + with op.batch_alter_table("document_annotations", recreate="auto") as batch_op: + batch_op.create_unique_constraint( + "uq_document_annotations_name_span_rowid_document_position", + ["name", "span_rowid", "document_position"], + ) diff --git a/src/phoenix/db/models.py b/src/phoenix/db/models.py index 707e67060c..88697544fd 100644 --- a/src/phoenix/db/models.py +++ b/src/phoenix/db/models.py @@ -739,12 +739,6 @@ class SpanAnnotation(Base): updated_at: Mapped[datetime] = mapped_column( UtcTimeStamp, server_default=func.now(), onupdate=func.now() ) - __table_args__ = ( - UniqueConstraint( - "name", - "span_rowid", - ), - ) class TraceAnnotation(Base): @@ -765,12 +759,6 @@ class TraceAnnotation(Base): updated_at: Mapped[datetime] = mapped_column( UtcTimeStamp, server_default=func.now(), onupdate=func.now() ) - __table_args__ = ( - UniqueConstraint( - "name", - "trace_rowid", - ), - ) class DocumentAnnotation(Base): @@ -794,14 +782,6 @@ class DocumentAnnotation(Base): ) span: Mapped["Span"] = relationship(back_populates="document_annotations") - __table_args__ = ( - UniqueConstraint( - "name", - "span_rowid", - "document_position", - ), - ) - class Dataset(Base): __tablename__ = "datasets" diff --git a/src/phoenix/server/api/dataloaders/annotation_summaries.py b/src/phoenix/server/api/dataloaders/annotation_summaries.py index b5bbc2406a..cdbe75faa8 100644 --- a/src/phoenix/server/api/dataloaders/annotation_summaries.py +++ b/src/phoenix/server/api/dataloaders/annotation_summaries.py @@ -1,11 +1,11 @@ from collections import defaultdict from datetime import datetime -from typing import Any, Literal, Optional +from typing import Any, Literal, Optional, Type, Union, cast import pandas as pd from aioitertools.itertools import groupby from cachetools import LFUCache, TTLCache -from sqlalchemy import Select, func, or_, select +from sqlalchemy import Select, and_, case, distinct, func, or_, select from strawberry.dataloader import AbstractCache, DataLoader from typing_extensions import TypeAlias, assert_never @@ -103,23 +103,64 @@ def _get_stmt( *annotation_names: Param, ) -> Select[Any]: kind, project_rowid, (start_time, end_time), filter_condition = segment - stmt = select() + + annotation_model: Union[Type[models.SpanAnnotation], Type[models.TraceAnnotation]] + entity_model: Union[Type[models.Span], Type[models.Trace]] + entity_join_model: Optional[Type[models.Base]] + entity_id_column: Any + if kind == "span": - msa = models.SpanAnnotation - name_column, label_column, score_column = msa.name, msa.label, msa.score - time_column = models.Span.start_time - stmt = stmt.join(models.Span).join_from(models.Span, models.Trace) - if filter_condition: - sf = SpanFilter(filter_condition) - stmt = sf(stmt) + annotation_model = models.SpanAnnotation + entity_model = models.Span + entity_join_model = models.Trace + entity_id_column = models.Span.id.label("entity_id") elif kind == "trace": - mta = models.TraceAnnotation - name_column, label_column, score_column = mta.name, mta.label, mta.score - time_column = models.Trace.start_time - stmt = stmt.join(models.Trace) + annotation_model = models.TraceAnnotation + entity_model = models.Trace + entity_join_model = None + entity_id_column = models.Trace.id.label("entity_id") else: assert_never(kind) - stmt = stmt.add_columns( + + name_column = annotation_model.name + label_column = annotation_model.label + score_column = annotation_model.score + time_column = entity_model.start_time + + # First query: count distinct entities per annotation name + # This is used later to calculate accurate fractions that account for entities without labels + entity_count_query = select( + name_column, func.count(distinct(entity_id_column)).label("entity_count") + ) + + if kind == "span": + entity_count_query = entity_count_query.join(cast(Type[models.Span], entity_model)) + entity_count_query = entity_count_query.join_from( + cast(Type[models.Span], entity_model), cast(Type[models.Trace], entity_join_model) + ) + entity_count_query = entity_count_query.where(models.Trace.project_rowid == project_rowid) + elif kind == "trace": + entity_count_query = entity_count_query.join(cast(Type[models.Trace], entity_model)) + entity_count_query = entity_count_query.where( + cast(Type[models.Trace], entity_model).project_rowid == project_rowid + ) + + entity_count_query = entity_count_query.where( + or_(score_column.is_not(None), label_column.is_not(None)) + ) + entity_count_query = entity_count_query.where(name_column.in_(annotation_names)) + + if start_time: + entity_count_query = entity_count_query.where(start_time <= time_column) + if end_time: + entity_count_query = entity_count_query.where(time_column < end_time) + + entity_count_query = entity_count_query.group_by(name_column) + entity_count_subquery = entity_count_query.subquery() + + # Main query: gets raw annotation data with counts per (span/trace)+name+label + base_stmt = select( + entity_id_column, name_column, label_column, func.count().label("record_count"), @@ -127,13 +168,142 @@ def _get_stmt( func.count(score_column).label("score_count"), func.sum(score_column).label("score_sum"), ) - stmt = stmt.group_by(name_column, label_column) - stmt = stmt.order_by(name_column, label_column) - stmt = stmt.where(models.Trace.project_rowid == project_rowid) - stmt = stmt.where(or_(score_column.is_not(None), label_column.is_not(None))) - stmt = stmt.where(name_column.in_(annotation_names)) + + if kind == "span": + base_stmt = base_stmt.join(cast(Type[models.Span], entity_model)) + base_stmt = base_stmt.join_from( + cast(Type[models.Span], entity_model), cast(Type[models.Trace], entity_join_model) + ) + base_stmt = base_stmt.where(models.Trace.project_rowid == project_rowid) + if filter_condition: + sf = SpanFilter(filter_condition) + base_stmt = sf(base_stmt) + elif kind == "trace": + base_stmt = base_stmt.join(cast(Type[models.Trace], entity_model)) + base_stmt = base_stmt.where( + cast(Type[models.Trace], entity_model).project_rowid == project_rowid + ) + else: + assert_never(kind) + + base_stmt = base_stmt.where(or_(score_column.is_not(None), label_column.is_not(None))) + base_stmt = base_stmt.where(name_column.in_(annotation_names)) + if start_time: - stmt = stmt.where(start_time <= time_column) + base_stmt = base_stmt.where(start_time <= time_column) if end_time: - stmt = stmt.where(time_column < end_time) - return stmt + base_stmt = base_stmt.where(time_column < end_time) + + # Group to get one row per (span/trace)+name+label combination + base_stmt = base_stmt.group_by(entity_id_column, name_column, label_column) + + base_subquery = base_stmt.subquery() + + # Calculate total counts per (span/trace)+name for computing fractions + entity_totals = ( + select( + base_subquery.c.entity_id, + base_subquery.c.name, + func.sum(base_subquery.c.label_count).label("total_label_count"), + func.sum(base_subquery.c.score_count).label("total_score_count"), + func.sum(base_subquery.c.score_sum).label("entity_score_sum"), + ) + .group_by(base_subquery.c.entity_id, base_subquery.c.name) + .subquery() + ) + + per_entity_fractions = ( + select( + base_subquery.c.entity_id, + base_subquery.c.name, + base_subquery.c.label, + base_subquery.c.record_count, + base_subquery.c.label_count, + base_subquery.c.score_count, + base_subquery.c.score_sum, + # Calculate label fraction + (base_subquery.c.label_count * 1.0 / entity_totals.c.total_label_count).label( + "label_fraction" + ), + # Calculate average score for the entity (if there are any scores) + case( + ( + entity_totals.c.total_score_count > 0, + entity_totals.c.entity_score_sum * 1.0 / entity_totals.c.total_score_count, + ), + else_=None, + ).label("entity_avg_score"), + ) + .join( + entity_totals, + and_( + base_subquery.c.entity_id == entity_totals.c.entity_id, + base_subquery.c.name == entity_totals.c.name, + ), + ) + .subquery() + ) + + # Aggregate metrics across (spans/traces) for each name+label combination. + label_entity_metrics = ( + select( + per_entity_fractions.c.name, + per_entity_fractions.c.label, + func.count(distinct(per_entity_fractions.c.entity_id)).label("entities_with_label"), + func.sum(per_entity_fractions.c.label_count).label("total_label_count"), + func.sum(per_entity_fractions.c.score_count).label("total_score_count"), + func.sum(per_entity_fractions.c.score_sum).label("total_score_sum"), + # Average of label fractions for entities that have this label + func.avg(per_entity_fractions.c.label_fraction).label("avg_label_fraction_present"), + # Average of per-entity average scores (but we handle overall aggregation separately) + ) + .group_by(per_entity_fractions.c.name, per_entity_fractions.c.label) + .subquery() + ) + + # Compute distinct per-entity average scores to ensure each entity counts only once. + distinct_entity_scores = ( + select( + per_entity_fractions.c.entity_id, + per_entity_fractions.c.name, + per_entity_fractions.c.entity_avg_score, + ) + .distinct() + .subquery() + ) + + overall_score_aggregates = ( + select( + distinct_entity_scores.c.name, + func.avg(distinct_entity_scores.c.entity_avg_score).label("overall_avg_score"), + ) + .group_by(distinct_entity_scores.c.name) + .subquery() + ) + + # Final result: adjust label fractions by the proportion of entities reporting this label + # and include the overall average score per annotation name. + final_stmt = ( + select( + label_entity_metrics.c.name, + label_entity_metrics.c.label, + ( + label_entity_metrics.c.avg_label_fraction_present + * label_entity_metrics.c.entities_with_label + / entity_count_subquery.c.entity_count + ).label("avg_label_fraction"), + overall_score_aggregates.c.overall_avg_score.label("avg_score"), # same for all labels + label_entity_metrics.c.total_label_count.label("label_count"), + label_entity_metrics.c.total_score_count.label("score_count"), + label_entity_metrics.c.total_score_sum.label("score_sum"), + label_entity_metrics.c.entities_with_label.label("record_count"), + ) + .join(entity_count_subquery, label_entity_metrics.c.name == entity_count_subquery.c.name) + .join( + overall_score_aggregates, + label_entity_metrics.c.name == overall_score_aggregates.c.name, + ) + .order_by(label_entity_metrics.c.name, label_entity_metrics.c.label) + ) + + return final_stmt diff --git a/src/phoenix/server/api/routers/v1/spans.py b/src/phoenix/server/api/routers/v1/spans.py index 08f125a0df..3903068d0a 100644 --- a/src/phoenix/server/api/routers/v1/spans.py +++ b/src/phoenix/server/api/routers/v1/spans.py @@ -7,7 +7,7 @@ import pandas as pd from fastapi import APIRouter, Header, HTTPException, Query from pydantic import Field -from sqlalchemy import select +from sqlalchemy import insert, select from starlette.requests import Request from starlette.responses import Response, StreamingResponse from starlette.status import HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY @@ -16,8 +16,7 @@ from phoenix.config import DEFAULT_PROJECT_NAME from phoenix.datetime_utils import normalize_datetime from phoenix.db import models -from phoenix.db.helpers import SupportedSQLDialect -from phoenix.db.insertion.helpers import as_kv, insert_on_conflict +from phoenix.db.insertion.helpers import as_kv from phoenix.db.insertion.types import Precursors from phoenix.server.api.routers.utils import df_to_bytes from phoenix.server.dml_event import SpanAnnotationInsertEvent @@ -211,7 +210,7 @@ class AnnotateSpansResponseBody(ResponseBody[list[InsertedSpanAnnotation]]): @router.post( "/span_annotations", operation_id="annotateSpans", - summary="Create or update span annotations", + summary="Create span annotations", responses=add_errors_to_responses( [{"status_code": HTTP_404_NOT_FOUND, "description": "Span not found"}] ), @@ -246,18 +245,12 @@ async def annotate_spans( status_code=HTTP_404_NOT_FOUND, ) inserted_ids = [] - dialect = SupportedSQLDialect(session.bind.dialect.name) for p in precursors: values = dict(as_kv(p.as_insertable(existing_spans[p.span_id]).row)) - span_annotation_id = await session.scalar( - insert_on_conflict( - values, - dialect=dialect, - table=models.SpanAnnotation, - unique_by=("name", "span_rowid"), - ).returning(models.SpanAnnotation.id) + result = await session.execute( + insert(models.SpanAnnotation).values(**values).returning(models.SpanAnnotation.id) ) - inserted_ids.append(span_annotation_id) + inserted_ids.append(result.scalar_one()) request.state.event_queue.put(SpanAnnotationInsertEvent(tuple(inserted_ids))) return AnnotateSpansResponseBody( data=[ diff --git a/src/phoenix/server/api/routers/v1/traces.py b/src/phoenix/server/api/routers/v1/traces.py index 1d352b5f76..3a5f835018 100644 --- a/src/phoenix/server/api/routers/v1/traces.py +++ b/src/phoenix/server/api/routers/v1/traces.py @@ -10,7 +10,7 @@ ExportTraceServiceResponse, ) from pydantic import Field -from sqlalchemy import select +from sqlalchemy import insert, select from starlette.concurrency import run_in_threadpool from starlette.datastructures import State from starlette.requests import Request @@ -23,8 +23,7 @@ from strawberry.relay import GlobalID from phoenix.db import models -from phoenix.db.helpers import SupportedSQLDialect -from phoenix.db.insertion.helpers import as_kv, insert_on_conflict +from phoenix.db.insertion.helpers import as_kv from phoenix.db.insertion.types import Precursors from phoenix.server.dml_event import TraceAnnotationInsertEvent from phoenix.trace.otel import decode_otlp_span @@ -144,7 +143,7 @@ class AnnotateTracesResponseBody(ResponseBody[list[InsertedTraceAnnotation]]): @router.post( "/trace_annotations", operation_id="annotateTraces", - summary="Create or update trace annotations", + summary="Create trace annotations", responses=add_errors_to_responses( [{"status_code": HTTP_404_NOT_FOUND, "description": "Trace not found"}] ), @@ -178,16 +177,10 @@ async def annotate_traces( status_code=HTTP_404_NOT_FOUND, ) inserted_ids = [] - dialect = SupportedSQLDialect(session.bind.dialect.name) for p in precursors: values = dict(as_kv(p.as_insertable(existing_traces[p.trace_id]).row)) trace_annotation_id = await session.scalar( - insert_on_conflict( - values, - dialect=dialect, - table=models.TraceAnnotation, - unique_by=("name", "trace_rowid"), - ).returning(models.TraceAnnotation.id) + insert(models.TraceAnnotation).values(**values).returning(models.TraceAnnotation.id) ) inserted_ids.append(trace_annotation_id) request.state.event_queue.put(TraceAnnotationInsertEvent(tuple(inserted_ids))) diff --git a/src/phoenix/server/api/types/AnnotationSummary.py b/src/phoenix/server/api/types/AnnotationSummary.py index 06cd9bdb51..b2cf8877b7 100644 --- a/src/phoenix/server/api/types/AnnotationSummary.py +++ b/src/phoenix/server/api/types/AnnotationSummary.py @@ -23,28 +23,26 @@ def count(self) -> int: @strawberry.field def labels(self) -> list[str]: - return self.df.label.dropna().tolist() + unique_labels = self.df["label"].dropna().unique() + return [str(label) for label in unique_labels] @strawberry.field def label_fractions(self) -> list[LabelFraction]: - if not (n := self.df.label_count.sum()): - return [] return [ LabelFraction( - label=cast(str, row.label), - fraction=row.label_count / n, + label=row.label, + fraction=float(row.avg_label_fraction), ) - for row in self.df.loc[ - self.df.label.notna(), - ["label", "label_count"], - ].itertuples() + for row in self.df.itertuples() + if row.label is not None ] @strawberry.field def mean_score(self) -> Optional[float]: - if not (n := self.df.score_count.sum()): + avg_scores = self.df["avg_score"].dropna() + if avg_scores.empty: return None - return cast(float, self.df.score_sum.sum() / n) + return float(avg_scores.mean()) # all avg_scores should be the same @strawberry.field def score_count(self) -> int: diff --git a/tests/integration/db_migrations/test_up_and_down_migrations.py b/tests/integration/db_migrations/test_up_and_down_migrations.py index 5bea25777d..8cc70e21ee 100644 --- a/tests/integration/db_migrations/test_up_and_down_migrations.py +++ b/tests/integration/db_migrations/test_up_and_down_migrations.py @@ -293,3 +293,8 @@ def test_up_and_down_migrations( _up(_engine, _alembic_config, "bc8fea3c2bc8") _down(_engine, _alembic_config, "4ded9e43755f") _up(_engine, _alembic_config, "bc8fea3c2bc8") + + for _ in range(2): + _up(_engine, _alembic_config, "2f9d1a65945f") + _down(_engine, _alembic_config, "bc8fea3c2bc8") + _up(_engine, _alembic_config, "2f9d1a65945f") diff --git a/tests/unit/db/insertion/test_helpers.py b/tests/unit/db/insertion/test_helpers.py index cde9ebbe89..c2be25655d 100644 --- a/tests/unit/db/insertion/test_helpers.py +++ b/tests/unit/db/insertion/test_helpers.py @@ -1,5 +1,4 @@ from asyncio import sleep -from datetime import datetime import pytest from sqlalchemy import insert, select @@ -68,109 +67,37 @@ async def test_handles_conflicts_in_expected_manner( db: DbSessionFactory, ) -> None: async with db() as session: - project_rowid = await session.scalar( - insert(models.Project).values(dict(name="abc")).returning(models.Project.id) + project_id = await session.scalar( + insert(models.Project) + .values(dict(name="abc", description="initial description")) + .returning(models.Project.id) ) - trace_rowid = await session.scalar( - insert(models.Trace) - .values( - dict( - project_rowid=project_rowid, - trace_id="xyz", - start_time=datetime.now(), - end_time=datetime.now(), - ) - ) - .returning(models.Trace.id) - ) - record = await session.scalar( - insert(models.TraceAnnotation) - .values( - dict( - name="uvw", - trace_rowid=trace_rowid, - annotator_kind="LLM", - score=12, - label="ijk", - metadata_={"1": "2"}, - ) - ) - .returning(models.TraceAnnotation) + project_record = await session.scalar( + select(models.Project).where(models.Project.id == project_id) ) - anno = await session.scalar( - select(models.TraceAnnotation) - .where(models.TraceAnnotation.trace_rowid == trace_rowid) - .order_by(models.TraceAnnotation.created_at) - ) - assert anno is not None - assert record is not None - assert anno.id == record.id - assert anno.created_at == record.created_at - assert anno.name == record.name - assert anno.trace_rowid == record.trace_rowid - assert anno.updated_at == record.updated_at - assert anno.score == record.score - assert anno.label == record.label - assert anno.explanation == record.explanation - assert anno.metadata_ == record.metadata_ - - await sleep(1) # increment `updated_at` by 1 second + assert project_record is not None async with db() as session: dialect = SupportedSQLDialect(session.bind.dialect.name) + new_values = dict(name="abc", description="updated description") + await sleep(1) await session.execute( insert_on_conflict( - dict( - name="uvw", - trace_rowid=trace_rowid, - annotator_kind="LLM", - score=None, - metadata_={}, - ), - dict( - name="rst", - trace_rowid=trace_rowid, - annotator_kind="LLM", - score=12, - metadata_={"1": "2"}, - ), - dict( - name="uvw", - trace_rowid=trace_rowid, - annotator_kind="HUMAN", - score=21, - metadata_={"2": "1"}, - ), + new_values, dialect=dialect, - table=models.TraceAnnotation, - unique_by=("name", "trace_rowid"), + table=models.Project, + unique_by=("name",), on_conflict=on_conflict, ) ) - annos = list( - await session.scalars( - select(models.TraceAnnotation) - .where(models.TraceAnnotation.trace_rowid == trace_rowid) - .order_by(models.TraceAnnotation.created_at) - ) + updated_project = await session.scalar( + select(models.Project).where(models.Project.id == project_id) ) - assert len(annos) == 2 - anno = annos[0] - assert anno.id == record.id - assert anno.created_at == record.created_at - assert anno.name == record.name - assert anno.trace_rowid == record.trace_rowid + assert updated_project is not None + if on_conflict is OnConflict.DO_NOTHING: - assert anno.updated_at == record.updated_at - assert anno.annotator_kind == record.annotator_kind - assert anno.score == record.score - assert anno.label == record.label - assert anno.explanation == record.explanation - assert anno.metadata_ == record.metadata_ + assert updated_project.description == "initial description" + assert updated_project.updated_at == project_record.updated_at else: - assert anno.updated_at > record.updated_at - assert anno.annotator_kind != record.annotator_kind - assert anno.score == 21 - assert anno.label is None - assert anno.explanation is None - assert anno.metadata_ == {"2": "1"} + assert updated_project.description == "updated description" + assert updated_project.updated_at > project_record.updated_at diff --git a/tests/unit/server/api/dataloaders/conftest.py b/tests/unit/server/api/dataloaders/conftest.py index ed48674ffd..44481addea 100644 --- a/tests/unit/server/api/dataloaders/conftest.py +++ b/tests/unit/server/api/dataloaders/conftest.py @@ -108,3 +108,228 @@ async def data_for_testing_dataloaders( annotator_kind="LLM", ) ) + + +@pytest.fixture +async def data_with_multiple_annotations(db: DbSessionFactory) -> None: + """ + Creates one project, one trace, and three spans for testing "quality" annotations. + + Span 1: two "good" annotations (scores: 0.85, 0.95) and one "bad" (0.3). + Span 2: one "good" (0.85) and one "bad" (0.3). + Span 3: one "good" (0.85). + + The fixture uses fixed values for span attributes so that non-null constraints are met. + """ + orig_time = datetime.fromisoformat("2021-01-01T00:00:00.000+00:00") + async with db() as session: + project_id = await session.scalar( + insert(models.Project).values(name="simple_multiple").returning(models.Project.id) + ) + trace_id = await session.scalar( + insert(models.Trace) + .values( + trace_id="trace1", + project_rowid=project_id, + start_time=orig_time, + end_time=orig_time + timedelta(minutes=1), + ) + .returning(models.Trace.id) + ) + span_ids = [] + for i in range(3): + span_id_val = await session.scalar( + insert(models.Span) + .values( + trace_rowid=trace_id, + span_id=f"span{i+1}", + name=f"span{i+1}", + parent_id="", + span_kind="UNKNOWN", + start_time=orig_time + timedelta(seconds=10 * i), + end_time=orig_time + timedelta(seconds=10 * i + 5), + attributes={"llm": {"token_count": {"prompt": 100, "completion": 100}}}, + events=[], # ensure non-null list + status_code="OK", + status_message="okay", + cumulative_error_count=0, + cumulative_llm_token_count_prompt=0, + cumulative_llm_token_count_completion=0, + llm_token_count_prompt=100, + llm_token_count_completion=100, + ) + .returning(models.Span.id) + ) + span_ids.append(span_id_val) + # Span 1 annotations + await session.execute( + insert(models.SpanAnnotation).values( + name="quality", + span_rowid=span_ids[0], + label="good", + score=0.85, + metadata_={}, + annotator_kind="LLM", + ) + ) + await session.execute( + insert(models.SpanAnnotation).values( + name="quality", + span_rowid=span_ids[0], + label="good", + score=0.95, + metadata_={}, + annotator_kind="LLM", + ) + ) + await session.execute( + insert(models.SpanAnnotation).values( + name="quality", + span_rowid=span_ids[0], + label="bad", + score=0.3, + metadata_={}, + annotator_kind="LLM", + ) + ) + # Span 2 annotations + await session.execute( + insert(models.SpanAnnotation).values( + name="quality", + span_rowid=span_ids[1], + label="good", + score=0.85, + metadata_={}, + annotator_kind="LLM", + ) + ) + await session.execute( + insert(models.SpanAnnotation).values( + name="quality", + span_rowid=span_ids[1], + label="bad", + score=0.3, + metadata_={}, + annotator_kind="LLM", + ) + ) + # Span 3 annotations + await session.execute( + insert(models.SpanAnnotation).values( + name="quality", + span_rowid=span_ids[2], + label="good", + score=0.85, + metadata_={}, + annotator_kind="LLM", + ) + ) + await session.commit() + + +@pytest.fixture +async def data_with_missing_labels(db: DbSessionFactory) -> None: + """ + Creates one project, one trace, and three spans for testing "distribution" annotations. + + Span 1: two "X" annotations (score=0.8) and one "Y" annotation (score=0.6). + Span 2: one "X" annotation (score=0.8). + Span 3: one "X" annotation (score=0.8). + + Non-null constraints are satisfied by providing fixed attributes and events. + """ + orig_time = datetime.fromisoformat("2021-01-01T00:00:00.000+00:00") + async with db() as session: + project_id = await session.scalar( + insert(models.Project).values(name="simple_missing").returning(models.Project.id) + ) + trace_id = await session.scalar( + insert(models.Trace) + .values( + trace_id="trace_missing", + project_rowid=project_id, + start_time=orig_time, + end_time=orig_time + timedelta(minutes=1), + ) + .returning(models.Trace.id) + ) + + span_ids = [] + for i in range(3): + span_id_val = await session.scalar( + insert(models.Span) + .values( + trace_rowid=trace_id, + span_id=f"missing_span{i+1}", + name=f"missing_span{i+1}", + parent_id="", + span_kind="UNKNOWN", + start_time=orig_time + timedelta(seconds=10 * i), + end_time=orig_time + timedelta(seconds=10 * i + 5), + attributes={"llm": {"token_count": {"prompt": 100, "completion": 100}}}, + events=[], + status_code="OK", + status_message="okay", + cumulative_error_count=0, + cumulative_llm_token_count_prompt=0, + cumulative_llm_token_count_completion=0, + llm_token_count_prompt=100, + llm_token_count_completion=100, + ) + .returning(models.Span.id) + ) + span_ids.append(span_id_val) + # Span 1: two "X" and one "Y" + await session.execute( + insert(models.SpanAnnotation).values( + name="distribution", + span_rowid=span_ids[0], + label="X", + score=0.8, + metadata_={}, + annotator_kind="LLM", + ) + ) + await session.execute( + insert(models.SpanAnnotation).values( + name="distribution", + span_rowid=span_ids[0], + label="X", + score=0.8, + metadata_={}, + annotator_kind="LLM", + ) + ) + await session.execute( + insert(models.SpanAnnotation).values( + name="distribution", + span_rowid=span_ids[0], + label="Y", + score=0.6, + metadata_={}, + annotator_kind="LLM", + ) + ) + # Span 2: only "X" + await session.execute( + insert(models.SpanAnnotation).values( + name="distribution", + span_rowid=span_ids[1], + label="X", + score=0.8, + metadata_={}, + annotator_kind="LLM", + ) + ) + # Span 3: only "X" + await session.execute( + insert(models.SpanAnnotation).values( + name="distribution", + span_rowid=span_ids[2], + label="X", + score=0.8, + metadata_={}, + annotator_kind="LLM", + ) + ) + await session.commit() diff --git a/tests/unit/server/api/dataloaders/test_annotation_summaries.py b/tests/unit/server/api/dataloaders/test_annotation_summaries.py index 9ca00d6c77..71551ac2b5 100644 --- a/tests/unit/server/api/dataloaders/test_annotation_summaries.py +++ b/tests/unit/server/api/dataloaders/test_annotation_summaries.py @@ -77,3 +77,90 @@ async def test_evaluation_summaries( summary.mean_score(), # type: ignore[call-arg] ) assert actual == pytest.approx(expected, 1e-7) + + +async def test_multiple_annotations_score_weighting( + db: DbSessionFactory, + data_with_multiple_annotations: None, +) -> None: + # Using the "quality" annotations fixture. + start_time = datetime.fromisoformat("2021-01-01T00:00:00.000+00:00") + end_time = datetime.fromisoformat("2021-01-01T01:00:00.000+00:00") + # Based on the fixture: + # Span 1: avg score = (0.85+0.95+0.3)/3 = 0.70 + # Span 2: avg score = (0.85+0.3)/2 = 0.575 + # Span 3: avg score = 0.85 + # Overall average score = (0.70+0.575+0.85)/3 ≈ 0.70833. + expected_avg_score = 0.70833 + + async with db() as session: + project_id = await session.scalar( + select(models.Project.id).where(models.Project.name == "simple_multiple") + ) + assert isinstance(project_id, int) + + loader = AnnotationSummaryDataLoader(db) + result = await loader.load( + ( + "span", + project_id, + TimeRange(start=start_time, end=end_time), + None, + "quality", + ) + ) + assert result is not None + assert result.mean_score() == pytest.approx(expected_avg_score, rel=1e-4) # type: ignore[call-arg] + + # Expected fractions: + # "good": (2/3 + 1/2 + 1) / 3 ≈ 0.722 + # "bad": (1/3 + 1/2 + 0) / 3 ≈ 0.277 + label_fracs = {lf.label: lf.fraction for lf in result.label_fractions()} # type: ignore[call-arg, attr-defined] + assert label_fracs["good"] == pytest.approx(0.722, rel=1e-2) + assert label_fracs["bad"] == pytest.approx(0.277, rel=1e-2) + assert abs(label_fracs["good"] + label_fracs["bad"] - 1.0) < 1e-2 + + +async def test_missing_label_aggregation( + db: DbSessionFactory, + data_with_missing_labels: None, +) -> None: + # Using the "distribution" annotations fixture. + start_time = datetime.fromisoformat("2021-01-01T00:00:00.000+00:00") + end_time = datetime.fromisoformat("2021-01-01T01:00:00.000+00:00") + # Based on the fixture: + # Span 1: For "distribution": "X" fraction = 2/3, "Y" fraction = 1/3. + # Span 2: "X" fraction = 1. + # Span 3: "X" fraction = 1. + # Overall label fractions for "distribution" annotation: + # "X": (0.667 + 1 + 1) / 3 ≈ 0.889, + # "Y": (0.333 + 0 + 0) / 3 ≈ 0.111. + loader = AnnotationSummaryDataLoader(db) + + async with db() as session: + project_id = await session.scalar( + select(models.Project.id).where(models.Project.name == "simple_missing") + ) + assert isinstance(project_id, int) + result = await loader.load( + ( + "span", + project_id, + TimeRange(start=start_time, end=end_time), + None, + "distribution", + ) + ) + assert result is not None + + label_fracs = {lf.label: lf.fraction for lf in result.label_fractions()} # type: ignore[call-arg, attr-defined] + assert label_fracs["X"] == pytest.approx(0.889, rel=1e-2) + assert label_fracs["Y"] == pytest.approx(0.111, rel=1e-2) + assert abs(sum(label_fracs.values()) - 1.0) < 1e-7 + + # The "distribution" annotation is grouped as follows: + # Span 1: .8, .8, .6 + # Span 2: .8 + # Span 3: .8 + # Overall average = ((0.8 + 0.8 + 0.6) / 3 + 0.8 + 0.8) / 3 ≈ 0.777 + assert result.mean_score() == pytest.approx(0.777, rel=1e-2) # type: ignore[call-arg] From 5ae0274bee39c2ecae699e22ed120d085c9195c9 Mon Sep 17 00:00:00 2001 From: Dustin Ngo Date: Wed, 19 Mar 2025 17:06:59 -0400 Subject: [PATCH 03/53] feat: Annotation Configurations (#6495) Co-authored-by: Alexander Song --- app/schema.graphql | 172 + .../src/__generated__/api/v1.ts | 5756 +++++++++-------- schemas/openapi.json | 1037 +-- ...f9d1a65945f_annotation_config_migration.py | 108 + src/phoenix/db/models.py | 109 + src/phoenix/server/api/mutations/__init__.py | 2 + .../mutations/annotation_config_mutations.py | 418 ++ src/phoenix/server/api/queries.py | 31 + src/phoenix/server/api/routers/v1/__init__.py | 2 + .../api/routers/v1/annotation_configs.py | 289 + .../server/api/types/AnnotationConfig.py | 131 + src/phoenix/server/api/types/Project.py | 41 + 12 files changed, 5037 insertions(+), 3059 deletions(-) create mode 100644 src/phoenix/server/api/mutations/annotation_config_mutations.py create mode 100644 src/phoenix/server/api/routers/v1/annotation_configs.py create mode 100644 src/phoenix/server/api/types/AnnotationConfig.py diff --git a/app/schema.graphql b/app/schema.graphql index 93f8d08a0a..b2de379956 100644 --- a/app/schema.graphql +++ b/app/schema.graphql @@ -1,5 +1,15 @@ directive @oneOf on INPUT_OBJECT +input AddAnnotationConfigToProjectInput { + projectId: GlobalID! + annotationConfigId: GlobalID! +} + +type AddAnnotationConfigToProjectPayload { + query: Query! + project: Project! +} + input AddExamplesToDatasetInput { datasetId: GlobalID! examples: [DatasetExampleInput!]! @@ -32,6 +42,26 @@ interface Annotation { explanation: String } +union AnnotationConfig = CategoricalAnnotationConfig | ContinuousAnnotationConfig | FreeformAnnotationConfig + +"""A connection to a list of items.""" +type AnnotationConfigConnection { + """Pagination data for this connection""" + pageInfo: PageInfo! + + """Contains the nodes in this connection""" + edges: [AnnotationConfigEdge!]! +} + +"""An edge in a connection.""" +type AnnotationConfigEdge { + """A cursor for use in pagination""" + cursor: String! + + """The item at the end of the edge""" + node: AnnotationConfig! +} + type AnnotationSummary { count: Int! labels: [String!]! @@ -41,6 +71,12 @@ type AnnotationSummary { labelCount: Int! } +enum AnnotationType { + CATEGORICAL + CONTINUOUS + FREEFORM +} + enum AnnotatorKind { LLM HUMAN @@ -99,6 +135,26 @@ enum CanonicalParameterName { ANTHROPIC_EXTENDED_THINKING } +type CategoricalAnnotationConfig implements Node { + """The Globally Unique ID of this object""" + id: GlobalID! + name: String! + annotationType: AnnotationType! + description: String + optimizationDirection: OptimizationDirection! + values: [CategoricalAnnotationValue!]! +} + +type CategoricalAnnotationValue { + label: String! + score: Float +} + +input CategoricalAnnotationValueInput { + label: String! + score: Float = null +} + type ChatCompletionFunctionCall { name: String! arguments: String! @@ -263,12 +319,35 @@ input ContentPartInput @oneOf { toolResult: ToolResultContentValueInput } +type ContinuousAnnotationConfig implements Node { + """The Globally Unique ID of this object""" + id: GlobalID! + name: String! + annotationType: AnnotationType! + description: String + optimizationDirection: OptimizationDirection! + lowerBound: Float + upperBound: Float +} + input CreateApiKeyInput { name: String! description: String expiresAt: DateTime } +input CreateCategoricalAnnotationConfigInput { + name: String! + optimizationDirection: OptimizationDirection! + description: String = null + values: [CategoricalAnnotationValueInput!]! +} + +type CreateCategoricalAnnotationConfigPayload { + query: Query! + annotationConfig: CategoricalAnnotationConfig! +} + input CreateChatPromptInput { name: Identifier! description: String = null @@ -281,12 +360,35 @@ input CreateChatPromptVersionInput { tags: [SetPromptVersionTagInput!] = null } +input CreateContinuousAnnotationConfigInput { + name: String! + optimizationDirection: OptimizationDirection! + description: String = null + lowerBound: Float = null + upperBound: Float = null +} + +type CreateContinuousAnnotationConfigPayload { + query: Query! + annotationConfig: ContinuousAnnotationConfig! +} + input CreateDatasetInput { name: String! description: String metadata: JSON } +input CreateFreeformAnnotationConfigInput { + name: String! + description: String = null +} + +type CreateFreeformAnnotationConfigPayload { + query: Query! + annotationConfig: FreeformAnnotationConfig! +} + input CreatePromptLabelInput { name: Identifier! description: String = null @@ -518,6 +620,15 @@ type DbTableStats { numBytes: Float! } +input DeleteAnnotationConfigInput { + configId: GlobalID! +} + +type DeleteAnnotationConfigPayload { + query: Query! + annotationConfig: AnnotationConfig! +} + input DeleteAnnotationsInput { annotationIds: [GlobalID!]! } @@ -1007,6 +1118,14 @@ type FloatInvocationParameter implements InvocationParameterBase { defaultValue: Float } +type FreeformAnnotationConfig implements Node { + """The Globally Unique ID of this object""" + id: GlobalID! + name: String! + annotationType: AnnotationType! + description: String +} + type FunctionCallChunk implements ChatCompletionSubscriptionPayload { datasetExampleId: GlobalID name: String! @@ -1222,6 +1341,14 @@ input ModelsInput { } type Mutation { + createCategoricalAnnotationConfig(input: CreateCategoricalAnnotationConfigInput!): CreateCategoricalAnnotationConfigPayload! + createContinuousAnnotationConfig(input: CreateContinuousAnnotationConfigInput!): CreateContinuousAnnotationConfigPayload! + createFreeformAnnotationConfig(input: CreateFreeformAnnotationConfigInput!): CreateFreeformAnnotationConfigPayload! + updateCategoricalAnnotationConfig(input: UpdateCategoricalAnnotationConfigInput!): UpdateCategoricalAnnotationConfigPayload! + updateContinuousAnnotationConfig(input: UpdateContinuousAnnotationConfigInput!): UpdateContinuousAnnotationConfigPayload! + updateFreeformAnnotationConfig(input: UpdateFreeformAnnotationConfigInput!): UpdateFreeformAnnotationConfigPayload! + deleteAnnotationConfig(input: DeleteAnnotationConfigInput!): DeleteAnnotationConfigPayload! + addAnnotationConfigToProject(input: [AddAnnotationConfigToProjectInput!]!): AddAnnotationConfigToProjectPayload! createSystemApiKey(input: CreateApiKeyInput!): CreateSystemApiKeyMutationPayload! createUserApiKey(input: CreateUserApiKeyInput!): CreateUserApiKeyMutationPayload! deleteSystemApiKey(input: DeleteApiKeyInput!): DeleteApiKeyMutationPayload! @@ -1288,6 +1415,11 @@ type NumericRange { end: Float! } +enum OptimizationDirection { + MINIMIZE + MAXIMIZE +} + """Information to aid in pagination.""" type PageInfo { """When paginating forwards, are there more items?""" @@ -1411,6 +1543,7 @@ type Project implements Node { documentEvaluationSummary(evaluationName: String!, timeRange: TimeRange, filterCondition: String): DocumentEvaluationSummary streamingLastUpdatedAt: DateTime validateSpanFilterCondition(condition: String!): ValidationResult! + annotationConfigs(first: Int = 50, last: Int = null, after: String = null, before: String = null): AnnotationConfigConnection! } """A connection to a list of items.""" @@ -1672,6 +1805,7 @@ type Query { viewer: User prompts(first: Int = 50, last: Int, after: String, before: String): PromptConnection! promptLabels(first: Int = 50, last: Int, after: String, before: String): PromptLabelConnection! + annotationConfigs(first: Int = 50, last: Int = null, after: String = null, before: String = null): AnnotationConfigConnection! clusters(clusters: [ClusterInput!]!): [Cluster!]! hdbscanClustering( """Event ID of the coordinates""" @@ -2192,6 +2326,44 @@ input UnsetPromptLabelInput { promptLabelId: GlobalID! } +input UpdateCategoricalAnnotationConfigInput { + configId: GlobalID! + name: String! + optimizationDirection: OptimizationDirection! + description: String = null + values: [CategoricalAnnotationValueInput!]! +} + +type UpdateCategoricalAnnotationConfigPayload { + query: Query! + annotationConfig: CategoricalAnnotationConfig! +} + +input UpdateContinuousAnnotationConfigInput { + configId: GlobalID! + name: String! + optimizationDirection: OptimizationDirection! + description: String = null + lowerBound: Float = null + upperBound: Float = null +} + +type UpdateContinuousAnnotationConfigPayload { + query: Query! + annotationConfig: ContinuousAnnotationConfig! +} + +input UpdateFreeformAnnotationConfigInput { + configId: GlobalID! + name: String! + description: String = null +} + +type UpdateFreeformAnnotationConfigPayload { + query: Query! + annotationConfig: FreeformAnnotationConfig! +} + type User implements Node { """The Globally Unique ID of this object""" id: GlobalID! diff --git a/js/packages/phoenix-client/src/__generated__/api/v1.ts b/js/packages/phoenix-client/src/__generated__/api/v1.ts index 32e42b2c78..e3344b68df 100644 --- a/js/packages/phoenix-client/src/__generated__/api/v1.ts +++ b/js/packages/phoenix-client/src/__generated__/api/v1.ts @@ -4,2667 +4,3113 @@ */ export interface paths { - "/v1/datasets": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List datasets */ - get: operations["listDatasets"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/datasets/{id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get dataset by ID */ - get: operations["getDataset"]; - put?: never; - post?: never; - /** Delete dataset by ID */ - delete: operations["deleteDatasetById"]; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/datasets/{id}/versions": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List dataset versions */ - get: operations["listDatasetVersionsByDatasetId"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/datasets/upload": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** Upload dataset from JSON, CSV, or PyArrow */ - post: operations["uploadDataset"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/datasets/{id}/examples": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get examples from a dataset */ - get: operations["getDatasetExamples"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/datasets/{id}/csv": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Download dataset examples as CSV file */ - get: operations["getDatasetCsv"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/datasets/{id}/jsonl/openai_ft": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Download dataset examples as OpenAI fine-tuning JSONL file */ - get: operations["getDatasetJSONLOpenAIFineTuning"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/datasets/{id}/jsonl/openai_evals": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Download dataset examples as OpenAI evals JSONL file */ - get: operations["getDatasetJSONLOpenAIEvals"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/datasets/{dataset_id}/experiments": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** List experiments by dataset */ - get: operations["listExperiments"]; - put?: never; - /** Create experiment on a dataset */ - post: operations["createExperiment"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/experiments/{experiment_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get experiment by ID */ - get: operations["getExperiment"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/experiments/{experiment_id}/json": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Download experiment runs as a JSON file */ - get: operations["getExperimentJSON"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/experiments/{experiment_id}/csv": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Download experiment runs as a CSV file */ - get: operations["getExperimentCSV"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/span_annotations": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** Create span annotations */ - post: operations["annotateSpans"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/evaluations": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** Get span, trace, or document evaluations from a project */ - get: operations["getEvaluations"]; - put?: never; - /** Add span, trace, or document evaluations */ - post: operations["addEvaluations"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/prompts": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * List all prompts - * @description Retrieve a paginated list of all prompts in the system. A prompt can have multiple versions. - */ - get: operations["getPrompts"]; - put?: never; - /** - * Create a new prompt - * @description Create a new prompt and its initial version. A prompt can have multiple versions. - */ - post: operations["postPromptVersion"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/prompts/{prompt_identifier}/versions": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * List prompt versions - * @description Retrieve all versions of a specific prompt with pagination support. Each prompt can have multiple versions with different configurations. - */ - get: operations["listPromptVersions"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/prompt_versions/{prompt_version_id}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get prompt version by ID - * @description Retrieve a specific prompt version using its unique identifier. A prompt version contains the actual template and configuration. - */ - get: operations["getPromptVersionByPromptVersionId"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/prompts/{prompt_identifier}/tags/{tag_name}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get prompt version by tag - * @description Retrieve a specific prompt version using its tag name. Tags are used to identify specific versions of a prompt. - */ - get: operations["getPromptVersionByTagName"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/prompts/{prompt_identifier}/latest": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get latest prompt version - * @description Retrieve the most recent version of a specific prompt. - */ - get: operations["getPromptVersionLatest"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/prompt_versions/{prompt_version_id}/tags": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * List prompt version tags - * @description Retrieve all tags associated with a specific prompt version. Tags are used to identify and categorize different versions of a prompt. - */ - get: operations["getPromptVersionTags"]; - put?: never; - /** - * Add tag to prompt version - * @description Add a new tag to a specific prompt version. Tags help identify and categorize different versions of a prompt. - */ - post: operations["createPromptVersionTag"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/projects": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * List all projects - * @description Retrieve a paginated list of all projects in the system. - */ - get: operations["getProjects"]; - put?: never; - /** - * Create a new project - * @description Create a new project with the specified configuration. - */ - post: operations["createProject"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/v1/projects/{project_identifier}": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * Get project by ID or name - * @description Retrieve a specific project using its unique identifier: either project ID or project name. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters. - */ - get: operations["getProject"]; - /** - * Update a project by ID or name - * @description Update an existing project with new configuration. Project names cannot be changed. The project identifier is either project ID or project name. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters. - */ - put: operations["updateProject"]; - post?: never; - /** - * Delete a project by ID or name - * @description Delete an existing project and all its associated data. The project identifier is either project ID or project name. The default project cannot be deleted. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters. - */ - delete: operations["deleteProject"]; - options?: never; - head?: never; - patch?: never; - trace?: never; + "/v1/annotation_configs": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; }; + /** List annotation configurations */ + get: operations["list_annotation_configs_v1_annotation_configs_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/annotation_configs/{config_identifier}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get an annotation configuration by ID or name */ + get: operations["get_annotation_config_by_name_or_id_v1_annotation_configs__config_identifier__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/annotation_configs/continuous": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create a continuous annotation configuration */ + post: operations["create_continuous_annotation_config_v1_annotation_configs_continuous_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/annotation_configs/categorical": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create a categorical annotation configuration */ + post: operations["create_categorical_annotation_config_v1_annotation_configs_categorical_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/annotation_configs/freeform": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create a freeform annotation configuration */ + post: operations["create_freeform_annotation_config_v1_annotation_configs_freeform_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/annotation_configs/{config_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + /** Delete an annotation configuration */ + delete: operations["delete_annotation_config_v1_annotation_configs__config_id__delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List datasets */ + get: operations["listDatasets"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get dataset by ID */ + get: operations["getDataset"]; + put?: never; + post?: never; + /** Delete dataset by ID */ + delete: operations["deleteDatasetById"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets/{id}/versions": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List dataset versions */ + get: operations["listDatasetVersionsByDatasetId"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets/upload": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Upload dataset from JSON, CSV, or PyArrow */ + post: operations["uploadDataset"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets/{id}/examples": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get examples from a dataset */ + get: operations["getDatasetExamples"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets/{id}/csv": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Download dataset examples as CSV file */ + get: operations["getDatasetCsv"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets/{id}/jsonl/openai_ft": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Download dataset examples as OpenAI fine-tuning JSONL file */ + get: operations["getDatasetJSONLOpenAIFineTuning"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets/{id}/jsonl/openai_evals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Download dataset examples as OpenAI evals JSONL file */ + get: operations["getDatasetJSONLOpenAIEvals"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/datasets/{dataset_id}/experiments": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List experiments by dataset */ + get: operations["listExperiments"]; + put?: never; + /** Create experiment on a dataset */ + post: operations["createExperiment"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/experiments/{experiment_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get experiment by ID */ + get: operations["getExperiment"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/experiments/{experiment_id}/json": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Download experiment runs as a JSON file */ + get: operations["getExperimentJSON"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/experiments/{experiment_id}/csv": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Download experiment runs as a CSV file */ + get: operations["getExperimentCSV"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/span_annotations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create span annotations */ + post: operations["annotateSpans"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/evaluations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get span, trace, or document evaluations from a project */ + get: operations["getEvaluations"]; + put?: never; + /** Add span, trace, or document evaluations */ + post: operations["addEvaluations"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/prompts": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List all prompts + * @description Retrieve a paginated list of all prompts in the system. A prompt can have multiple versions. + */ + get: operations["getPrompts"]; + put?: never; + /** + * Create a new prompt + * @description Create a new prompt and its initial version. A prompt can have multiple versions. + */ + post: operations["postPromptVersion"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/prompts/{prompt_identifier}/versions": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List prompt versions + * @description Retrieve all versions of a specific prompt with pagination support. Each prompt can have multiple versions with different configurations. + */ + get: operations["listPromptVersions"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/prompt_versions/{prompt_version_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get prompt version by ID + * @description Retrieve a specific prompt version using its unique identifier. A prompt version contains the actual template and configuration. + */ + get: operations["getPromptVersionByPromptVersionId"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/prompts/{prompt_identifier}/tags/{tag_name}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get prompt version by tag + * @description Retrieve a specific prompt version using its tag name. Tags are used to identify specific versions of a prompt. + */ + get: operations["getPromptVersionByTagName"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/prompts/{prompt_identifier}/latest": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get latest prompt version + * @description Retrieve the most recent version of a specific prompt. + */ + get: operations["getPromptVersionLatest"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/prompt_versions/{prompt_version_id}/tags": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List prompt version tags + * @description Retrieve all tags associated with a specific prompt version. Tags are used to identify and categorize different versions of a prompt. + */ + get: operations["getPromptVersionTags"]; + put?: never; + /** + * Add tag to prompt version + * @description Add a new tag to a specific prompt version. Tags help identify and categorize different versions of a prompt. + */ + post: operations["createPromptVersionTag"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/projects": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List all projects + * @description Retrieve a paginated list of all projects in the system. + */ + get: operations["getProjects"]; + put?: never; + /** + * Create a new project + * @description Create a new project with the specified configuration. + */ + post: operations["createProject"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/projects/{project_identifier}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get project by ID or name + * @description Retrieve a specific project using its unique identifier: either project ID or project name. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters. + */ + get: operations["getProject"]; + /** + * Update a project by ID or name + * @description Update an existing project with new configuration. Project names cannot be changed. The project identifier is either project ID or project name. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters. + */ + put: operations["updateProject"]; + post?: never; + /** + * Delete a project by ID or name + * @description Delete an existing project and all its associated data. The project identifier is either project ID or project name. The default project cannot be deleted. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters. + */ + delete: operations["deleteProject"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; } export type webhooks = Record; export interface components { - schemas: { - /** AnnotateSpansRequestBody */ - AnnotateSpansRequestBody: { - /** Data */ - data: components["schemas"]["SpanAnnotation"][]; - }; - /** AnnotateSpansResponseBody */ - AnnotateSpansResponseBody: { - /** Data */ - data: components["schemas"]["InsertedSpanAnnotation"][]; - }; - /** - * CreateExperimentRequestBody - * @description Details of the experiment to be created - */ - CreateExperimentRequestBody: { - /** - * Name - * @description Name of the experiment (if omitted, a random name will be generated) - */ - name?: string | null; - /** - * Description - * @description An optional description of the experiment - */ - description?: string | null; - /** - * Metadata - * @description Metadata for the experiment - */ - metadata?: { - [key: string]: unknown; - } | null; - /** - * Version Id - * @description ID of the dataset version over which the experiment will be run (if omitted, the latest version will be used) - */ - version_id?: string | null; - /** - * Repetitions - * @description Number of times the experiment should be repeated for each example - * @default 1 - */ - repetitions: number; - }; - /** CreateExperimentResponseBody */ - CreateExperimentResponseBody: { - data: components["schemas"]["Experiment"]; - }; - /** CreateProjectRequestBody */ - CreateProjectRequestBody: { - /** Name */ - name: string; - /** Description */ - description?: string | null; - }; - /** CreateProjectResponseBody */ - CreateProjectResponseBody: { - data: components["schemas"]["Project"]; - }; - /** CreatePromptRequestBody */ - CreatePromptRequestBody: { - prompt: components["schemas"]["PromptData"]; - version: components["schemas"]["PromptVersionData"]; - }; - /** CreatePromptResponseBody */ - CreatePromptResponseBody: { - data: components["schemas"]["PromptVersion"]; - }; - /** Dataset */ - Dataset: { - /** Id */ - id: string; - /** Name */ - name: string; - /** Description */ - description: string | null; - /** Metadata */ - metadata: { - [key: string]: unknown; - }; - /** - * Created At - * Format: date-time - */ - created_at: string; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - }; - /** DatasetExample */ - DatasetExample: { - /** Id */ - id: string; - /** Input */ - input: { - [key: string]: unknown; - }; - /** Output */ - output: { - [key: string]: unknown; - }; - /** Metadata */ - metadata: { - [key: string]: unknown; - }; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - }; - /** DatasetVersion */ - DatasetVersion: { - /** Version Id */ - version_id: string; - /** Description */ - description: string | null; - /** Metadata */ - metadata: { - [key: string]: unknown; - }; - /** - * Created At - * Format: date-time - */ - created_at: string; - }; - /** DatasetWithExampleCount */ - DatasetWithExampleCount: { - /** Id */ - id: string; - /** Name */ - name: string; - /** Description */ - description: string | null; - /** Metadata */ - metadata: { - [key: string]: unknown; - }; - /** - * Created At - * Format: date-time - */ - created_at: string; - /** - * Updated At - * Format: date-time - */ - updated_at: string; - /** Example Count */ - example_count: number; - }; - /** Experiment */ - Experiment: { - /** - * Id - * @description The ID of the experiment - */ - id: string; - /** - * Dataset Id - * @description The ID of the dataset associated with the experiment - */ - dataset_id: string; - /** - * Dataset Version Id - * @description The ID of the dataset version associated with the experiment - */ - dataset_version_id: string; - /** - * Repetitions - * @description Number of times the experiment is repeated - */ - repetitions: number; - /** - * Metadata - * @description Metadata of the experiment - */ - metadata: { - [key: string]: unknown; - }; - /** - * Project Name - * @description The name of the project associated with the experiment - */ - project_name: string | null; - /** - * Created At - * Format: date-time - * @description The creation timestamp of the experiment - */ - created_at: string; - /** - * Updated At - * Format: date-time - * @description The last update timestamp of the experiment - */ - updated_at: string; - }; - /** GetDatasetResponseBody */ - GetDatasetResponseBody: { - data: components["schemas"]["DatasetWithExampleCount"]; - }; - /** GetExperimentResponseBody */ - GetExperimentResponseBody: { - data: components["schemas"]["Experiment"]; - }; - /** GetProjectResponseBody */ - GetProjectResponseBody: { - data: components["schemas"]["Project"]; - }; - /** GetProjectsResponseBody */ - GetProjectsResponseBody: { - /** Data */ - data: components["schemas"]["Project"][]; - /** Next Cursor */ - next_cursor: string | null; - }; - /** GetPromptResponseBody */ - GetPromptResponseBody: { - data: components["schemas"]["PromptVersion"]; - }; - /** GetPromptVersionTagsResponseBody */ - GetPromptVersionTagsResponseBody: { - /** Data */ - data: components["schemas"]["PromptVersionTag"][]; - /** Next Cursor */ - next_cursor: string | null; - }; - /** GetPromptVersionsResponseBody */ - GetPromptVersionsResponseBody: { - /** Data */ - data: components["schemas"]["PromptVersion"][]; - /** Next Cursor */ - next_cursor: string | null; - }; - /** GetPromptsResponseBody */ - GetPromptsResponseBody: { - /** Data */ - data: components["schemas"]["Prompt"][]; - /** Next Cursor */ - next_cursor: string | null; - }; - /** HTTPValidationError */ - HTTPValidationError: { - /** Detail */ - detail?: components["schemas"]["ValidationError"][]; - }; - /** Identifier */ - Identifier: string; - /** InsertedSpanAnnotation */ - InsertedSpanAnnotation: { - /** - * Id - * @description The ID of the inserted span annotation - */ - id: string; - }; - /** ListDatasetExamplesData */ - ListDatasetExamplesData: { - /** Dataset Id */ - dataset_id: string; - /** Version Id */ - version_id: string; - /** Examples */ - examples: components["schemas"]["DatasetExample"][]; - }; - /** ListDatasetExamplesResponseBody */ - ListDatasetExamplesResponseBody: { - data: components["schemas"]["ListDatasetExamplesData"]; - }; - /** ListDatasetVersionsResponseBody */ - ListDatasetVersionsResponseBody: { - /** Data */ - data: components["schemas"]["DatasetVersion"][]; - /** Next Cursor */ - next_cursor: string | null; - }; - /** ListDatasetsResponseBody */ - ListDatasetsResponseBody: { - /** Data */ - data: components["schemas"]["Dataset"][]; - /** Next Cursor */ - next_cursor: string | null; - }; - /** ListExperimentsResponseBody */ - ListExperimentsResponseBody: { - /** Data */ - data: components["schemas"]["Experiment"][]; - }; - /** - * ModelProvider - * @enum {string} - */ - ModelProvider: "OPENAI" | "AZURE_OPENAI" | "ANTHROPIC" | "GOOGLE"; - /** Project */ - Project: { - /** Name */ - name: string; - /** Description */ - description?: string | null; - /** Id */ - id: string; - }; - /** Prompt */ - Prompt: { - name: components["schemas"]["Identifier"]; - /** Description */ - description?: string | null; - /** Source Prompt Id */ - source_prompt_id?: string | null; - /** Id */ - id: string; - }; - /** PromptAnthropicInvocationParameters */ - PromptAnthropicInvocationParameters: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "anthropic"; - anthropic: components["schemas"]["PromptAnthropicInvocationParametersContent"]; - }; - /** PromptAnthropicInvocationParametersContent */ - PromptAnthropicInvocationParametersContent: { - /** Max Tokens */ - max_tokens: number; - /** Temperature */ - temperature?: number; - /** Top P */ - top_p?: number; - /** Stop Sequences */ - stop_sequences?: string[]; - /** Thinking */ - thinking?: components["schemas"]["PromptAnthropicThinkingConfigDisabled"] | components["schemas"]["PromptAnthropicThinkingConfigEnabled"]; - }; - /** PromptAnthropicThinkingConfigDisabled */ - PromptAnthropicThinkingConfigDisabled: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "disabled"; - }; - /** PromptAnthropicThinkingConfigEnabled */ - PromptAnthropicThinkingConfigEnabled: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "enabled"; - /** Budget Tokens */ - budget_tokens: number; - }; - /** PromptAzureOpenAIInvocationParameters */ - PromptAzureOpenAIInvocationParameters: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "azure_openai"; - azure_openai: components["schemas"]["PromptAzureOpenAIInvocationParametersContent"]; - }; - /** PromptAzureOpenAIInvocationParametersContent */ - PromptAzureOpenAIInvocationParametersContent: { - /** Temperature */ - temperature?: number; - /** Max Tokens */ - max_tokens?: number; - /** Max Completion Tokens */ - max_completion_tokens?: number; - /** Frequency Penalty */ - frequency_penalty?: number; - /** Presence Penalty */ - presence_penalty?: number; - /** Top P */ - top_p?: number; - /** Seed */ - seed?: number; - /** - * Reasoning Effort - * @enum {string} - */ - reasoning_effort?: "low" | "medium" | "high"; - }; - /** PromptChatTemplate */ - PromptChatTemplate: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "chat"; - /** Messages */ - messages: components["schemas"]["PromptMessage"][]; - }; - /** PromptData */ - PromptData: { - name: components["schemas"]["Identifier"]; - /** Description */ - description?: string | null; - /** Source Prompt Id */ - source_prompt_id?: string | null; - }; - /** PromptGoogleInvocationParameters */ - PromptGoogleInvocationParameters: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "google"; - google: components["schemas"]["PromptGoogleInvocationParametersContent"]; - }; - /** PromptGoogleInvocationParametersContent */ - PromptGoogleInvocationParametersContent: { - /** Temperature */ - temperature?: number; - /** Max Output Tokens */ - max_output_tokens?: number; - /** Stop Sequences */ - stop_sequences?: string[]; - /** Presence Penalty */ - presence_penalty?: number; - /** Frequency Penalty */ - frequency_penalty?: number; - /** Top P */ - top_p?: number; - /** Top K */ - top_k?: number; - }; - /** PromptMessage */ - PromptMessage: { - /** - * Role - * @enum {string} - */ - role: "user" | "assistant" | "model" | "ai" | "tool" | "system" | "developer"; - /** Content */ - content: string | (components["schemas"]["TextContentPart"] | components["schemas"]["ToolCallContentPart"] | components["schemas"]["ToolResultContentPart"])[]; - }; - /** PromptOpenAIInvocationParameters */ - PromptOpenAIInvocationParameters: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "openai"; - openai: components["schemas"]["PromptOpenAIInvocationParametersContent"]; - }; - /** PromptOpenAIInvocationParametersContent */ - PromptOpenAIInvocationParametersContent: { - /** Temperature */ - temperature?: number; - /** Max Tokens */ - max_tokens?: number; - /** Max Completion Tokens */ - max_completion_tokens?: number; - /** Frequency Penalty */ - frequency_penalty?: number; - /** Presence Penalty */ - presence_penalty?: number; - /** Top P */ - top_p?: number; - /** Seed */ - seed?: number; - /** - * Reasoning Effort - * @enum {string} - */ - reasoning_effort?: "low" | "medium" | "high"; - }; - /** PromptResponseFormatJSONSchema */ - PromptResponseFormatJSONSchema: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "json_schema"; - json_schema: components["schemas"]["PromptResponseFormatJSONSchemaDefinition"]; - }; - /** PromptResponseFormatJSONSchemaDefinition */ - PromptResponseFormatJSONSchemaDefinition: { - /** Name */ - name: string; - /** Description */ - description?: string; - /** Schema */ - schema?: { - [key: string]: unknown; - }; - /** Strict */ - strict?: boolean; - }; - /** PromptStringTemplate */ - PromptStringTemplate: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "string"; - /** Template */ - template: string; - }; - /** - * PromptTemplateFormat - * @enum {string} - */ - PromptTemplateFormat: "MUSTACHE" | "F_STRING" | "NONE"; - /** - * PromptTemplateType - * @enum {string} - */ - PromptTemplateType: "STR" | "CHAT"; - /** PromptToolChoiceNone */ - PromptToolChoiceNone: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "none"; - }; - /** PromptToolChoiceOneOrMore */ - PromptToolChoiceOneOrMore: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "one_or_more"; - }; - /** PromptToolChoiceSpecificFunctionTool */ - PromptToolChoiceSpecificFunctionTool: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "specific_function"; - /** Function Name */ - function_name: string; - }; - /** PromptToolChoiceZeroOrMore */ - PromptToolChoiceZeroOrMore: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "zero_or_more"; - }; - /** PromptToolFunction */ - PromptToolFunction: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "function"; - function: components["schemas"]["PromptToolFunctionDefinition"]; - }; - /** PromptToolFunctionDefinition */ - PromptToolFunctionDefinition: { - /** Name */ - name: string; - /** Description */ - description?: string; - /** Parameters */ - parameters?: { - [key: string]: unknown; - }; - /** Strict */ - strict?: boolean; - }; - /** PromptTools */ - PromptTools: { - /** - * Type - * @constant - */ - type: "tools"; - /** Tools */ - tools: components["schemas"]["PromptToolFunction"][]; - /** Tool Choice */ - tool_choice?: components["schemas"]["PromptToolChoiceNone"] | components["schemas"]["PromptToolChoiceZeroOrMore"] | components["schemas"]["PromptToolChoiceOneOrMore"] | components["schemas"]["PromptToolChoiceSpecificFunctionTool"]; - /** Disable Parallel Tool Calls */ - disable_parallel_tool_calls?: boolean; - }; - /** PromptVersion */ - PromptVersion: { - /** Description */ - description?: string | null; - model_provider: components["schemas"]["ModelProvider"]; - /** Model Name */ - model_name: string; - /** Template */ - template: components["schemas"]["PromptChatTemplate"] | components["schemas"]["PromptStringTemplate"]; - template_type: components["schemas"]["PromptTemplateType"]; - template_format: components["schemas"]["PromptTemplateFormat"]; - /** Invocation Parameters */ - invocation_parameters: components["schemas"]["PromptOpenAIInvocationParameters"] | components["schemas"]["PromptAzureOpenAIInvocationParameters"] | components["schemas"]["PromptAnthropicInvocationParameters"] | components["schemas"]["PromptGoogleInvocationParameters"]; - tools?: components["schemas"]["PromptTools"] | null; - /** Response Format */ - response_format?: components["schemas"]["PromptResponseFormatJSONSchema"] | null; - /** Id */ - id: string; - }; - /** PromptVersionData */ - PromptVersionData: { - /** Description */ - description?: string | null; - model_provider: components["schemas"]["ModelProvider"]; - /** Model Name */ - model_name: string; - /** Template */ - template: components["schemas"]["PromptChatTemplate"] | components["schemas"]["PromptStringTemplate"]; - template_type: components["schemas"]["PromptTemplateType"]; - template_format: components["schemas"]["PromptTemplateFormat"]; - /** Invocation Parameters */ - invocation_parameters: components["schemas"]["PromptOpenAIInvocationParameters"] | components["schemas"]["PromptAzureOpenAIInvocationParameters"] | components["schemas"]["PromptAnthropicInvocationParameters"] | components["schemas"]["PromptGoogleInvocationParameters"]; - tools?: components["schemas"]["PromptTools"] | null; - /** Response Format */ - response_format?: components["schemas"]["PromptResponseFormatJSONSchema"] | null; - }; - /** PromptVersionTag */ - PromptVersionTag: { - name: components["schemas"]["Identifier"]; - /** Description */ - description?: string | null; - /** Id */ - id: string; - }; - /** PromptVersionTagData */ - PromptVersionTagData: { - name: components["schemas"]["Identifier"]; - /** Description */ - description?: string | null; - }; - /** SpanAnnotation */ - SpanAnnotation: { - /** - * Span Id - * @description OpenTelemetry Span ID (hex format w/o 0x prefix) - */ - span_id: string; - /** - * Name - * @description The name of the annotation - */ - name: string; - /** - * Annotator Kind - * @description The kind of annotator used for the annotation - * @enum {string} - */ - annotator_kind: "LLM" | "HUMAN"; - /** @description The result of the annotation */ - result?: components["schemas"]["SpanAnnotationResult"] | null; - /** - * Metadata - * @description Metadata for the annotation - */ - metadata?: { - [key: string]: unknown; - } | null; - }; - /** SpanAnnotationResult */ - SpanAnnotationResult: { - /** - * Label - * @description The label assigned by the annotation - */ - label?: string | null; - /** - * Score - * @description The score assigned by the annotation - */ - score?: number | null; - /** - * Explanation - * @description Explanation of the annotation result - */ - explanation?: string | null; - }; - /** TextContentPart */ - TextContentPart: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "text"; - /** Text */ - text: string; - }; - /** ToolCallContentPart */ - ToolCallContentPart: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "tool_call"; - /** Tool Call Id */ - tool_call_id: string; - /** Tool Call */ - tool_call: components["schemas"]["ToolCallFunction"]; - }; - /** ToolCallFunction */ - ToolCallFunction: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "function"; - /** Name */ - name: string; - /** Arguments */ - arguments: string; - }; - /** ToolResultContentPart */ - ToolResultContentPart: { - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "tool_result"; - /** Tool Call Id */ - tool_call_id: string; - /** Tool Result */ - tool_result: boolean | number | string | { - [key: string]: unknown; - } | unknown[] | null; - }; - /** UpdateProjectRequestBody */ - UpdateProjectRequestBody: { - /** Description */ - description?: string | null; - }; - /** UpdateProjectResponseBody */ - UpdateProjectResponseBody: { - data: components["schemas"]["Project"]; - }; - /** UploadDatasetData */ - UploadDatasetData: { - /** Dataset Id */ - dataset_id: string; - }; - /** UploadDatasetResponseBody */ - UploadDatasetResponseBody: { - data: components["schemas"]["UploadDatasetData"]; - }; - /** ValidationError */ - ValidationError: { - /** Location */ - loc: (string | number)[]; - /** Message */ - msg: string; - /** Error Type */ - type: string; - }; - }; - responses: never; - parameters: never; - requestBodies: never; - headers: never; - pathItems: never; + schemas: { + /** AnnotateSpansRequestBody */ + AnnotateSpansRequestBody: { + /** Data */ + data: components["schemas"]["SpanAnnotation"][]; + }; + /** AnnotateSpansResponseBody */ + AnnotateSpansResponseBody: { + /** Data */ + data: components["schemas"]["InsertedSpanAnnotation"][]; + }; + /** AnnotationConfigResponse */ + AnnotationConfigResponse: { + /** Id */ + id: string; + /** Name */ + name: string; + annotation_type: components["schemas"]["AnnotationType"]; + optimization_direction?: + | components["schemas"]["OptimizationDirection"] + | null; + /** Description */ + description?: string | null; + /** Lower Bound */ + lower_bound?: number | null; + /** Upper Bound */ + upper_bound?: number | null; + /** Values */ + values?: components["schemas"]["CategoricalAnnotationValue"][] | null; + }; + /** + * AnnotationType + * @enum {string} + */ + AnnotationType: "CONTINUOUS" | "CATEGORICAL" | "FREEFORM"; + /** CategoricalAnnotationValue */ + CategoricalAnnotationValue: { + /** Label */ + label: string; + /** Score */ + score?: number | null; + }; + /** CreateCategoricalAnnotationConfigPayload */ + CreateCategoricalAnnotationConfigPayload: { + /** Name */ + name: string; + optimization_direction: components["schemas"]["OptimizationDirection"]; + /** Description */ + description?: string | null; + /** Values */ + values: components["schemas"]["CreateCategoricalAnnotationValuePayload"][]; + }; + /** CreateCategoricalAnnotationValuePayload */ + CreateCategoricalAnnotationValuePayload: { + /** Label */ + label: string; + /** Score */ + score?: number | null; + }; + /** CreateContinuousAnnotationConfigPayload */ + CreateContinuousAnnotationConfigPayload: { + /** Name */ + name: string; + optimization_direction: components["schemas"]["OptimizationDirection"]; + /** Description */ + description?: string | null; + /** Lower Bound */ + lower_bound?: number | null; + /** Upper Bound */ + upper_bound?: number | null; + }; + /** + * CreateExperimentRequestBody + * @description Details of the experiment to be created + */ + CreateExperimentRequestBody: { + /** + * Name + * @description Name of the experiment (if omitted, a random name will be generated) + */ + name?: string | null; + /** + * Description + * @description An optional description of the experiment + */ + description?: string | null; + /** + * Metadata + * @description Metadata for the experiment + */ + metadata?: { + [key: string]: unknown; + } | null; + /** + * Version Id + * @description ID of the dataset version over which the experiment will be run (if omitted, the latest version will be used) + */ + version_id?: string | null; + /** + * Repetitions + * @description Number of times the experiment should be repeated for each example + * @default 1 + */ + repetitions: number; + }; + /** CreateExperimentResponseBody */ + CreateExperimentResponseBody: { + data: components["schemas"]["Experiment"]; + }; + /** CreateFreeformAnnotationConfigPayload */ + CreateFreeformAnnotationConfigPayload: { + /** Name */ + name: string; + /** Description */ + description?: string | null; + }; + /** CreatePromptRequestBody */ + CreatePromptRequestBody: { + prompt: components["schemas"]["PromptData"]; + version: components["schemas"]["PromptVersionData"]; + }; + /** CreatePromptResponseBody */ + CreatePromptResponseBody: { + data: components["schemas"]["PromptVersion"]; + }; + /** Dataset */ + Dataset: { + /** Id */ + id: string; + /** Name */ + name: string; + /** Description */ + description: string | null; + /** Metadata */ + metadata: { + [key: string]: unknown; + }; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** DatasetExample */ + DatasetExample: { + /** Id */ + id: string; + /** Input */ + input: { + [key: string]: unknown; + }; + /** Output */ + output: { + [key: string]: unknown; + }; + /** Metadata */ + metadata: { + [key: string]: unknown; + }; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** DatasetVersion */ + DatasetVersion: { + /** Version Id */ + version_id: string; + /** Description */ + description: string | null; + /** Metadata */ + metadata: { + [key: string]: unknown; + }; + /** + * Created At + * Format: date-time + */ + created_at: string; + }; + /** DatasetWithExampleCount */ + DatasetWithExampleCount: { + /** Id */ + id: string; + /** Name */ + name: string; + /** Description */ + description: string | null; + /** Metadata */ + metadata: { + [key: string]: unknown; + }; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + /** Example Count */ + example_count: number; + }; + /** Experiment */ + Experiment: { + /** + * Id + * @description The ID of the experiment + */ + id: string; + /** + * Dataset Id + * @description The ID of the dataset associated with the experiment + */ + dataset_id: string; + /** + * Dataset Version Id + * @description The ID of the dataset version associated with the experiment + */ + dataset_version_id: string; + /** + * Repetitions + * @description Number of times the experiment is repeated + */ + repetitions: number; + /** + * Metadata + * @description Metadata of the experiment + */ + metadata: { + [key: string]: unknown; + }; + /** + * Project Name + * @description The name of the project associated with the experiment + */ + project_name: string | null; + /** + * Created At + * Format: date-time + * @description The creation timestamp of the experiment + */ + created_at: string; + /** + * Updated At + * Format: date-time + * @description The last update timestamp of the experiment + */ + updated_at: string; + }; + /** GetDatasetResponseBody */ + GetDatasetResponseBody: { + data: components["schemas"]["DatasetWithExampleCount"]; + }; + /** GetExperimentResponseBody */ + GetExperimentResponseBody: { + data: components["schemas"]["Experiment"]; + }; + /** GetProjectResponseBody */ + GetProjectResponseBody: { + data: components["schemas"]["Project"]; + }; + /** GetProjectsResponseBody */ + GetProjectsResponseBody: { + /** Data */ + data: components["schemas"]["Project"][]; + /** Next Cursor */ + next_cursor: string | null; + }; + /** GetPromptResponseBody */ + GetPromptResponseBody: { + data: components["schemas"]["PromptVersion"]; + }; + /** GetPromptVersionTagsResponseBody */ + GetPromptVersionTagsResponseBody: { + /** Data */ + data: components["schemas"]["PromptVersionTag"][]; + /** Next Cursor */ + next_cursor: string | null; + }; + /** GetPromptVersionsResponseBody */ + GetPromptVersionsResponseBody: { + /** Data */ + data: components["schemas"]["PromptVersion"][]; + /** Next Cursor */ + next_cursor: string | null; + }; + /** GetPromptsResponseBody */ + GetPromptsResponseBody: { + /** Data */ + data: components["schemas"]["Prompt"][]; + /** Next Cursor */ + next_cursor: string | null; + }; + /** HTTPValidationError */ + HTTPValidationError: { + /** Detail */ + detail?: components["schemas"]["ValidationError"][]; + }; + /** Identifier */ + Identifier: string; + /** InsertedSpanAnnotation */ + InsertedSpanAnnotation: { + /** + * Id + * @description The ID of the inserted span annotation + */ + id: string; + }; + /** ListDatasetExamplesData */ + ListDatasetExamplesData: { + /** Dataset Id */ + dataset_id: string; + /** Version Id */ + version_id: string; + /** Examples */ + examples: components["schemas"]["DatasetExample"][]; + }; + /** ListDatasetExamplesResponseBody */ + ListDatasetExamplesResponseBody: { + data: components["schemas"]["ListDatasetExamplesData"]; + }; + /** ListDatasetVersionsResponseBody */ + ListDatasetVersionsResponseBody: { + /** Data */ + data: components["schemas"]["DatasetVersion"][]; + /** Next Cursor */ + next_cursor: string | null; + }; + /** ListDatasetsResponseBody */ + ListDatasetsResponseBody: { + /** Data */ + data: components["schemas"]["Dataset"][]; + /** Next Cursor */ + next_cursor: string | null; + }; + /** ListExperimentsResponseBody */ + ListExperimentsResponseBody: { + /** Data */ + data: components["schemas"]["Experiment"][]; + }; + /** + * ModelProvider + * @enum {string} + */ + ModelProvider: "OPENAI" | "AZURE_OPENAI" | "ANTHROPIC" | "GOOGLE"; + /** + * OptimizationDirection + * @enum {string} + */ + OptimizationDirection: "MINIMIZE" | "MAXIMIZE"; + /** Prompt */ + Prompt: { + name: components["schemas"]["Identifier"]; + /** Description */ + description?: string | null; + /** Source Prompt Id */ + source_prompt_id?: string | null; + /** Id */ + id: string; + }; + /** PromptAnthropicInvocationParameters */ + PromptAnthropicInvocationParameters: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "anthropic"; + anthropic: components["schemas"]["PromptAnthropicInvocationParametersContent"]; + }; + /** PromptAnthropicInvocationParametersContent */ + PromptAnthropicInvocationParametersContent: { + /** Max Tokens */ + max_tokens: number; + /** Temperature */ + temperature?: number; + /** Top P */ + top_p?: number; + /** Stop Sequences */ + stop_sequences?: string[]; + /** Thinking */ + thinking?: + | components["schemas"]["PromptAnthropicThinkingConfigDisabled"] + | components["schemas"]["PromptAnthropicThinkingConfigEnabled"]; + }; + /** PromptAnthropicThinkingConfigDisabled */ + PromptAnthropicThinkingConfigDisabled: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "disabled"; + }; + /** PromptAnthropicThinkingConfigEnabled */ + PromptAnthropicThinkingConfigEnabled: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "enabled"; + /** Budget Tokens */ + budget_tokens: number; + }; + /** PromptAzureOpenAIInvocationParameters */ + PromptAzureOpenAIInvocationParameters: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "azure_openai"; + azure_openai: components["schemas"]["PromptAzureOpenAIInvocationParametersContent"]; + }; + /** PromptAzureOpenAIInvocationParametersContent */ + PromptAzureOpenAIInvocationParametersContent: { + /** Temperature */ + temperature?: number; + /** Max Tokens */ + max_tokens?: number; + /** Max Completion Tokens */ + max_completion_tokens?: number; + /** Frequency Penalty */ + frequency_penalty?: number; + /** Presence Penalty */ + presence_penalty?: number; + /** Top P */ + top_p?: number; + /** Seed */ + seed?: number; + /** + * Reasoning Effort + * @enum {string} + */ + reasoning_effort?: "low" | "medium" | "high"; + }; + /** PromptChatTemplate */ + PromptChatTemplate: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "chat"; + /** Messages */ + messages: components["schemas"]["PromptMessage"][]; + }; + /** PromptData */ + PromptData: { + name: components["schemas"]["Identifier"]; + /** Description */ + description?: string | null; + /** Source Prompt Id */ + source_prompt_id?: string | null; + }; + /** PromptGoogleInvocationParameters */ + PromptGoogleInvocationParameters: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "google"; + google: components["schemas"]["PromptGoogleInvocationParametersContent"]; + }; + /** PromptGoogleInvocationParametersContent */ + PromptGoogleInvocationParametersContent: { + /** Temperature */ + temperature?: number; + /** Max Output Tokens */ + max_output_tokens?: number; + /** Stop Sequences */ + stop_sequences?: string[]; + /** Presence Penalty */ + presence_penalty?: number; + /** Frequency Penalty */ + frequency_penalty?: number; + /** Top P */ + top_p?: number; + /** Top K */ + top_k?: number; + }; + /** PromptMessage */ + PromptMessage: { + /** + * Role + * @enum {string} + */ + role: + | "user" + | "assistant" + | "model" + | "ai" + | "tool" + | "system" + | "developer"; + /** Content */ + content: + | string + | ( + | components["schemas"]["TextContentPart"] + | components["schemas"]["ToolCallContentPart"] + | components["schemas"]["ToolResultContentPart"] + )[]; + }; + /** PromptOpenAIInvocationParameters */ + PromptOpenAIInvocationParameters: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "openai"; + openai: components["schemas"]["PromptOpenAIInvocationParametersContent"]; + }; + /** PromptOpenAIInvocationParametersContent */ + PromptOpenAIInvocationParametersContent: { + /** Temperature */ + temperature?: number; + /** Max Tokens */ + max_tokens?: number; + /** Max Completion Tokens */ + max_completion_tokens?: number; + /** Frequency Penalty */ + frequency_penalty?: number; + /** Presence Penalty */ + presence_penalty?: number; + /** Top P */ + top_p?: number; + /** Seed */ + seed?: number; + /** + * Reasoning Effort + * @enum {string} + */ + reasoning_effort?: "low" | "medium" | "high"; + }; + /** PromptResponseFormatJSONSchema */ + PromptResponseFormatJSONSchema: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "json_schema"; + json_schema: components["schemas"]["PromptResponseFormatJSONSchemaDefinition"]; + }; + /** PromptResponseFormatJSONSchemaDefinition */ + PromptResponseFormatJSONSchemaDefinition: { + /** Name */ + name: string; + /** Description */ + description?: string; + /** Schema */ + schema?: { + [key: string]: unknown; + }; + /** Strict */ + strict?: boolean; + }; + /** PromptStringTemplate */ + PromptStringTemplate: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "string"; + /** Template */ + template: string; + }; + /** + * PromptTemplateFormat + * @enum {string} + */ + PromptTemplateFormat: "MUSTACHE" | "F_STRING" | "NONE"; + /** + * PromptTemplateType + * @enum {string} + */ + PromptTemplateType: "STR" | "CHAT"; + /** PromptToolChoiceNone */ + PromptToolChoiceNone: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "none"; + }; + /** PromptToolChoiceOneOrMore */ + PromptToolChoiceOneOrMore: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "one_or_more"; + }; + /** PromptToolChoiceSpecificFunctionTool */ + PromptToolChoiceSpecificFunctionTool: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "specific_function"; + /** Function Name */ + function_name: string; + }; + /** PromptToolChoiceZeroOrMore */ + PromptToolChoiceZeroOrMore: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "zero_or_more"; + }; + /** PromptToolFunction */ + PromptToolFunction: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "function"; + function: components["schemas"]["PromptToolFunctionDefinition"]; + }; + /** PromptToolFunctionDefinition */ + PromptToolFunctionDefinition: { + /** Name */ + name: string; + /** Description */ + description?: string; + /** Parameters */ + parameters?: { + [key: string]: unknown; + }; + /** Strict */ + strict?: boolean; + }; + /** PromptTools */ + PromptTools: { + /** + * Type + * @constant + */ + type: "tools"; + /** Tools */ + tools: components["schemas"]["PromptToolFunction"][]; + /** Tool Choice */ + tool_choice?: + | components["schemas"]["PromptToolChoiceNone"] + | components["schemas"]["PromptToolChoiceZeroOrMore"] + | components["schemas"]["PromptToolChoiceOneOrMore"] + | components["schemas"]["PromptToolChoiceSpecificFunctionTool"]; + /** Disable Parallel Tool Calls */ + disable_parallel_tool_calls?: boolean; + }; + /** PromptVersion */ + PromptVersion: { + /** Description */ + description?: string | null; + model_provider: components["schemas"]["ModelProvider"]; + /** Model Name */ + model_name: string; + /** Template */ + template: + | components["schemas"]["PromptChatTemplate"] + | components["schemas"]["PromptStringTemplate"]; + template_type: components["schemas"]["PromptTemplateType"]; + template_format: components["schemas"]["PromptTemplateFormat"]; + /** Invocation Parameters */ + invocation_parameters: + | components["schemas"]["PromptOpenAIInvocationParameters"] + | components["schemas"]["PromptAzureOpenAIInvocationParameters"] + | components["schemas"]["PromptAnthropicInvocationParameters"] + | components["schemas"]["PromptGoogleInvocationParameters"]; + tools?: components["schemas"]["PromptTools"] | null; + /** Response Format */ + response_format?: + | components["schemas"]["PromptResponseFormatJSONSchema"] + | null; + /** Id */ + id: string; + }; + /** PromptVersionData */ + PromptVersionData: { + /** Description */ + description?: string | null; + model_provider: components["schemas"]["ModelProvider"]; + /** Model Name */ + model_name: string; + /** Template */ + template: + | components["schemas"]["PromptChatTemplate"] + | components["schemas"]["PromptStringTemplate"]; + template_type: components["schemas"]["PromptTemplateType"]; + template_format: components["schemas"]["PromptTemplateFormat"]; + /** Invocation Parameters */ + invocation_parameters: + | components["schemas"]["PromptOpenAIInvocationParameters"] + | components["schemas"]["PromptAzureOpenAIInvocationParameters"] + | components["schemas"]["PromptAnthropicInvocationParameters"] + | components["schemas"]["PromptGoogleInvocationParameters"]; + tools?: components["schemas"]["PromptTools"] | null; + /** Response Format */ + response_format?: + | components["schemas"]["PromptResponseFormatJSONSchema"] + | null; + }; + /** PromptVersionTag */ + PromptVersionTag: { + name: components["schemas"]["Identifier"]; + /** Description */ + description?: string | null; + /** Id */ + id: string; + }; + /** PromptVersionTagData */ + PromptVersionTagData: { + name: components["schemas"]["Identifier"]; + /** Description */ + description?: string | null; + }; + /** SpanAnnotation */ + SpanAnnotation: { + /** + * Span Id + * @description OpenTelemetry Span ID (hex format w/o 0x prefix) + */ + span_id: string; + /** + * Name + * @description The name of the annotation + */ + name: string; + /** + * Annotator Kind + * @description The kind of annotator used for the annotation + * @enum {string} + */ + annotator_kind: "LLM" | "HUMAN"; + /** @description The result of the annotation */ + result?: components["schemas"]["SpanAnnotationResult"] | null; + /** + * Metadata + * @description Metadata for the annotation + */ + metadata?: { + [key: string]: unknown; + } | null; + }; + /** SpanAnnotationResult */ + SpanAnnotationResult: { + /** + * Label + * @description The label assigned by the annotation + */ + label?: string | null; + /** + * Score + * @description The score assigned by the annotation + */ + score?: number | null; + /** + * Explanation + * @description Explanation of the annotation result + */ + explanation?: string | null; + }; + /** TextContentPart */ + TextContentPart: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "text"; + /** Text */ + text: string; + }; + /** ToolCallContentPart */ + ToolCallContentPart: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "tool_call"; + /** Tool Call Id */ + tool_call_id: string; + /** Tool Call */ + tool_call: components["schemas"]["ToolCallFunction"]; + }; + /** ToolCallFunction */ + ToolCallFunction: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "function"; + /** Name */ + name: string; + /** Arguments */ + arguments: string; + }; + /** ToolResultContentPart */ + ToolResultContentPart: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "tool_result"; + /** Tool Call Id */ + tool_call_id: string; + /** Tool Result */ + tool_result: + | boolean + | number + | string + | { + [key: string]: unknown; + } + | unknown[] + | null; + }; + /** UpdateProjectRequestBody */ + UpdateProjectRequestBody: { + /** Description */ + description?: string | null; + }; + /** UpdateProjectResponseBody */ + UpdateProjectResponseBody: { + data: components["schemas"]["Project"]; + }; + /** UploadDatasetData */ + UploadDatasetData: { + /** Dataset Id */ + dataset_id: string; + }; + /** UploadDatasetResponseBody */ + UploadDatasetResponseBody: { + data: components["schemas"]["UploadDatasetData"]; + }; + /** ValidationError */ + ValidationError: { + /** Location */ + loc: (string | number)[]; + /** Message */ + msg: string; + /** Error Type */ + type: string; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; } export type $defs = Record; export interface operations { - listDatasets: { - parameters: { - query?: { - /** @description Cursor for pagination */ - cursor?: string | null; - /** @description An optional dataset name to filter by */ - name?: string | null; - /** @description The max number of datasets to return at a time. */ - limit?: number; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["ListDatasetsResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getDataset: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The ID of the dataset */ - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetDatasetResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - deleteDatasetById: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The ID of the dataset to delete. */ - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 204: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Dataset not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Invalid dataset ID */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - listDatasetVersionsByDatasetId: { - parameters: { - query?: { - /** @description Cursor for pagination */ - cursor?: string | null; - /** @description The max number of dataset versions to return at a time */ - limit?: number; - }; - header?: never; - path: { - /** @description The ID of the dataset */ - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["ListDatasetVersionsResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - uploadDataset: { - parameters: { - query?: { - /** @description If true, fulfill request synchronously and return JSON containing dataset_id. */ - sync?: boolean; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: { - content: { - "application/json": { - /** @enum {string} */ - action?: "create" | "append"; - name: string; - description?: string; - inputs: Record[]; - outputs?: Record[]; - metadata?: Record[]; - }; - "multipart/form-data": { - /** @enum {string} */ - action?: "create" | "append"; - name: string; - description?: string; - "input_keys[]": string[]; - "output_keys[]": string[]; - "metadata_keys[]"?: string[]; - /** Format: binary */ - file: string; - }; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UploadDatasetResponseBody"] | null; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Dataset of the same name already exists */ - 409: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Invalid request body */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getDatasetExamples: { - parameters: { - query?: { - /** @description The ID of the dataset version (if omitted, returns data from the latest version) */ - version_id?: string | null; - }; - header?: never; - path: { - /** @description The ID of the dataset */ - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["ListDatasetExamplesResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - getDatasetCsv: { - parameters: { - query?: { - /** @description The ID of the dataset version (if omitted, returns data from the latest version) */ - version_id?: string | null; - }; - header?: never; - path: { - /** @description The ID of the dataset */ - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "text/csv": string; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getDatasetJSONLOpenAIFineTuning: { - parameters: { - query?: { - /** @description The ID of the dataset version (if omitted, returns data from the latest version) */ - version_id?: string | null; - }; - header?: never; - path: { - /** @description The ID of the dataset */ - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Invalid dataset or version ID */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getDatasetJSONLOpenAIEvals: { - parameters: { - query?: { - /** @description The ID of the dataset version (if omitted, returns data from the latest version) */ - version_id?: string | null; - }; - header?: never; - path: { - /** @description The ID of the dataset */ - id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Invalid dataset or version ID */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - listExperiments: { - parameters: { - query?: never; - header?: never; - path: { - dataset_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Experiments retrieved successfully */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["ListExperimentsResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - createExperiment: { - parameters: { - query?: never; - header?: never; - path: { - dataset_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["CreateExperimentRequestBody"]; - }; - }; - responses: { - /** @description Experiment retrieved successfully */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["CreateExperimentResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Dataset or DatasetVersion not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - getExperiment: { - parameters: { - query?: never; - header?: never; - path: { - experiment_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Experiment retrieved successfully */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetExperimentResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Experiment not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - getExperimentJSON: { - parameters: { - query?: never; - header?: never; - path: { - experiment_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Experiment not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - getExperimentCSV: { - parameters: { - query?: never; - header?: never; - path: { - experiment_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - "text/csv": string; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - annotateSpans: { - parameters: { - query?: { - /** @description If true, fulfill request synchronously. */ - sync?: boolean; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["AnnotateSpansRequestBody"]; - }; - }; - responses: { - /** @description Span annotations inserted successfully */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["AnnotateSpansResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Span not found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - getEvaluations: { - parameters: { - query?: { - /** @description The name of the project to get evaluations from (if omitted, evaluations will be drawn from the `default` project) */ - project_name?: string | null; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - addEvaluations: { - parameters: { - query?: never; - header?: { - "content-type"?: string | null; - "content-encoding"?: string | null; - }; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/x-protobuf": string; - "application/x-pandas-arrow": string; - }; - }; - responses: { - /** @description Successful Response */ - 204: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unsupported content type, only gzipped protobuf and pandas-arrow are supported */ - 415: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getPrompts: { - parameters: { - query?: { - /** @description Cursor for pagination (base64-encoded prompt ID) */ - cursor?: string | null; - /** @description The max number of prompts to return at a time. */ - limit?: number; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description A list of prompts with pagination information */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetPromptsResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - postPromptVersion: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["CreatePromptRequestBody"]; - }; - }; - responses: { - /** @description The newly created prompt version */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["CreatePromptResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - listPromptVersions: { - parameters: { - query?: { - /** @description Cursor for pagination (base64-encoded promptVersion ID) */ - cursor?: string | null; - /** @description The max number of prompt versions to return at a time. */ - limit?: number; - }; - header?: never; - path: { - /** @description The identifier of the prompt, i.e. name or ID. */ - prompt_identifier: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description A list of prompt versions with pagination information */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetPromptVersionsResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getPromptVersionByPromptVersionId: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The ID of the prompt version. */ - prompt_version_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description The requested prompt version */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetPromptResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getPromptVersionByTagName: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The identifier of the prompt, i.e. name or ID. */ - prompt_identifier: string; - /** @description The tag of the prompt version */ - tag_name: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description The prompt version with the specified tag */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetPromptResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getPromptVersionLatest: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The identifier of the prompt, i.e. name or ID. */ - prompt_identifier: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description The latest version of the specified prompt */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetPromptResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getPromptVersionTags: { - parameters: { - query?: { - /** @description Cursor for pagination (base64-encoded promptVersionTag ID) */ - cursor?: string | null; - /** @description The max number of tags to return at a time. */ - limit?: number; - }; - header?: never; - path: { - /** @description The ID of the prompt version. */ - prompt_version_id: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description A list of tags associated with the prompt version */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetPromptVersionTagsResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - createPromptVersionTag: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The ID of the prompt version. */ - prompt_version_id: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["PromptVersionTagData"]; - }; - }; - responses: { - /** @description No content returned on successful tag creation */ - 204: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getProjects: { - parameters: { - query?: { - /** @description Cursor for pagination (project ID) */ - cursor?: string | null; - /** @description The max number of projects to return at a time. */ - limit?: number; - /** @description Include experiment projects in the response. Experiment projects are created from running experiments. */ - include_experiment_projects?: boolean; - }; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description A list of projects with pagination information */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetProjectsResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - createProject: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["CreateProjectRequestBody"]; - }; - }; - responses: { - /** @description The newly created project */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["CreateProjectResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - getProject: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The project identifier: either project ID or project name. If using a project name, it cannot contain slash (/), question mark (?), or pound sign (#) characters. */ - project_identifier: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description The requested project */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["GetProjectResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - updateProject: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The project identifier: either project ID or project name. If using a project name, it cannot contain slash (/), question mark (?), or pound sign (#) characters. */ - project_identifier: string; - }; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["UpdateProjectRequestBody"]; - }; - }; - responses: { - /** @description The updated project */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["UpdateProjectResponseBody"]; - }; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; - }; - deleteProject: { - parameters: { - query?: never; - header?: never; - path: { - /** @description The project identifier: either project ID or project name. If using a project name, it cannot contain slash (/), question mark (?), or pound sign (#) characters. */ - project_identifier: string; - }; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description No content returned on successful deletion */ - 204: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Forbidden */ - 403: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Not Found */ - 404: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - /** @description Unprocessable Entity */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "text/plain": string; - }; - }; - }; + list_annotation_configs_v1_annotation_configs_get: { + parameters: { + query?: { + /** @description Maximum number of configs to return */ + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AnnotationConfigResponse"][]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_annotation_config_by_name_or_id_v1_annotation_configs__config_identifier__get: { + parameters: { + query?: never; + header?: never; + path: { + /** @description ID or name of the annotation configuration */ + config_identifier: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AnnotationConfigResponse"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + create_continuous_annotation_config_v1_annotation_configs_continuous_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateContinuousAnnotationConfigPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AnnotationConfigResponse"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + create_categorical_annotation_config_v1_annotation_configs_categorical_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateCategoricalAnnotationConfigPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AnnotationConfigResponse"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + create_freeform_annotation_config_v1_annotation_configs_freeform_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateFreeformAnnotationConfigPayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AnnotationConfigResponse"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + delete_annotation_config_v1_annotation_configs__config_id__delete: { + parameters: { + query?: never; + header?: never; + path: { + /** @description ID of the annotation configuration */ + config_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": boolean; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + listDatasets: { + parameters: { + query?: { + /** @description Cursor for pagination */ + cursor?: string | null; + /** @description An optional dataset name to filter by */ + name?: string | null; + /** @description The max number of datasets to return at a time. */ + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ListDatasetsResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getDataset: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The ID of the dataset */ + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetDatasetResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + deleteDatasetById: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The ID of the dataset to delete. */ + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Dataset not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Invalid dataset ID */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + listDatasetVersionsByDatasetId: { + parameters: { + query?: { + /** @description Cursor for pagination */ + cursor?: string | null; + /** @description The max number of dataset versions to return at a time */ + limit?: number; + }; + header?: never; + path: { + /** @description The ID of the dataset */ + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ListDatasetVersionsResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + uploadDataset: { + parameters: { + query?: { + /** @description If true, fulfill request synchronously and return JSON containing dataset_id. */ + sync?: boolean; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + "application/json": { + /** @enum {string} */ + action?: "create" | "append"; + name: string; + description?: string; + inputs: Record[]; + outputs?: Record[]; + metadata?: Record[]; + }; + "multipart/form-data": { + /** @enum {string} */ + action?: "create" | "append"; + name: string; + description?: string; + "input_keys[]": string[]; + "output_keys[]": string[]; + "metadata_keys[]"?: string[]; + /** Format: binary */ + file: string; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": + | components["schemas"]["UploadDatasetResponseBody"] + | null; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Dataset of the same name already exists */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Invalid request body */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getDatasetExamples: { + parameters: { + query?: { + /** @description The ID of the dataset version (if omitted, returns data from the latest version) */ + version_id?: string | null; + }; + header?: never; + path: { + /** @description The ID of the dataset */ + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ListDatasetExamplesResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + getDatasetCsv: { + parameters: { + query?: { + /** @description The ID of the dataset version (if omitted, returns data from the latest version) */ + version_id?: string | null; + }; + header?: never; + path: { + /** @description The ID of the dataset */ + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "text/csv": string; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getDatasetJSONLOpenAIFineTuning: { + parameters: { + query?: { + /** @description The ID of the dataset version (if omitted, returns data from the latest version) */ + version_id?: string | null; + }; + header?: never; + path: { + /** @description The ID of the dataset */ + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Invalid dataset or version ID */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getDatasetJSONLOpenAIEvals: { + parameters: { + query?: { + /** @description The ID of the dataset version (if omitted, returns data from the latest version) */ + version_id?: string | null; + }; + header?: never; + path: { + /** @description The ID of the dataset */ + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Invalid dataset or version ID */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + listExperiments: { + parameters: { + query?: never; + header?: never; + path: { + dataset_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Experiments retrieved successfully */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ListExperimentsResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + createExperiment: { + parameters: { + query?: never; + header?: never; + path: { + dataset_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateExperimentRequestBody"]; + }; + }; + responses: { + /** @description Experiment retrieved successfully */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CreateExperimentResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Dataset or DatasetVersion not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + getExperiment: { + parameters: { + query?: never; + header?: never; + path: { + experiment_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Experiment retrieved successfully */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetExperimentResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Experiment not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + getExperimentJSON: { + parameters: { + query?: never; + header?: never; + path: { + experiment_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Experiment not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + getExperimentCSV: { + parameters: { + query?: never; + header?: never; + path: { + experiment_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + "text/csv": string; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + annotateSpans: { + parameters: { + query?: { + /** @description If true, fulfill request synchronously. */ + sync?: boolean; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AnnotateSpansRequestBody"]; + }; + }; + responses: { + /** @description Span annotations inserted successfully */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AnnotateSpansResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Span not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + getEvaluations: { + parameters: { + query?: { + /** @description The name of the project to get evaluations from (if omitted, evaluations will be drawn from the `default` project) */ + project_name?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + addEvaluations: { + parameters: { + query?: never; + header?: { + "content-type"?: string | null; + "content-encoding"?: string | null; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/x-protobuf": string; + "application/x-pandas-arrow": string; + }; + }; + responses: { + /** @description Successful Response */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unsupported content type, only gzipped protobuf and pandas-arrow are supported */ + 415: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getPrompts: { + parameters: { + query?: { + /** @description Cursor for pagination (base64-encoded prompt ID) */ + cursor?: string | null; + /** @description The max number of prompts to return at a time. */ + limit?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description A list of prompts with pagination information */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetPromptsResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + postPromptVersion: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreatePromptRequestBody"]; + }; + }; + responses: { + /** @description The newly created prompt version */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CreatePromptResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + listPromptVersions: { + parameters: { + query?: { + /** @description Cursor for pagination (base64-encoded promptVersion ID) */ + cursor?: string | null; + /** @description The max number of prompt versions to return at a time. */ + limit?: number; + }; + header?: never; + path: { + /** @description The identifier of the prompt, i.e. name or ID. */ + prompt_identifier: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description A list of prompt versions with pagination information */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetPromptVersionsResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getPromptVersionByPromptVersionId: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The ID of the prompt version. */ + prompt_version_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The requested prompt version */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetPromptResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getPromptVersionByTagName: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The identifier of the prompt, i.e. name or ID. */ + prompt_identifier: string; + /** @description The tag of the prompt version */ + tag_name: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The prompt version with the specified tag */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetPromptResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getPromptVersionLatest: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The identifier of the prompt, i.e. name or ID. */ + prompt_identifier: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The latest version of the specified prompt */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetPromptResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getPromptVersionTags: { + parameters: { + query?: { + /** @description Cursor for pagination (base64-encoded promptVersionTag ID) */ + cursor?: string | null; + /** @description The max number of tags to return at a time. */ + limit?: number; + }; + header?: never; + path: { + /** @description The ID of the prompt version. */ + prompt_version_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description A list of tags associated with the prompt version */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetPromptVersionTagsResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + createPromptVersionTag: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The ID of the prompt version. */ + prompt_version_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PromptVersionTagData"]; + }; + }; + responses: { + /** @description No content returned on successful tag creation */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getProjects: { + parameters: { + query?: { + /** @description Cursor for pagination (project ID) */ + cursor?: string | null; + /** @description The max number of projects to return at a time. */ + limit?: number; + /** @description Include experiment projects in the response. Experiment projects are created from running experiments. */ + include_experiment_projects?: boolean; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description A list of projects with pagination information */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetProjectsResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + createProject: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateProjectRequestBody"]; + }; + }; + responses: { + /** @description The newly created project */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CreateProjectResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + getProject: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The project identifier: either project ID or project name. If using a project name, it cannot contain slash (/), question mark (?), or pound sign (#) characters. */ + project_identifier: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The requested project */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GetProjectResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + updateProject: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The project identifier: either project ID or project name. If using a project name, it cannot contain slash (/), question mark (?), or pound sign (#) characters. */ + project_identifier: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["UpdateProjectRequestBody"]; + }; + }; + responses: { + /** @description The updated project */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UpdateProjectResponseBody"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + deleteProject: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The project identifier: either project ID or project name. If using a project name, it cannot contain slash (/), question mark (?), or pound sign (#) characters. */ + project_identifier: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description No content returned on successful deletion */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; }; + }; } diff --git a/schemas/openapi.json b/schemas/openapi.json index dae8f472be..e49c645c87 100644 --- a/schemas/openapi.json +++ b/schemas/openapi.json @@ -6,11 +6,319 @@ "version": "1.0" }, "paths": { - "/v1/datasets": { + "/v1/annotation_configs": { "get": { - "tags": [ - "datasets" + "tags": ["annotation_configs"], + "summary": "List annotation configurations", + "operationId": "list_annotation_configs_v1_annotation_configs_get", + "parameters": [ + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "exclusiveMinimum": 0, + "description": "Maximum number of configs to return", + "default": 50, + "title": "Limit" + }, + "description": "Maximum number of configs to return" + } ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AnnotationConfigResponse" + }, + "title": "Response List Annotation Configs V1 Annotation Configs Get" + } + } + } + }, + "403": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Forbidden" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/annotation_configs/{config_identifier}": { + "get": { + "tags": ["annotation_configs"], + "summary": "Get an annotation configuration by ID or name", + "operationId": "get_annotation_config_by_name_or_id_v1_annotation_configs__config_identifier__get", + "parameters": [ + { + "name": "config_identifier", + "in": "path", + "required": true, + "schema": { + "type": "string", + "description": "ID or name of the annotation configuration", + "title": "Config Identifier" + }, + "description": "ID or name of the annotation configuration" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AnnotationConfigResponse" + } + } + } + }, + "403": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Forbidden" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/annotation_configs/continuous": { + "post": { + "tags": ["annotation_configs"], + "summary": "Create a continuous annotation configuration", + "operationId": "create_continuous_annotation_config_v1_annotation_configs_continuous_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateContinuousAnnotationConfigPayload" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AnnotationConfigResponse" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/annotation_configs/categorical": { + "post": { + "tags": ["annotation_configs"], + "summary": "Create a categorical annotation configuration", + "operationId": "create_categorical_annotation_config_v1_annotation_configs_categorical_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateCategoricalAnnotationConfigPayload" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AnnotationConfigResponse" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/annotation_configs/freeform": { + "post": { + "tags": ["annotation_configs"], + "summary": "Create a freeform annotation configuration", + "operationId": "create_freeform_annotation_config_v1_annotation_configs_freeform_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateFreeformAnnotationConfigPayload" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AnnotationConfigResponse" + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/annotation_configs/{config_id}": { + "delete": { + "tags": ["annotation_configs"], + "summary": "Delete an annotation configuration", + "operationId": "delete_annotation_config_v1_annotation_configs__config_id__delete", + "parameters": [ + { + "name": "config_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "description": "ID of the annotation configuration", + "title": "Config Id" + }, + "description": "ID of the annotation configuration" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "boolean", + "title": "Response Delete Annotation Config V1 Annotation Configs Config Id Delete" + } + } + } + }, + "403": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Forbidden" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/datasets": { + "get": { + "tags": ["datasets"], "summary": "List datasets", "operationId": "listDatasets", "parameters": [ @@ -100,9 +408,7 @@ }, "/v1/datasets/{id}": { "delete": { - "tags": [ - "datasets" - ], + "tags": ["datasets"], "summary": "Delete dataset by ID", "operationId": "deleteDatasetById", "parameters": [ @@ -155,9 +461,7 @@ } }, "get": { - "tags": [ - "datasets" - ], + "tags": ["datasets"], "summary": "Get dataset by ID", "operationId": "getDataset", "parameters": [ @@ -219,9 +523,7 @@ }, "/v1/datasets/{id}/versions": { "get": { - "tags": [ - "datasets" - ], + "tags": ["datasets"], "summary": "List dataset versions", "operationId": "listDatasetVersionsByDatasetId", "parameters": [ @@ -304,9 +606,7 @@ }, "/v1/datasets/upload": { "post": { - "tags": [ - "datasets" - ], + "tags": ["datasets"], "summary": "Upload dataset from JSON, CSV, or PyArrow", "operationId": "uploadDataset", "parameters": [ @@ -378,17 +678,11 @@ "application/json": { "schema": { "type": "object", - "required": [ - "name", - "inputs" - ], + "required": ["name", "inputs"], "properties": { "action": { "type": "string", - "enum": [ - "create", - "append" - ] + "enum": ["create", "append"] }, "name": { "type": "string" @@ -420,19 +714,11 @@ "multipart/form-data": { "schema": { "type": "object", - "required": [ - "name", - "input_keys[]", - "output_keys[]", - "file" - ], + "required": ["name", "input_keys[]", "output_keys[]", "file"], "properties": { "action": { "type": "string", - "enum": [ - "create", - "append" - ] + "enum": ["create", "append"] }, "name": { "type": "string" @@ -474,9 +760,7 @@ }, "/v1/datasets/{id}/examples": { "get": { - "tags": [ - "datasets" - ], + "tags": ["datasets"], "summary": "Get examples from a dataset", "operationId": "getDatasetExamples", "parameters": [ @@ -556,9 +840,7 @@ }, "/v1/datasets/{id}/csv": { "get": { - "tags": [ - "datasets" - ], + "tags": ["datasets"], "summary": "Download dataset examples as CSV file", "operationId": "getDatasetCsv", "parameters": [ @@ -629,9 +911,7 @@ }, "/v1/datasets/{id}/jsonl/openai_ft": { "get": { - "tags": [ - "datasets" - ], + "tags": ["datasets"], "summary": "Download dataset examples as OpenAI fine-tuning JSONL file", "operationId": "getDatasetJSONLOpenAIFineTuning", "parameters": [ @@ -701,9 +981,7 @@ }, "/v1/datasets/{id}/jsonl/openai_evals": { "get": { - "tags": [ - "datasets" - ], + "tags": ["datasets"], "summary": "Download dataset examples as OpenAI evals JSONL file", "operationId": "getDatasetJSONLOpenAIEvals", "parameters": [ @@ -773,9 +1051,7 @@ }, "/v1/datasets/{dataset_id}/experiments": { "post": { - "tags": [ - "experiments" - ], + "tags": ["experiments"], "summary": "Create experiment on a dataset", "operationId": "createExperiment", "parameters": [ @@ -843,9 +1119,7 @@ } }, "get": { - "tags": [ - "experiments" - ], + "tags": ["experiments"], "summary": "List experiments by dataset", "operationId": "listExperiments", "parameters": [ @@ -895,9 +1169,7 @@ }, "/v1/experiments/{experiment_id}": { "get": { - "tags": [ - "experiments" - ], + "tags": ["experiments"], "summary": "Get experiment by ID", "operationId": "getExperiment", "parameters": [ @@ -957,9 +1229,7 @@ }, "/v1/experiments/{experiment_id}/json": { "get": { - "tags": [ - "experiments" - ], + "tags": ["experiments"], "summary": "Download experiment runs as a JSON file", "operationId": "getExperimentJSON", "parameters": [ @@ -1019,9 +1289,7 @@ }, "/v1/experiments/{experiment_id}/csv": { "get": { - "tags": [ - "experiments" - ], + "tags": ["experiments"], "summary": "Download experiment runs as a CSV file", "operationId": "getExperimentCSV", "parameters": [ @@ -1075,9 +1343,7 @@ }, "/v1/span_annotations": { "post": { - "tags": [ - "spans" - ], + "tags": ["spans"], "summary": "Create span annotations", "operationId": "annotateSpans", "parameters": [ @@ -1150,9 +1416,7 @@ }, "/v1/evaluations": { "post": { - "tags": [ - "traces" - ], + "tags": ["traces"], "summary": "Add span, trace, or document evaluations", "operationId": "addEvaluations", "parameters": [ @@ -1243,9 +1507,7 @@ } }, "get": { - "tags": [ - "traces" - ], + "tags": ["traces"], "summary": "Get span, trace, or document evaluations from a project", "operationId": "getEvaluations", "parameters": [ @@ -1312,9 +1574,7 @@ }, "/v1/prompts": { "get": { - "tags": [ - "prompts" - ], + "tags": ["prompts"], "summary": "List all prompts", "description": "Retrieve a paginated list of all prompts in the system. A prompt can have multiple versions.", "operationId": "getPrompts", @@ -1385,9 +1645,7 @@ } }, "post": { - "tags": [ - "prompts" - ], + "tags": ["prompts"], "summary": "Create a new prompt", "description": "Create a new prompt and its initial version. A prompt can have multiple versions.", "operationId": "postPromptVersion", @@ -1437,9 +1695,7 @@ }, "/v1/prompts/{prompt_identifier}/versions": { "get": { - "tags": [ - "prompts" - ], + "tags": ["prompts"], "summary": "List prompt versions", "description": "Retrieve all versions of a specific prompt with pagination support. Each prompt can have multiple versions with different configurations.", "operationId": "listPromptVersions", @@ -1533,9 +1789,7 @@ }, "/v1/prompt_versions/{prompt_version_id}": { "get": { - "tags": [ - "prompts" - ], + "tags": ["prompts"], "summary": "Get prompt version by ID", "description": "Retrieve a specific prompt version using its unique identifier. A prompt version contains the actual template and configuration.", "operationId": "getPromptVersionByPromptVersionId", @@ -1598,9 +1852,7 @@ }, "/v1/prompts/{prompt_identifier}/tags/{tag_name}": { "get": { - "tags": [ - "prompts" - ], + "tags": ["prompts"], "summary": "Get prompt version by tag", "description": "Retrieve a specific prompt version using its tag name. Tags are used to identify specific versions of a prompt.", "operationId": "getPromptVersionByTagName", @@ -1674,9 +1926,7 @@ }, "/v1/prompts/{prompt_identifier}/latest": { "get": { - "tags": [ - "prompts" - ], + "tags": ["prompts"], "summary": "Get latest prompt version", "description": "Retrieve the most recent version of a specific prompt.", "operationId": "getPromptVersionLatest", @@ -1739,9 +1989,7 @@ }, "/v1/prompt_versions/{prompt_version_id}/tags": { "get": { - "tags": [ - "prompts" - ], + "tags": ["prompts"], "summary": "List prompt version tags", "description": "Retrieve all tags associated with a specific prompt version. Tags are used to identify and categorize different versions of a prompt.", "operationId": "getPromptVersionTags", @@ -1833,9 +2081,7 @@ } }, "post": { - "tags": [ - "prompts" - ], + "tags": ["prompts"], "summary": "Add tag to prompt version", "description": "Add a new tag to a specific prompt version. Tags help identify and categorize different versions of a prompt.", "operationId": "createPromptVersionTag", @@ -1901,9 +2147,7 @@ }, "/v1/projects": { "get": { - "tags": [ - "projects" - ], + "tags": ["projects"], "summary": "List all projects", "description": "Retrieve a paginated list of all projects in the system.", "operationId": "getProjects", @@ -1986,9 +2230,7 @@ } }, "post": { - "tags": [ - "projects" - ], + "tags": ["projects"], "summary": "Create a new project", "description": "Create a new project with the specified configuration.", "operationId": "createProject", @@ -2038,9 +2280,7 @@ }, "/v1/projects/{project_identifier}": { "get": { - "tags": [ - "projects" - ], + "tags": ["projects"], "summary": "Get project by ID or name", "description": "Retrieve a specific project using its unique identifier: either project ID or project name. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters.", "operationId": "getProject", @@ -2101,9 +2341,7 @@ } }, "put": { - "tags": [ - "projects" - ], + "tags": ["projects"], "summary": "Update a project by ID or name", "description": "Update an existing project with new configuration. Project names cannot be changed. The project identifier is either project ID or project name. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters.", "operationId": "updateProject", @@ -2174,9 +2412,7 @@ } }, "delete": { - "tags": [ - "projects" - ], + "tags": ["projects"], "summary": "Delete a project by ID or name", "description": "Delete an existing project and all its associated data. The project identifier is either project ID or project name. The default project cannot be deleted. Note: When using a project name as the identifier, it cannot contain slash (/), question mark (?), or pound sign (#) characters.", "operationId": "deleteProject", @@ -2227,43 +2463,242 @@ }, "description": "Unprocessable Entity" } - } - } - } - }, - "components": { - "schemas": { - "AnnotateSpansRequestBody": { + } + } + } + }, + "components": { + "schemas": { + "AnnotateSpansRequestBody": { + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/SpanAnnotation" + }, + "type": "array", + "title": "Data" + } + }, + "type": "object", + "required": ["data"], + "title": "AnnotateSpansRequestBody" + }, + "AnnotateSpansResponseBody": { + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/InsertedSpanAnnotation" + }, + "type": "array", + "title": "Data" + } + }, + "type": "object", + "required": ["data"], + "title": "AnnotateSpansResponseBody" + }, + "AnnotationConfigResponse": { + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "name": { + "type": "string", + "title": "Name" + }, + "annotation_type": { + "$ref": "#/components/schemas/AnnotationType" + }, + "optimization_direction": { + "anyOf": [ + { + "$ref": "#/components/schemas/OptimizationDirection" + }, + { + "type": "null" + } + ] + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "lower_bound": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Lower Bound" + }, + "upper_bound": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Upper Bound" + }, + "values": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/CategoricalAnnotationValue" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Values" + } + }, + "type": "object", + "required": ["id", "name", "annotation_type"], + "title": "AnnotationConfigResponse" + }, + "AnnotationType": { + "type": "string", + "enum": ["CONTINUOUS", "CATEGORICAL", "FREEFORM"], + "title": "AnnotationType" + }, + "CategoricalAnnotationValue": { + "properties": { + "label": { + "type": "string", + "title": "Label" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" + } + }, + "type": "object", + "required": ["label"], + "title": "CategoricalAnnotationValue" + }, + "CreateCategoricalAnnotationConfigPayload": { "properties": { - "data": { + "name": { + "type": "string", + "title": "Name" + }, + "optimization_direction": { + "$ref": "#/components/schemas/OptimizationDirection" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "values": { "items": { - "$ref": "#/components/schemas/SpanAnnotation" + "$ref": "#/components/schemas/CreateCategoricalAnnotationValuePayload" }, "type": "array", - "title": "Data" + "title": "Values" } }, "type": "object", - "required": [ - "data" - ], - "title": "AnnotateSpansRequestBody" + "required": ["name", "optimization_direction", "values"], + "title": "CreateCategoricalAnnotationConfigPayload" }, - "AnnotateSpansResponseBody": { + "CreateCategoricalAnnotationValuePayload": { "properties": { - "data": { - "items": { - "$ref": "#/components/schemas/InsertedSpanAnnotation" - }, - "type": "array", - "title": "Data" + "label": { + "type": "string", + "title": "Label" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" } }, "type": "object", - "required": [ - "data" - ], - "title": "AnnotateSpansResponseBody" + "required": ["label"], + "title": "CreateCategoricalAnnotationValuePayload" + }, + "CreateContinuousAnnotationConfigPayload": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "optimization_direction": { + "$ref": "#/components/schemas/OptimizationDirection" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "lower_bound": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Lower Bound" + }, + "upper_bound": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Upper Bound" + } + }, + "type": "object", + "required": ["name", "optimization_direction"], + "title": "CreateContinuousAnnotationConfigPayload" }, "CreateExperimentRequestBody": { "properties": { @@ -2334,16 +2769,13 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "CreateExperimentResponseBody" }, - "CreateProjectRequestBody": { + "CreateFreeformAnnotationConfigPayload": { "properties": { "name": { "type": "string", - "minLength": 1, "title": "Name" }, "description": { @@ -2359,22 +2791,8 @@ } }, "type": "object", - "required": [ - "name" - ], - "title": "CreateProjectRequestBody" - }, - "CreateProjectResponseBody": { - "properties": { - "data": { - "$ref": "#/components/schemas/Project" - } - }, - "type": "object", - "required": [ - "data" - ], - "title": "CreateProjectResponseBody" + "required": ["name"], + "title": "CreateFreeformAnnotationConfigPayload" }, "CreatePromptRequestBody": { "properties": { @@ -2386,10 +2804,7 @@ } }, "type": "object", - "required": [ - "prompt", - "version" - ], + "required": ["prompt", "version"], "title": "CreatePromptRequestBody" }, "CreatePromptResponseBody": { @@ -2399,9 +2814,7 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "CreatePromptResponseBody" }, "Dataset": { @@ -2480,13 +2893,7 @@ } }, "type": "object", - "required": [ - "id", - "input", - "output", - "metadata", - "updated_at" - ], + "required": ["id", "input", "output", "metadata", "updated_at"], "title": "DatasetExample" }, "DatasetVersion": { @@ -2518,12 +2925,7 @@ } }, "type": "object", - "required": [ - "version_id", - "description", - "metadata", - "created_at" - ], + "required": ["version_id", "description", "metadata", "created_at"], "title": "DatasetVersion" }, "DatasetWithExampleCount": { @@ -2652,9 +3054,7 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "GetDatasetResponseBody" }, "GetExperimentResponseBody": { @@ -2664,9 +3064,7 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "GetExperimentResponseBody" }, "GetProjectResponseBody": { @@ -2676,9 +3074,7 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "GetProjectResponseBody" }, "GetProjectsResponseBody": { @@ -2703,10 +3099,7 @@ } }, "type": "object", - "required": [ - "data", - "next_cursor" - ], + "required": ["data", "next_cursor"], "title": "GetProjectsResponseBody" }, "GetPromptResponseBody": { @@ -2716,9 +3109,7 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "GetPromptResponseBody" }, "GetPromptVersionTagsResponseBody": { @@ -2743,10 +3134,7 @@ } }, "type": "object", - "required": [ - "data", - "next_cursor" - ], + "required": ["data", "next_cursor"], "title": "GetPromptVersionTagsResponseBody" }, "GetPromptVersionsResponseBody": { @@ -2771,10 +3159,7 @@ } }, "type": "object", - "required": [ - "data", - "next_cursor" - ], + "required": ["data", "next_cursor"], "title": "GetPromptVersionsResponseBody" }, "GetPromptsResponseBody": { @@ -2799,10 +3184,7 @@ } }, "type": "object", - "required": [ - "data", - "next_cursor" - ], + "required": ["data", "next_cursor"], "title": "GetPromptsResponseBody" }, "HTTPValidationError": { @@ -2832,9 +3214,7 @@ } }, "type": "object", - "required": [ - "id" - ], + "required": ["id"], "title": "InsertedSpanAnnotation" }, "ListDatasetExamplesData": { @@ -2856,11 +3236,7 @@ } }, "type": "object", - "required": [ - "dataset_id", - "version_id", - "examples" - ], + "required": ["dataset_id", "version_id", "examples"], "title": "ListDatasetExamplesData" }, "ListDatasetExamplesResponseBody": { @@ -2870,9 +3246,7 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "ListDatasetExamplesResponseBody" }, "ListDatasetVersionsResponseBody": { @@ -2897,10 +3271,7 @@ } }, "type": "object", - "required": [ - "data", - "next_cursor" - ], + "required": ["data", "next_cursor"], "title": "ListDatasetVersionsResponseBody" }, "ListDatasetsResponseBody": { @@ -2925,10 +3296,7 @@ } }, "type": "object", - "required": [ - "data", - "next_cursor" - ], + "required": ["data", "next_cursor"], "title": "ListDatasetsResponseBody" }, "ListExperimentsResponseBody": { @@ -2942,50 +3310,18 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "ListExperimentsResponseBody" }, "ModelProvider": { "type": "string", - "enum": [ - "OPENAI", - "AZURE_OPENAI", - "ANTHROPIC", - "GOOGLE" - ], + "enum": ["OPENAI", "AZURE_OPENAI", "ANTHROPIC", "GOOGLE"], "title": "ModelProvider" }, - "Project": { - "properties": { - "name": { - "type": "string", - "minLength": 1, - "title": "Name" - }, - "description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Description" - }, - "id": { - "type": "string", - "title": "Id" - } - }, - "type": "object", - "required": [ - "name", - "id" - ], - "title": "Project" + "OptimizationDirection": { + "type": "string", + "enum": ["MINIMIZE", "MAXIMIZE"], + "title": "OptimizationDirection" }, "Prompt": { "properties": { @@ -3020,10 +3356,7 @@ } }, "type": "object", - "required": [ - "name", - "id" - ], + "required": ["name", "id"], "title": "Prompt" }, "PromptAnthropicInvocationParameters": { @@ -3039,10 +3372,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "anthropic" - ], + "required": ["type", "anthropic"], "title": "PromptAnthropicInvocationParameters" }, "PromptAnthropicInvocationParametersContent": { @@ -3087,9 +3417,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "max_tokens" - ], + "required": ["max_tokens"], "title": "PromptAnthropicInvocationParametersContent" }, "PromptAnthropicThinkingConfigDisabled": { @@ -3102,9 +3430,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type" - ], + "required": ["type"], "title": "PromptAnthropicThinkingConfigDisabled" }, "PromptAnthropicThinkingConfigEnabled": { @@ -3122,10 +3448,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "budget_tokens" - ], + "required": ["type", "budget_tokens"], "title": "PromptAnthropicThinkingConfigEnabled" }, "PromptAzureOpenAIInvocationParameters": { @@ -3141,10 +3464,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "azure_openai" - ], + "required": ["type", "azure_openai"], "title": "PromptAzureOpenAIInvocationParameters" }, "PromptAzureOpenAIInvocationParametersContent": { @@ -3179,11 +3499,7 @@ }, "reasoning_effort": { "type": "string", - "enum": [ - "low", - "medium", - "high" - ], + "enum": ["low", "medium", "high"], "title": "Reasoning Effort" } }, @@ -3208,10 +3524,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "messages" - ], + "required": ["type", "messages"], "title": "PromptChatTemplate" }, "PromptData": { @@ -3243,9 +3556,7 @@ } }, "type": "object", - "required": [ - "name" - ], + "required": ["name"], "title": "PromptData" }, "PromptGoogleInvocationParameters": { @@ -3261,10 +3572,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "google" - ], + "required": ["type", "google"], "title": "PromptGoogleInvocationParameters" }, "PromptGoogleInvocationParametersContent": { @@ -3356,10 +3664,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "role", - "content" - ], + "required": ["role", "content"], "title": "PromptMessage" }, "PromptOpenAIInvocationParameters": { @@ -3375,10 +3680,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "openai" - ], + "required": ["type", "openai"], "title": "PromptOpenAIInvocationParameters" }, "PromptOpenAIInvocationParametersContent": { @@ -3413,11 +3715,7 @@ }, "reasoning_effort": { "type": "string", - "enum": [ - "low", - "medium", - "high" - ], + "enum": ["low", "medium", "high"], "title": "Reasoning Effort" } }, @@ -3438,10 +3736,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "json_schema" - ], + "required": ["type", "json_schema"], "title": "PromptResponseFormatJSONSchema" }, "PromptResponseFormatJSONSchemaDefinition": { @@ -3466,9 +3761,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "name" - ], + "required": ["name"], "title": "PromptResponseFormatJSONSchemaDefinition" }, "PromptStringTemplate": { @@ -3485,27 +3778,17 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "template" - ], + "required": ["type", "template"], "title": "PromptStringTemplate" }, "PromptTemplateFormat": { "type": "string", - "enum": [ - "MUSTACHE", - "F_STRING", - "NONE" - ], + "enum": ["MUSTACHE", "F_STRING", "NONE"], "title": "PromptTemplateFormat" }, "PromptTemplateType": { "type": "string", - "enum": [ - "STR", - "CHAT" - ], + "enum": ["STR", "CHAT"], "title": "PromptTemplateType" }, "PromptToolChoiceNone": { @@ -3518,9 +3801,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type" - ], + "required": ["type"], "title": "PromptToolChoiceNone" }, "PromptToolChoiceOneOrMore": { @@ -3533,9 +3814,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type" - ], + "required": ["type"], "title": "PromptToolChoiceOneOrMore" }, "PromptToolChoiceSpecificFunctionTool": { @@ -3552,10 +3831,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "function_name" - ], + "required": ["type", "function_name"], "title": "PromptToolChoiceSpecificFunctionTool" }, "PromptToolChoiceZeroOrMore": { @@ -3568,9 +3844,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type" - ], + "required": ["type"], "title": "PromptToolChoiceZeroOrMore" }, "PromptToolFunction": { @@ -3586,10 +3860,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "function" - ], + "required": ["type", "function"], "title": "PromptToolFunction" }, "PromptToolFunctionDefinition": { @@ -3614,9 +3885,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "name" - ], + "required": ["name"], "title": "PromptToolFunctionDefinition" }, "PromptTools": { @@ -3677,10 +3946,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "tools" - ], + "required": ["type", "tools"], "title": "PromptTools" }, "PromptVersion": { @@ -3936,10 +4202,7 @@ } }, "type": "object", - "required": [ - "name", - "id" - ], + "required": ["name", "id"], "title": "PromptVersionTag" }, "PromptVersionTagData": { @@ -3960,9 +4223,7 @@ } }, "type": "object", - "required": [ - "name" - ], + "required": ["name"], "title": "PromptVersionTagData" }, "SpanAnnotation": { @@ -3979,10 +4240,7 @@ }, "annotator_kind": { "type": "string", - "enum": [ - "LLM", - "HUMAN" - ], + "enum": ["LLM", "HUMAN"], "title": "Annotator Kind", "description": "The kind of annotator used for the annotation" }, @@ -4012,11 +4270,7 @@ } }, "type": "object", - "required": [ - "span_id", - "name", - "annotator_kind" - ], + "required": ["span_id", "name", "annotator_kind"], "title": "SpanAnnotation" }, "SpanAnnotationResult": { @@ -4075,10 +4329,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "text" - ], + "required": ["type", "text"], "title": "TextContentPart" }, "ToolCallContentPart": { @@ -4109,11 +4360,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "tool_call_id", - "tool_call" - ], + "required": ["type", "tool_call_id", "tool_call"], "title": "ToolCallContentPart" }, "ToolCallFunction": { @@ -4134,11 +4381,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "name", - "arguments" - ], + "required": ["type", "name", "arguments"], "title": "ToolCallFunction" }, "ToolResultContentPart": { @@ -4183,11 +4426,7 @@ }, "additionalProperties": false, "type": "object", - "required": [ - "type", - "tool_call_id", - "tool_result" - ], + "required": ["type", "tool_call_id", "tool_result"], "title": "ToolResultContentPart" }, "UpdateProjectRequestBody": { @@ -4214,9 +4453,7 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "UpdateProjectResponseBody" }, "UploadDatasetData": { @@ -4227,9 +4464,7 @@ } }, "type": "object", - "required": [ - "dataset_id" - ], + "required": ["dataset_id"], "title": "UploadDatasetData" }, "UploadDatasetResponseBody": { @@ -4239,9 +4474,7 @@ } }, "type": "object", - "required": [ - "data" - ], + "required": ["data"], "title": "UploadDatasetResponseBody" }, "ValidationError": { @@ -4270,13 +4503,9 @@ } }, "type": "object", - "required": [ - "loc", - "msg", - "type" - ], + "required": ["loc", "msg", "type"], "title": "ValidationError" } } } -} \ No newline at end of file +} diff --git a/src/phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py b/src/phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py index 00c6e416cc..b42a101939 100644 --- a/src/phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py +++ b/src/phoenix/db/migrations/versions/2f9d1a65945f_annotation_config_migration.py @@ -8,6 +8,7 @@ from typing import Sequence, Union +import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. @@ -28,8 +29,115 @@ def upgrade() -> None: type_="unique", ) + op.create_table( + "annotation_configs", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("name", sa.String, nullable=False, unique=True), + sa.Column( + "annotation_type", + sa.String, + sa.CheckConstraint( + "annotation_type IN ('CATEGORICAL', 'CONTINUOUS', 'FREEFORM')", + name="valid_annotation_type", + ), + nullable=False, + ), + sa.Column("description", sa.String, nullable=True), + ) + + op.create_table( + "continuous_annotation_configs", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "annotation_config_id", + sa.Integer, + sa.ForeignKey("annotation_configs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column( + "optimization_direction", + sa.String, + sa.CheckConstraint( + "optimization_direction IN ('MINIMIZE', 'MAXIMIZE')", + name="valid_optimization_direction", + ), + nullable=False, + ), + sa.Column("lower_bound", sa.Float, nullable=True), + sa.Column("upper_bound", sa.Float, nullable=True), + ) + + op.create_table( + "categorical_annotation_configs", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "annotation_config_id", + sa.Integer, + sa.ForeignKey("annotation_configs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column( + "optimization_direction", + sa.String, + sa.CheckConstraint( + "optimization_direction IN ('MINIMIZE', 'MAXIMIZE')", + name="valid_optimization_direction", + ), + nullable=False, + ), + ) + + op.create_table( + "categorical_annotation_values", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "categorical_annotation_config_id", + sa.Integer, + sa.ForeignKey("categorical_annotation_configs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("label", sa.String, nullable=False), + sa.Column("score", sa.Float, nullable=True), + sa.UniqueConstraint( + "categorical_annotation_config_id", + "label", + ), + ) + + op.create_table( + "project_annotation_configs", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "project_id", + sa.Integer, + sa.ForeignKey("projects.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column( + "annotation_config_id", + sa.Integer, + sa.ForeignKey("annotation_configs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.UniqueConstraint( + "project_id", + "annotation_config_id", + ), + ) + def downgrade() -> None: + op.drop_table("project_annotation_configs") + op.drop_table("categorical_annotation_values") + op.drop_table("categorical_annotation_configs") + op.drop_table("continuous_annotation_configs") + op.drop_table("annotation_configs") + with op.batch_alter_table("span_annotations", recreate="auto") as batch_op: batch_op.create_unique_constraint( "uq_span_annotations_name_span_rowid", diff --git a/src/phoenix/db/models.py b/src/phoenix/db/models.py index 88697544fd..8cd7c108d8 100644 --- a/src/phoenix/db/models.py +++ b/src/phoenix/db/models.py @@ -1281,3 +1281,112 @@ class PromptVersionTag(Base): ) __table_args__ = (UniqueConstraint("name", "prompt_id"),) + + +class AnnotationConfig(Base): + __tablename__ = "annotation_configs" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String, nullable=False, unique=True) + annotation_type: Mapped[str] = mapped_column( + String, + CheckConstraint( + "annotation_type IN ('CATEGORICAL', 'CONTINUOUS', 'FREEFORM')", + name="valid_annotation_type", + ), + nullable=False, + ) + description: Mapped[Optional[str]] = mapped_column(String, nullable=True) + + continuous_annotation_config = relationship( + "ContinuousAnnotationConfig", back_populates="annotation_config", uselist=False + ) + categorical_annotation_config = relationship( + "CategoricalAnnotationConfig", back_populates="annotation_config", uselist=False + ) + + +class ContinuousAnnotationConfig(Base): + __tablename__ = "continuous_annotation_configs" + + id: Mapped[int] = mapped_column(primary_key=True) + annotation_config_id: Mapped[int] = mapped_column( + ForeignKey("annotation_configs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + optimization_direction: Mapped[str] = mapped_column( + String, + CheckConstraint( + "optimization_direction IN ('MINIMIZE', 'MAXIMIZE')", + name="valid_optimization_direction", + ), + nullable=False, + ) + lower_bound: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + upper_bound: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + + annotation_config = relationship( + "AnnotationConfig", back_populates="continuous_annotation_config" + ) + + +class CategoricalAnnotationConfig(Base): + __tablename__ = "categorical_annotation_configs" + + id: Mapped[int] = mapped_column(primary_key=True) + annotation_config_id: Mapped[int] = mapped_column( + ForeignKey("annotation_configs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + optimization_direction: Mapped[str] = mapped_column( + String, + CheckConstraint( + "optimization_direction IN ('MINIMIZE', 'MAXIMIZE')", + name="valid_optimization_direction", + ), + nullable=False, + ) + + annotation_config = relationship( + "AnnotationConfig", back_populates="categorical_annotation_config" + ) + values = relationship( + "CategoricalAnnotationValue", + back_populates="categorical_annotation_config", + cascade="all, delete-orphan", + ) + + +class CategoricalAnnotationValue(Base): + __tablename__ = "categorical_annotation_values" + + id: Mapped[int] = mapped_column(primary_key=True) + categorical_annotation_config_id: Mapped[int] = mapped_column( + ForeignKey("categorical_annotation_configs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + label: Mapped[str] = mapped_column(String, nullable=False) + score: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + + categorical_annotation_config = relationship( + "CategoricalAnnotationConfig", back_populates="values" + ) + + __table_args__ = (UniqueConstraint("categorical_annotation_config_id", "label"),) + + +class ProjectAnnotationConfig(Base): + __tablename__ = "project_annotation_configs" + + id: Mapped[int] = mapped_column(primary_key=True) + project_id: Mapped[int] = mapped_column( + ForeignKey("projects.id", ondelete="CASCADE"), nullable=False, index=True + ) + annotation_config_id: Mapped[int] = mapped_column( + ForeignKey("annotation_configs.id", ondelete="CASCADE"), nullable=False, index=True + ) + + __table_args__ = (UniqueConstraint("project_id", "annotation_config_id"),) diff --git a/src/phoenix/server/api/mutations/__init__.py b/src/phoenix/server/api/mutations/__init__.py index 010f9b28e4..848401b3f3 100644 --- a/src/phoenix/server/api/mutations/__init__.py +++ b/src/phoenix/server/api/mutations/__init__.py @@ -1,5 +1,6 @@ import strawberry +from phoenix.server.api.mutations.annotation_config_mutations import AnnotationConfigMutationMixin from phoenix.server.api.mutations.api_key_mutations import ApiKeyMutationMixin from phoenix.server.api.mutations.chat_mutations import ( ChatCompletionMutationMixin, @@ -19,6 +20,7 @@ @strawberry.type class Mutation( + AnnotationConfigMutationMixin, ApiKeyMutationMixin, ChatCompletionMutationMixin, DatasetMutationMixin, diff --git a/src/phoenix/server/api/mutations/annotation_config_mutations.py b/src/phoenix/server/api/mutations/annotation_config_mutations.py new file mode 100644 index 0000000000..6e4a4663c8 --- /dev/null +++ b/src/phoenix/server/api/mutations/annotation_config_mutations.py @@ -0,0 +1,418 @@ +from typing import List, Optional + +import strawberry +from sqlalchemy import delete, select +from sqlalchemy.exc import IntegrityError as PostgreSQLIntegrityError +from sqlalchemy.orm import joinedload +from sqlean.dbapi2 import IntegrityError as SQLiteIntegrityError # type: ignore[import-untyped] +from strawberry.relay.types import GlobalID +from strawberry.types import Info + +from phoenix.db import models +from phoenix.server.api.context import Context +from phoenix.server.api.exceptions import BadRequest, Conflict, NotFound +from phoenix.server.api.queries import Query +from phoenix.server.api.types.AnnotationConfig import ( + AnnotationConfig, + AnnotationType, + CategoricalAnnotationConfig, + ContinuousAnnotationConfig, + FreeformAnnotationConfig, + OptimizationDirection, + to_gql_annotation_config, + to_gql_categorical_annotation_config, + to_gql_continuous_annotation_config, + to_gql_freeform_annotation_config, +) +from phoenix.server.api.types.node import from_global_id_with_expected_type +from phoenix.server.api.types.Project import Project + +ANNOTATION_TYPE_NAMES = ( + CategoricalAnnotationConfig.__name__, + ContinuousAnnotationConfig.__name__, + FreeformAnnotationConfig.__name__, +) + + +@strawberry.input +class CreateContinuousAnnotationConfigInput: + name: str + optimization_direction: OptimizationDirection + description: Optional[str] = None + lower_bound: Optional[float] = None + upper_bound: Optional[float] = None + + +@strawberry.type +class CreateContinuousAnnotationConfigPayload: + query: Query + annotation_config: ContinuousAnnotationConfig + + +@strawberry.input +class CategoricalAnnotationValueInput: + label: str + score: Optional[float] = None + + +@strawberry.input +class CreateCategoricalAnnotationConfigInput: + name: str + optimization_direction: OptimizationDirection + description: Optional[str] = None + values: List[CategoricalAnnotationValueInput] + + +@strawberry.type +class CreateCategoricalAnnotationConfigPayload: + query: Query + annotation_config: CategoricalAnnotationConfig + + +@strawberry.input +class CreateFreeformAnnotationConfigInput: + name: str + description: Optional[str] = None + + +@strawberry.type +class CreateFreeformAnnotationConfigPayload: + query: Query + annotation_config: FreeformAnnotationConfig + + +@strawberry.input +class UpdateCategoricalAnnotationConfigInput: + config_id: GlobalID + name: str + optimization_direction: OptimizationDirection + description: Optional[str] = None + values: List[CategoricalAnnotationValueInput] + + +@strawberry.type +class UpdateCategoricalAnnotationConfigPayload: + query: Query + annotation_config: CategoricalAnnotationConfig + + +@strawberry.input +class DeleteAnnotationConfigInput: + config_id: GlobalID + + +@strawberry.type +class DeleteAnnotationConfigPayload: + query: Query + annotation_config: AnnotationConfig + + +@strawberry.input +class AddAnnotationConfigToProjectInput: + project_id: GlobalID + annotation_config_id: GlobalID + + +@strawberry.type +class AddAnnotationConfigToProjectPayload: + query: Query + project: Project + + +@strawberry.input +class UpdateContinuousAnnotationConfigInput: + config_id: GlobalID + name: str + optimization_direction: OptimizationDirection + description: Optional[str] = None + lower_bound: Optional[float] = None + upper_bound: Optional[float] = None + + +@strawberry.type +class UpdateContinuousAnnotationConfigPayload: + query: Query + annotation_config: ContinuousAnnotationConfig + + +@strawberry.input +class UpdateFreeformAnnotationConfigInput: + config_id: GlobalID + name: str + description: Optional[str] = None + + +@strawberry.type +class UpdateFreeformAnnotationConfigPayload: + query: Query + annotation_config: FreeformAnnotationConfig + + +@strawberry.type +class AnnotationConfigMutationMixin: + @strawberry.mutation + async def create_categorical_annotation_config( + self, + info: Info[Context, None], + input: CreateCategoricalAnnotationConfigInput, + ) -> CreateCategoricalAnnotationConfigPayload: + async with info.context.db() as session: + annotation_config = models.AnnotationConfig( + name=input.name, + annotation_type=AnnotationType.CATEGORICAL.value, + description=input.description, + ) + categorical_annotation_config = models.CategoricalAnnotationConfig( + optimization_direction=input.optimization_direction.value, + ) + for value in input.values: + categorical_annotation_config.values.append( + models.CategoricalAnnotationValue( + label=value.label, + score=value.score, + ) + ) + annotation_config.categorical_annotation_config = categorical_annotation_config + session.add(annotation_config) + try: + await session.commit() + except (PostgreSQLIntegrityError, SQLiteIntegrityError): + raise Conflict(f"Annotation configuration with name '{input.name}' already exists") + return CreateCategoricalAnnotationConfigPayload( + query=Query(), + annotation_config=to_gql_categorical_annotation_config(annotation_config), + ) + + @strawberry.mutation + async def create_continuous_annotation_config( + self, + info: Info[Context, None], + input: CreateContinuousAnnotationConfigInput, + ) -> CreateContinuousAnnotationConfigPayload: + async with info.context.db() as session: + annotation_config = models.AnnotationConfig( + name=input.name, + annotation_type=AnnotationType.CONTINUOUS.value, + description=input.description, + ) + continuous_annotation_config = models.ContinuousAnnotationConfig( + optimization_direction=input.optimization_direction.value, + lower_bound=input.lower_bound, + upper_bound=input.upper_bound, + ) + annotation_config.continuous_annotation_config = continuous_annotation_config + session.add(annotation_config) + try: + await session.commit() + except (PostgreSQLIntegrityError, SQLiteIntegrityError): + raise Conflict("The annotation config has a conflict") + return CreateContinuousAnnotationConfigPayload( + query=Query(), + annotation_config=to_gql_continuous_annotation_config(annotation_config), + ) + + @strawberry.mutation + async def create_freeform_annotation_config( + self, + info: Info[Context, None], + input: CreateFreeformAnnotationConfigInput, + ) -> CreateFreeformAnnotationConfigPayload: + async with info.context.db() as session: + config = models.AnnotationConfig( + name=input.name, + annotation_type="FREEFORM", + description=input.description, + ) + session.add(config) + try: + await session.commit() + except (PostgreSQLIntegrityError, SQLiteIntegrityError): + raise Conflict(f"Annotation configuration with name '{input.name}' already exists") + return CreateFreeformAnnotationConfigPayload( + query=Query(), + annotation_config=to_gql_freeform_annotation_config(config), + ) + + @strawberry.mutation + async def update_categorical_annotation_config( + self, + info: Info[Context, None], + input: UpdateCategoricalAnnotationConfigInput, + ) -> UpdateCategoricalAnnotationConfigPayload: + config_id = from_global_id_with_expected_type( + global_id=input.config_id, expected_type_name=CategoricalAnnotationConfig.__name__ + ) + async with info.context.db() as session: + annotation_config = await session.scalar( + select(models.AnnotationConfig) + .options( + joinedload(models.AnnotationConfig.categorical_annotation_config).joinedload( + models.CategoricalAnnotationConfig.values + ) + ) + .where(models.AnnotationConfig.id == config_id) + ) + if not annotation_config: + raise NotFound(f"Annotation configuration with ID '{input.config_id}' not found") + + annotation_config.name = input.name + annotation_config.description = input.description + + assert annotation_config.categorical_annotation_config is not None + annotation_config.categorical_annotation_config.optimization_direction = ( + input.optimization_direction.value + ) + + await session.execute( + delete(models.CategoricalAnnotationValue).where( + models.CategoricalAnnotationValue.categorical_annotation_config_id + == annotation_config.categorical_annotation_config.id + ) + ) + + annotation_config.categorical_annotation_config.values.clear() + for val in input.values: + annotation_config.categorical_annotation_config.values.append( + models.CategoricalAnnotationValue( + label=val.label, + score=val.score, + ) + ) + + session.add(annotation_config) + try: + await session.commit() + except (PostgreSQLIntegrityError, SQLiteIntegrityError): + raise Conflict("The annotation config has a conflict") + + return UpdateCategoricalAnnotationConfigPayload( + query=Query(), + annotation_config=to_gql_categorical_annotation_config(annotation_config), + ) + + @strawberry.mutation + async def update_continuous_annotation_config( + self, + info: Info[Context, None], + input: UpdateContinuousAnnotationConfigInput, + ) -> UpdateContinuousAnnotationConfigPayload: + config_id = from_global_id_with_expected_type( + global_id=input.config_id, expected_type_name=ContinuousAnnotationConfig.__name__ + ) + async with info.context.db() as session: + annotation_config = await session.scalar( + select(models.AnnotationConfig) + .options(joinedload(models.AnnotationConfig.continuous_annotation_config)) + .where(models.AnnotationConfig.id == config_id) + ) + if not annotation_config: + raise NotFound(f"Annotation configuration with ID '{input.config_id}' not found") + + annotation_config.name = input.name + annotation_config.description = input.description + + assert annotation_config.continuous_annotation_config is not None + annotation_config.continuous_annotation_config.optimization_direction = ( + input.optimization_direction.value + ) + annotation_config.continuous_annotation_config.lower_bound = input.lower_bound + annotation_config.continuous_annotation_config.upper_bound = input.upper_bound + + session.add(annotation_config) + try: + await session.commit() + except (PostgreSQLIntegrityError, SQLiteIntegrityError): + raise Conflict(f"Annotation configuration with name '{input.name}' already exists") + + return UpdateContinuousAnnotationConfigPayload( + query=Query(), + annotation_config=to_gql_continuous_annotation_config(annotation_config), + ) + + @strawberry.mutation + async def update_freeform_annotation_config( + self, + info: Info[Context, None], + input: UpdateFreeformAnnotationConfigInput, + ) -> UpdateFreeformAnnotationConfigPayload: + config_id = from_global_id_with_expected_type( + global_id=input.config_id, expected_type_name=FreeformAnnotationConfig.__name__ + ) + async with info.context.db() as session: + annotation_config = await session.scalar( + select(models.AnnotationConfig).where(models.AnnotationConfig.id == config_id) + ) + if not annotation_config: + raise NotFound(f"Annotation configuration with ID '{input.config_id}' not found") + + annotation_config.name = input.name + annotation_config.description = input.description + + session.add(annotation_config) + try: + await session.commit() + except (PostgreSQLIntegrityError, SQLiteIntegrityError): + raise Conflict(f"Annotation configuration with name '{input.name}' already exists") + + return UpdateFreeformAnnotationConfigPayload( + query=Query(), + annotation_config=to_gql_freeform_annotation_config(annotation_config), + ) + + @strawberry.mutation + async def delete_annotation_config( + self, + info: Info[Context, None], + input: DeleteAnnotationConfigInput, + ) -> DeleteAnnotationConfigPayload: + if (type_name := input.config_id.type_name) not in ANNOTATION_TYPE_NAMES: + raise BadRequest(f"Unexpected type name in Relay ID: {type_name}") + config_id = int(input.config_id.node_id) + async with info.context.db() as session: + annotation_config = await session.scalar( + select(models.AnnotationConfig) + .where(models.AnnotationConfig.id == config_id) + .options( + joinedload(models.AnnotationConfig.continuous_annotation_config), + joinedload(models.AnnotationConfig.categorical_annotation_config).joinedload( + models.CategoricalAnnotationConfig.values + ), + ) + ) + if annotation_config is None: + raise NotFound(f"Annotation configuration with ID '{input.config_id}' not found") + await session.execute( + delete(models.AnnotationConfig).where(models.AnnotationConfig.id == config_id) + ) + return DeleteAnnotationConfigPayload( + query=Query(), + annotation_config=to_gql_annotation_config(annotation_config), + ) + + @strawberry.mutation + async def add_annotation_config_to_project( + self, + info: Info[Context, None], + input: list[AddAnnotationConfigToProjectInput], + ) -> AddAnnotationConfigToProjectPayload: + async with info.context.db() as session: + for item in input: + project_id = from_global_id_with_expected_type( + global_id=item.project_id, expected_type_name="Project" + ) + if (type_name := item.annotation_config_id.type_name) not in ANNOTATION_TYPE_NAMES: + raise BadRequest(f"Unexpected type name in Relay ID: {type_name}") + annotation_config_id = int(item.annotation_config_id.node_id) + project_annotation_config = models.ProjectAnnotationConfig( + project_id=project_id, + annotation_config_id=annotation_config_id, + ) + session.add(project_annotation_config) + try: + await session.commit() + except (PostgreSQLIntegrityError, SQLiteIntegrityError): + raise Conflict("The annotation config has already been added to the project") + return AddAnnotationConfigToProjectPayload( + query=Query(), + project=Project(project_rowid=project_id), + ) diff --git a/src/phoenix/server/api/queries.py b/src/phoenix/server/api/queries.py index 429ffe5aaf..7d96c9c359 100644 --- a/src/phoenix/server/api/queries.py +++ b/src/phoenix/server/api/queries.py @@ -42,6 +42,7 @@ from phoenix.server.api.input_types.Coordinates import InputCoordinate2D, InputCoordinate3D from phoenix.server.api.input_types.DatasetSort import DatasetSort from phoenix.server.api.input_types.InvocationParameters import InvocationParameter +from phoenix.server.api.types.AnnotationConfig import AnnotationConfig, to_gql_annotation_config from phoenix.server.api.types.Cluster import Cluster, to_gql_clusters from phoenix.server.api.types.Dataset import Dataset, to_gql_dataset from phoenix.server.api.types.DatasetExample import DatasetExample @@ -660,6 +661,36 @@ async def prompt_labels( args=args, ) + @strawberry.field + async def annotation_configs( + self, + info: Info[Context, None], + first: Optional[int] = 50, + last: Optional[int] = None, + after: Optional[str] = None, + before: Optional[str] = None, + ) -> Connection[AnnotationConfig]: + args = ConnectionArgs( + first=first, + after=after if isinstance(after, CursorString) else None, + last=last, + before=before if isinstance(before, CursorString) else None, + ) + async with info.context.db() as session: + stmt = ( + select(models.AnnotationConfig) + .order_by(models.AnnotationConfig.name) + .options( + joinedload(models.AnnotationConfig.continuous_annotation_config), + joinedload(models.AnnotationConfig.categorical_annotation_config).joinedload( + models.CategoricalAnnotationConfig.values + ), + ) + ) + configs = (await session.stream_scalars(stmt)).unique() + data = [to_gql_annotation_config(config) async for config in configs] + return connection_from_list(data=data, args=args) + @strawberry.field def clusters( self, diff --git a/src/phoenix/server/api/routers/v1/__init__.py b/src/phoenix/server/api/routers/v1/__init__.py index 9f0b614c69..fed2c7e9f5 100644 --- a/src/phoenix/server/api/routers/v1/__init__.py +++ b/src/phoenix/server/api/routers/v1/__init__.py @@ -4,6 +4,7 @@ from phoenix.server.bearer_auth import is_authenticated +from .annotation_configs import router as annotation_configs_router from .datasets import router as datasets_router from .evaluations import router as evaluations_router from .experiment_evaluations import router as experiment_evaluations_router @@ -56,6 +57,7 @@ def create_v1_router(authentication_enabled: bool) -> APIRouter: ] ), ) + router.include_router(annotation_configs_router) router.include_router(datasets_router) router.include_router(experiments_router) router.include_router(experiment_runs_router) diff --git a/src/phoenix/server/api/routers/v1/annotation_configs.py b/src/phoenix/server/api/routers/v1/annotation_configs.py new file mode 100644 index 0000000000..48c9f0a9de --- /dev/null +++ b/src/phoenix/server/api/routers/v1/annotation_configs.py @@ -0,0 +1,289 @@ +import logging +from enum import Enum +from typing import Any, List, Optional + +from fastapi import APIRouter, HTTPException, Path, Query +from pydantic import BaseModel +from sqlalchemy import delete, select +from sqlalchemy.orm import selectinload +from starlette.requests import Request +from starlette.status import ( + HTTP_400_BAD_REQUEST, + HTTP_404_NOT_FOUND, + HTTP_422_UNPROCESSABLE_ENTITY, +) +from strawberry.relay import GlobalID +from typing_extensions import assert_never + +from phoenix.db import models +from phoenix.server.api.types.AnnotationConfig import ( + CategoricalAnnotationConfig, + ContinuousAnnotationConfig, + FreeformAnnotationConfig, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["annotation_configs"]) + + +class CategoricalAnnotationValue(BaseModel): + label: str + score: Optional[float] = None + + +class OptimizationDirection(Enum): + MINIMIZE = "MINIMIZE" + MAXIMIZE = "MAXIMIZE" + + +class AnnotationType(Enum): + CONTINUOUS = "CONTINUOUS" + CATEGORICAL = "CATEGORICAL" + FREEFORM = "FREEFORM" + + +class AnnotationConfigResponse(BaseModel): + id: str + name: str + annotation_type: AnnotationType + optimization_direction: Optional[OptimizationDirection] = None + description: Optional[str] = None + lower_bound: Optional[float] = None + upper_bound: Optional[float] = None + values: Optional[List[CategoricalAnnotationValue]] = None + + +def annotation_config_to_response(config: models.AnnotationConfig) -> AnnotationConfigResponse: + """Convert an AnnotationConfig SQLAlchemy model instance to our response model.""" + base: dict[str, Any] = { + "name": config.name, + "annotation_type": config.annotation_type, + "description": config.description, + } + annotation_type = AnnotationType(config.annotation_type) + if annotation_type is AnnotationType.CONTINUOUS: + base["id"] = str(GlobalID(ContinuousAnnotationConfig.__name__, str(config.id))) + base["optimization_direction"] = config.continuous_annotation_config.optimization_direction + base["lower_bound"] = config.continuous_annotation_config.lower_bound + base["upper_bound"] = config.continuous_annotation_config.upper_bound + elif annotation_type is AnnotationType.CATEGORICAL: + base["id"] = str(GlobalID(CategoricalAnnotationConfig.__name__, str(config.id))) + base["optimization_direction"] = config.categorical_annotation_config.optimization_direction + base["values"] = [ + CategoricalAnnotationValue(label=val.label, score=val.score) + for val in config.categorical_annotation_config.values + ] + elif annotation_type is AnnotationType.FREEFORM: + base["id"] = str(GlobalID(FreeformAnnotationConfig.__name__, str(config.id))) + else: + assert_never(annotation_type) + return AnnotationConfigResponse(**base) + + +class CreateContinuousAnnotationConfigPayload(BaseModel): + name: str + optimization_direction: OptimizationDirection + description: Optional[str] = None + lower_bound: Optional[float] = None + upper_bound: Optional[float] = None + + +class CreateCategoricalAnnotationValuePayload(BaseModel): + label: str + score: Optional[float] = None + + +class CreateCategoricalAnnotationConfigPayload(BaseModel): + name: str + optimization_direction: OptimizationDirection + description: Optional[str] = None + values: List[CreateCategoricalAnnotationValuePayload] + + +class CreateFreeformAnnotationConfigPayload(BaseModel): + name: str + description: Optional[str] = None + + +@router.get( + "/annotation_configs", + response_model=List[AnnotationConfigResponse], + summary="List annotation configurations", +) +async def list_annotation_configs( + request: Request, + limit: int = Query(50, gt=0, description="Maximum number of configs to return"), +) -> List[AnnotationConfigResponse]: + async with request.app.state.db() as session: + result = await session.execute( + select(models.AnnotationConfig) + .options( + selectinload(models.AnnotationConfig.continuous_annotation_config), + selectinload(models.AnnotationConfig.categorical_annotation_config).selectinload( + models.CategoricalAnnotationConfig.values + ), + ) + .order_by(models.AnnotationConfig.name) + .limit(limit) + ) + configs = result.scalars().all() + return [annotation_config_to_response(config) for config in configs] + + +@router.get( + "/annotation_configs/{config_identifier}", + response_model=AnnotationConfigResponse, + summary="Get an annotation configuration by ID or name", +) +async def get_annotation_config_by_name_or_id( + request: Request, + config_identifier: str = Path(..., description="ID or name of the annotation configuration"), +) -> AnnotationConfigResponse: + async with request.app.state.db() as session: + query = select(models.AnnotationConfig).options( + selectinload(models.AnnotationConfig.continuous_annotation_config), + selectinload(models.AnnotationConfig.categorical_annotation_config).selectinload( + models.CategoricalAnnotationConfig.values + ), + ) + # Try to interpret the identifier as an integer ID; if not, use it as a name. + try: + db_id = _get_annotation_config_db_id(config_identifier) + query = query.where(models.AnnotationConfig.id == db_id) + except ValueError: + query = query.where(models.AnnotationConfig.name == config_identifier) + config = await session.scalar(query) + if not config: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, detail="Annotation configuration not found" + ) + return annotation_config_to_response(config) + + +@router.post( + "/annotation_configs/continuous", + response_model=AnnotationConfigResponse, + summary="Create a continuous annotation configuration", +) +async def create_continuous_annotation_config( + request: Request, + payload: CreateContinuousAnnotationConfigPayload, +) -> AnnotationConfigResponse: + async with request.app.state.db() as session: + annotation_config = models.AnnotationConfig( + name=payload.name, + annotation_type="CONTINUOUS", + description=payload.description, + ) + continuous_annotation_config = models.ContinuousAnnotationConfig( + optimization_direction=payload.optimization_direction.value, + lower_bound=payload.lower_bound, + upper_bound=payload.upper_bound, + ) + annotation_config.continuous_annotation_config = continuous_annotation_config + session.add(annotation_config) + try: + await session.commit() + except Exception as e: + raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) + return annotation_config_to_response(annotation_config) + + +@router.post( + "/annotation_configs/categorical", + response_model=AnnotationConfigResponse, + summary="Create a categorical annotation configuration", +) +async def create_categorical_annotation_config( + request: Request, + payload: CreateCategoricalAnnotationConfigPayload, +) -> AnnotationConfigResponse: + async with request.app.state.db() as session: + annotation_config = models.AnnotationConfig( + name=payload.name, + annotation_type="CATEGORICAL", + description=payload.description, + ) + categorical_annotation_config = models.CategoricalAnnotationConfig( + optimization_direction=payload.optimization_direction.value, + ) + for value in payload.values: + categorical_annotation_config.values.append( + models.CategoricalAnnotationValue( + label=value.label, + score=value.score, + ) + ) + annotation_config.categorical_annotation_config = categorical_annotation_config + session.add(annotation_config) + try: + await session.commit() + except Exception as e: + raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) + return annotation_config_to_response(annotation_config) + + +@router.post( + "/annotation_configs/freeform", + response_model=AnnotationConfigResponse, + summary="Create a freeform annotation configuration", +) +async def create_freeform_annotation_config( + request: Request, + payload: CreateFreeformAnnotationConfigPayload, +) -> AnnotationConfigResponse: + async with request.app.state.db() as session: + annotation_config = models.AnnotationConfig( + name=payload.name, + annotation_type="FREEFORM", + description=payload.description, + ) + session.add(annotation_config) + try: + await session.commit() + except Exception as e: + raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)) + return annotation_config_to_response(annotation_config) + + +@router.delete( + "/annotation_configs/{config_id}", + response_model=bool, + summary="Delete an annotation configuration", +) +async def delete_annotation_config( + request: Request, + config_id: str = Path(..., description="ID of the annotation configuration"), +) -> bool: + config_gid = GlobalID.from_id(config_id) + if config_gid.type_name not in ( + CategoricalAnnotationConfig.__name__, + ContinuousAnnotationConfig.__name__, + FreeformAnnotationConfig.__name__, + ): + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, detail="Invalid annotation configuration ID" + ) + config_rowid = int(config_gid.node_id) + async with request.app.state.db() as session: + stmt = delete(models.AnnotationConfig).where(models.AnnotationConfig.id == config_rowid) + result = await session.execute(stmt) + if result.rowcount == 0: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, detail="Annotation configuration not found" + ) + await session.commit() + return True + + +def _get_annotation_config_db_id(config_gid: str) -> int: + gid = GlobalID.from_id(config_gid) + type_name, node_id = gid.type_name, int(gid.node_id) + if type_name not in ( + CategoricalAnnotationConfig.__name__, + ContinuousAnnotationConfig.__name__, + FreeformAnnotationConfig.__name__, + ): + raise ValueError(f"Invalid annotation configuration ID: {config_gid}") + return node_id diff --git a/src/phoenix/server/api/types/AnnotationConfig.py b/src/phoenix/server/api/types/AnnotationConfig.py new file mode 100644 index 0000000000..61b44fde56 --- /dev/null +++ b/src/phoenix/server/api/types/AnnotationConfig.py @@ -0,0 +1,131 @@ +from enum import Enum +from typing import Annotated, List, Optional, Union + +import strawberry +from strawberry.relay import Node, NodeID +from typing_extensions import TypeAlias, assert_never + +from phoenix.db import models + + +@strawberry.enum +class AnnotationType(Enum): + CATEGORICAL = "CATEGORICAL" + CONTINUOUS = "CONTINUOUS" + FREEFORM = "FREEFORM" + + +@strawberry.enum +class OptimizationDirection(Enum): + MINIMIZE = "MINIMIZE" + MAXIMIZE = "MAXIMIZE" + + +@strawberry.type +class CategoricalAnnotationValue: + label: str + score: Optional[float] + + +@strawberry.type +class CategoricalAnnotationConfig(Node): + id_attr: NodeID[int] + name: str + annotation_type: AnnotationType + description: Optional[str] + optimization_direction: OptimizationDirection + values: List[CategoricalAnnotationValue] + + +@strawberry.type +class ContinuousAnnotationConfig(Node): + id_attr: NodeID[int] + name: str + annotation_type: AnnotationType + description: Optional[str] + optimization_direction: OptimizationDirection + lower_bound: Optional[float] + upper_bound: Optional[float] + + +@strawberry.type +class FreeformAnnotationConfig(Node): + id_attr: NodeID[int] + name: str + annotation_type: AnnotationType + description: Optional[str] + + +AnnotationConfig: TypeAlias = Annotated[ + Union[CategoricalAnnotationConfig, ContinuousAnnotationConfig, FreeformAnnotationConfig], + strawberry.union("AnnotationConfig"), +] + + +def to_gql_categorical_annotation_config( + annotation_config: models.AnnotationConfig, +) -> CategoricalAnnotationConfig: + gql_annotation_type = AnnotationType(annotation_config.annotation_type) + assert gql_annotation_type is AnnotationType.CATEGORICAL + categorical_config = annotation_config.categorical_annotation_config + assert categorical_config is not None + values = [ + CategoricalAnnotationValue( + label=val.label, + score=val.score, + ) + for val in categorical_config.values + ] + return CategoricalAnnotationConfig( + id_attr=annotation_config.id, + name=annotation_config.name, + annotation_type=AnnotationType.CATEGORICAL, + optimization_direction=OptimizationDirection(categorical_config.optimization_direction), + description=annotation_config.description, + values=values, + ) + + +def to_gql_continuous_annotation_config( + annotation_config: models.AnnotationConfig, +) -> ContinuousAnnotationConfig: + gql_annotation_type = AnnotationType(annotation_config.annotation_type) + assert gql_annotation_type is AnnotationType.CONTINUOUS + continuous_config = annotation_config.continuous_annotation_config + assert continuous_config is not None + return ContinuousAnnotationConfig( + id_attr=annotation_config.id, + name=annotation_config.name, + annotation_type=AnnotationType.CONTINUOUS, + optimization_direction=OptimizationDirection(continuous_config.optimization_direction), + description=annotation_config.description, + lower_bound=continuous_config.lower_bound, + upper_bound=continuous_config.upper_bound, + ) + + +def to_gql_freeform_annotation_config( + annotation_config: models.AnnotationConfig, +) -> FreeformAnnotationConfig: + gql_annotation_type = AnnotationType(annotation_config.annotation_type) + assert gql_annotation_type is AnnotationType.FREEFORM + return FreeformAnnotationConfig( + id_attr=annotation_config.id, + name=annotation_config.name, + annotation_type=AnnotationType.FREEFORM, + description=annotation_config.description, + ) + + +def to_gql_annotation_config(annotation_config: models.AnnotationConfig) -> AnnotationConfig: + """ + Convert an SQLAlchemy AnnotationConfig instance to one of the GraphQL types. + """ + gql_annotation_type = AnnotationType(annotation_config.annotation_type) + if gql_annotation_type is AnnotationType.CONTINUOUS: + return to_gql_continuous_annotation_config(annotation_config) + elif gql_annotation_type == AnnotationType.CATEGORICAL: + return to_gql_categorical_annotation_config(annotation_config) + elif gql_annotation_type is AnnotationType.FREEFORM: + return to_gql_freeform_annotation_config(annotation_config) + assert_never(annotation_config) diff --git a/src/phoenix/server/api/types/Project.py b/src/phoenix/server/api/types/Project.py index 93b6b9a5f8..df734903c3 100644 --- a/src/phoenix/server/api/types/Project.py +++ b/src/phoenix/server/api/types/Project.py @@ -6,6 +6,7 @@ from aioitertools.itertools import islice from openinference.semconv.trace import SpanAttributes from sqlalchemy import desc, distinct, func, or_, select +from sqlalchemy.orm import joinedload from sqlalchemy.sql.elements import ColumnElement from sqlalchemy.sql.expression import tuple_ from strawberry import ID, UNSET, Private @@ -22,13 +23,16 @@ ) from phoenix.server.api.input_types.SpanSort import SpanSort, SpanSortConfig from phoenix.server.api.input_types.TimeRange import TimeRange +from phoenix.server.api.types.AnnotationConfig import AnnotationConfig, to_gql_annotation_config from phoenix.server.api.types.AnnotationSummary import AnnotationSummary from phoenix.server.api.types.DocumentEvaluationSummary import DocumentEvaluationSummary from phoenix.server.api.types.pagination import ( + ConnectionArgs, Cursor, CursorSortColumn, CursorString, connection_from_cursors_and_nodes, + connection_from_list, ) from phoenix.server.api.types.ProjectSession import ProjectSession, to_gql_project_session from phoenix.server.api.types.SortDir import SortDir @@ -536,6 +540,43 @@ async def validate_span_filter_condition(self, condition: str) -> ValidationResu error_message=e.msg, ) + @strawberry.field + async def annotation_configs( + self, + info: Info[Context, None], + first: Optional[int] = 50, + last: Optional[int] = None, + after: Optional[str] = None, + before: Optional[str] = None, + ) -> Connection[AnnotationConfig]: + args = ConnectionArgs( + first=first, + after=after if isinstance(after, CursorString) else None, + last=last, + before=before if isinstance(before, CursorString) else None, + ) + async with info.context.db() as session: + annotation_configs = ( + await session.stream_scalars( + select(models.AnnotationConfig) + .join( + models.ProjectAnnotationConfig, + models.AnnotationConfig.id + == models.ProjectAnnotationConfig.annotation_config_id, + ) + .where(models.ProjectAnnotationConfig.project_id == self.project_rowid) + .order_by(models.AnnotationConfig.name) + .options( + joinedload( + models.AnnotationConfig.categorical_annotation_config + ).joinedload(models.CategoricalAnnotationConfig.values), + joinedload(models.AnnotationConfig.continuous_annotation_config), + ) + ) + ).unique() + data = [to_gql_annotation_config(config) async for config in annotation_configs] + return connection_from_list(data=data, args=args) + INPUT_VALUE = SpanAttributes.INPUT_VALUE.split(".") OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE.split(".") From 6956aefeab99245a8482410ccdf4cf4b0b3186c7 Mon Sep 17 00:00:00 2001 From: Xander Song Date: Thu, 20 Mar 2025 09:03:32 -0700 Subject: [PATCH 04/53] fix(annotation-configs): make mutations read-only (#6850) --- .../mutations/annotation_config_mutations.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/phoenix/server/api/mutations/annotation_config_mutations.py b/src/phoenix/server/api/mutations/annotation_config_mutations.py index 6e4a4663c8..a1614f5e4f 100644 --- a/src/phoenix/server/api/mutations/annotation_config_mutations.py +++ b/src/phoenix/server/api/mutations/annotation_config_mutations.py @@ -9,6 +9,7 @@ from strawberry.types import Info from phoenix.db import models +from phoenix.server.api.auth import IsNotReadOnly from phoenix.server.api.context import Context from phoenix.server.api.exceptions import BadRequest, Conflict, NotFound from phoenix.server.api.queries import Query @@ -150,7 +151,7 @@ class UpdateFreeformAnnotationConfigPayload: @strawberry.type class AnnotationConfigMutationMixin: - @strawberry.mutation + @strawberry.mutation(permission_classes=[IsNotReadOnly]) # type: ignore[misc] async def create_categorical_annotation_config( self, info: Info[Context, None], @@ -183,7 +184,7 @@ async def create_categorical_annotation_config( annotation_config=to_gql_categorical_annotation_config(annotation_config), ) - @strawberry.mutation + @strawberry.mutation(permission_classes=[IsNotReadOnly]) # type: ignore[misc] async def create_continuous_annotation_config( self, info: Info[Context, None], @@ -211,7 +212,7 @@ async def create_continuous_annotation_config( annotation_config=to_gql_continuous_annotation_config(annotation_config), ) - @strawberry.mutation + @strawberry.mutation(permission_classes=[IsNotReadOnly]) # type: ignore[misc] async def create_freeform_annotation_config( self, info: Info[Context, None], @@ -233,7 +234,7 @@ async def create_freeform_annotation_config( annotation_config=to_gql_freeform_annotation_config(config), ) - @strawberry.mutation + @strawberry.mutation(permission_classes=[IsNotReadOnly]) # type: ignore[misc] async def update_categorical_annotation_config( self, info: Info[Context, None], @@ -290,7 +291,7 @@ async def update_categorical_annotation_config( annotation_config=to_gql_categorical_annotation_config(annotation_config), ) - @strawberry.mutation + @strawberry.mutation(permission_classes=[IsNotReadOnly]) # type: ignore[misc] async def update_continuous_annotation_config( self, info: Info[Context, None], @@ -329,7 +330,7 @@ async def update_continuous_annotation_config( annotation_config=to_gql_continuous_annotation_config(annotation_config), ) - @strawberry.mutation + @strawberry.mutation(permission_classes=[IsNotReadOnly]) # type: ignore[misc] async def update_freeform_annotation_config( self, info: Info[Context, None], @@ -359,7 +360,7 @@ async def update_freeform_annotation_config( annotation_config=to_gql_freeform_annotation_config(annotation_config), ) - @strawberry.mutation + @strawberry.mutation(permission_classes=[IsNotReadOnly]) # type: ignore[misc] async def delete_annotation_config( self, info: Info[Context, None], @@ -389,7 +390,7 @@ async def delete_annotation_config( annotation_config=to_gql_annotation_config(annotation_config), ) - @strawberry.mutation + @strawberry.mutation(permission_classes=[IsNotReadOnly]) # type: ignore[misc] async def add_annotation_config_to_project( self, info: Info[Context, None], From 507bbf09ddd7145f4c7e758317a421547c07a58b Mon Sep 17 00:00:00 2001 From: Anthony Powell Date: Mon, 24 Mar 2025 16:35:32 -0400 Subject: [PATCH 05/53] feat(annotations): Annotation Config UI (#6856) * feat(annotations): Annotation Config UI * Fix style merging in AnnotationLabel * Handle large values * Rename card * Persist annotation config via gql * Convert annotation config popover to dialog * Implement annotation selection toolbar + deletion * Remove bad lint rule * Replace dialog extra buttons with cancel button * styling changes * clean up annotation label * Remove hover state from annotation label unless clickable --- app/src/Routes.tsx | 10 + .../components/annotation/AnnotationLabel.tsx | 43 +- app/src/components/radio/RadioGroup.tsx | 29 +- .../pages/settings/AnnotationConfigDialog.tsx | 447 ++++++++++++++++++ .../AnnotationConfigSelectionToolbar.tsx | 162 +++++++ .../pages/settings/AnnotationConfigTable.tsx | 326 +++++++++++++ .../settings/SettingsAnnotationsPage.tsx | 251 ++++++++++ app/src/pages/settings/SettingsPage.tsx | 25 +- .../AnnotationConfigTableFragment.graphql.ts | 197 ++++++++ ...goricalAnnotationConfigMutation.graphql.ts | 113 +++++ ...tinuousAnnotationConfigMutation.graphql.ts | 110 +++++ ...reeformAnnotationConfigMutation.graphql.ts | 106 +++++ ...eDeleteAnnotationConfigMutation.graphql.ts | 144 ++++++ ...SettingsAnnotationsPageFragment.graphql.ts | 40 ++ ...goricalAnnotationConfigMutation.graphql.ts | 114 +++++ ...tinuousAnnotationConfigMutation.graphql.ts | 111 +++++ ...reeformAnnotationConfigMutation.graphql.ts | 107 +++++ ...tingsAnnotationsPageLoaderQuery.graphql.ts | 217 +++++++++ .../settings/settingsAnnotationsPageLoader.ts | 21 + app/src/pages/settings/types.ts | 42 ++ 20 files changed, 2577 insertions(+), 38 deletions(-) create mode 100644 app/src/pages/settings/AnnotationConfigDialog.tsx create mode 100644 app/src/pages/settings/AnnotationConfigSelectionToolbar.tsx create mode 100644 app/src/pages/settings/AnnotationConfigTable.tsx create mode 100644 app/src/pages/settings/SettingsAnnotationsPage.tsx create mode 100644 app/src/pages/settings/__generated__/AnnotationConfigTableFragment.graphql.ts create mode 100644 app/src/pages/settings/__generated__/SettingsAnnotationsPageCreateCategoricalAnnotationConfigMutation.graphql.ts create mode 100644 app/src/pages/settings/__generated__/SettingsAnnotationsPageCreateContinuousAnnotationConfigMutation.graphql.ts create mode 100644 app/src/pages/settings/__generated__/SettingsAnnotationsPageCreateFreeformAnnotationConfigMutation.graphql.ts create mode 100644 app/src/pages/settings/__generated__/SettingsAnnotationsPageDeleteAnnotationConfigMutation.graphql.ts create mode 100644 app/src/pages/settings/__generated__/SettingsAnnotationsPageFragment.graphql.ts create mode 100644 app/src/pages/settings/__generated__/SettingsAnnotationsPageUpdateCategoricalAnnotationConfigMutation.graphql.ts create mode 100644 app/src/pages/settings/__generated__/SettingsAnnotationsPageUpdateContinuousAnnotationConfigMutation.graphql.ts create mode 100644 app/src/pages/settings/__generated__/SettingsAnnotationsPageUpdateFreeformAnnotationConfigMutation.graphql.ts create mode 100644 app/src/pages/settings/__generated__/settingsAnnotationsPageLoaderQuery.graphql.ts create mode 100644 app/src/pages/settings/settingsAnnotationsPageLoader.ts create mode 100644 app/src/pages/settings/types.ts diff --git a/app/src/Routes.tsx b/app/src/Routes.tsx index b6cacc0a84..e31ec61c02 100644 --- a/app/src/Routes.tsx +++ b/app/src/Routes.tsx @@ -9,6 +9,8 @@ import { RouterProvider } from "react-router/dom"; import { SettingsAIProvidersPage } from "@phoenix/pages/settings/SettingsAIProvidersPage"; import { settingsAIProvidersPageLoader } from "@phoenix/pages/settings/settingsAIProvidersPageLoader"; +import { SettingsAnnotationsPage } from "@phoenix/pages/settings/SettingsAnnotationsPage"; +import { settingsAnnotationsPageLoader } from "@phoenix/pages/settings/settingsAnnotationsPageLoader"; import { SettingsGeneralPage } from "@phoenix/pages/settings/SettingsGeneralPage"; import { datasetLoaderQuery$data } from "./pages/dataset/__generated__/datasetLoaderQuery.graphql"; @@ -320,6 +322,14 @@ const router = createBrowserRouter( crumb: () => "providers", }} /> + } + handle={{ + crumb: () => "annotations", + }} + /> diff --git a/app/src/components/annotation/AnnotationLabel.tsx b/app/src/components/annotation/AnnotationLabel.tsx index 28f3d3d4e0..f20fa2f82f 100644 --- a/app/src/components/annotation/AnnotationLabel.tsx +++ b/app/src/components/annotation/AnnotationLabel.tsx @@ -8,7 +8,7 @@ import { formatFloat } from "@phoenix/utils/numberFormatUtils"; import { AnnotationColorSwatch } from "./AnnotationColorSwatch"; import { Annotation } from "./types"; -type AnnotationDisplayPreference = "label" | "score"; +type AnnotationDisplayPreference = "label" | "score" | "none"; export const baseAnnotationLabelCSS = css` border-radius: var(--ac-global-dimension-size-50); @@ -16,9 +16,13 @@ export const baseAnnotationLabelCSS = css` padding: var(--ac-global-dimension-size-50) var(--ac-global-dimension-size-100); transition: background-color 0.2s; - &:hover { - background-color: var(--ac-global-color-grey-300); + &[data-clickable="true"] { + cursor: pointer; + &:hover { + background-color: var(--ac-global-color-grey-300); + } } + .ac-icon-wrap { font-size: 12px; } @@ -55,6 +59,8 @@ const getAnnotationDisplayValue = ( annotation.label || "n/a" ); + case "none": + return ""; default: assertUnreachable(displayPreference); } @@ -64,15 +70,20 @@ export function AnnotationLabel({ annotation, onClick, annotationDisplayPreference = "score", + className, }: { annotation: Annotation; onClick?: () => void; /** * The preferred value to display in the annotation label. * If the provided value is not available, it will fallback to an available value. + * - "label": Display the annotation label. + * - "score": Display the annotation score. + * - "none": Do not display the annotation label or score. * @default "score" */ annotationDisplayPreference?: AnnotationDisplayPreference; + className?: string; }) { const clickable = typeof onClick == "function"; const labelValue = getAnnotationDisplayValue( @@ -83,7 +94,9 @@ export function AnnotationLabel({ return (
-
- {labelValue} -
+ {labelValue && ( +
+ {labelValue} +
+ )} {clickable ? } /> : null} diff --git a/app/src/components/radio/RadioGroup.tsx b/app/src/components/radio/RadioGroup.tsx index 560d16ee46..bf6d40ff4b 100644 --- a/app/src/components/radio/RadioGroup.tsx +++ b/app/src/components/radio/RadioGroup.tsx @@ -6,6 +6,7 @@ import { import { css } from "@emotion/react"; import { classNames } from "@phoenix/components"; +import { fieldBaseCSS } from "@phoenix/components/field/styles"; import { SizingProps, StylableProps } from "@phoenix/components/types"; const baseRadioGroupCSS = css(` @@ -33,8 +34,22 @@ const baseRadioGroupCSS = css(` border-radius: 0 var(--ac-global-rounding-small) var(--ac-global-rounding-small) 0; } + &[data-direction="row"] { + flex-direction: row; + flex-wrap: wrap; + + .react-aria-Label { + flex-basis: 100%; + } + + [slot="description"] { + flex-basis: 100%; + } + } + &[data-direction="column"] { flex-direction: column; + align-items: flex-start; } &[data-size="S"] { @@ -49,6 +64,16 @@ const baseRadioGroupCSS = css(` } } + &[data-disabled] { + opacity: 0.5; + } + + &[data-readonly] { + .ac-radio:before { + opacity: 0.5; + } + } + &:has(.ac-radio[data-focus-visible]) { border-radius: var(--ac-global-rounding-small); outline: 1px solid var(--ac-global-input-field-border-color-active); @@ -63,7 +88,7 @@ export const RadioGroup = ({ size, css: cssProp, className, - direction, + direction = "row", ...props }: RadioGroupProps & SizingProps & @@ -73,7 +98,7 @@ export const RadioGroup = ({ data-size={size} data-direction={direction} className={classNames("ac-radio-group", className)} - css={css(baseRadioGroupCSS, cssProp)} + css={css(fieldBaseCSS, baseRadioGroupCSS, cssProp)} {...props} /> ); diff --git a/app/src/pages/settings/AnnotationConfigDialog.tsx b/app/src/pages/settings/AnnotationConfigDialog.tsx new file mode 100644 index 0000000000..39aae0d61b --- /dev/null +++ b/app/src/pages/settings/AnnotationConfigDialog.tsx @@ -0,0 +1,447 @@ +import React from "react"; +import { Controller, useFieldArray, useForm } from "react-hook-form"; +import { css } from "@emotion/react"; + +import { Card } from "@arizeai/components"; + +import { + Button, + Dialog, + FieldError, + Flex, + Form, + Icon, + Icons, + Input, + Label, + NumberField, + Radio, + RadioGroup, + Text, + TextArea, + TextField, + View, +} from "@phoenix/components"; +import { useNotifyError, useNotifySuccess } from "@phoenix/contexts"; + +import { + AnnotationConfig, + AnnotationConfigCategorical, + AnnotationConfigContinuous, + AnnotationConfigFreeform, + AnnotationConfigOptimizationDirection, + AnnotationConfigType, +} from "./types"; + +const ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".split(""); + +const optimizationDirections = [ + "MAXIMIZE", + "MINIMIZE", +] satisfies AnnotationConfigOptimizationDirection[]; + +const types = [ + "CATEGORICAL", + "CONTINUOUS", + "FREEFORM", +] satisfies AnnotationConfigType[]; + +export const AnnotationConfigDialog = ({ + onAddAnnotationConfig, + initialAnnotationConfig, +}: { + onAddAnnotationConfig: ( + config: AnnotationConfig, + { + onCompleted, + onError, + }?: { onCompleted?: () => void; onError?: (error: string) => void } + ) => void; + initialAnnotationConfig?: Partial; +}) => { + const notifyError = useNotifyError(); + const notifySuccess = useNotifySuccess(); + const mode: "new" | "edit" = initialAnnotationConfig ? "edit" : "new"; + const { control, handleSubmit, watch } = useForm({ + defaultValues: initialAnnotationConfig || { + annotationType: "CATEGORICAL", + values: [{ label: "", score: 0 }], + optimizationDirection: "MAXIMIZE", + }, + }); + const { fields, append, remove } = useFieldArray({ + control, + name: "values", + }); + const onSubmit = (data: AnnotationConfig, close: () => void) => { + const onCompleted = () => { + notifySuccess({ + title: + mode === "new" + ? "Annotation config created!" + : "Annotation config updated!", + }); + close(); + }; + const onError = (error: string) => { + notifyError({ + title: + mode === "new" + ? "Failed to create annotation config" + : "Failed to update annotation config", + message: error, + }); + }; + switch (data.annotationType) { + case "CATEGORICAL": { + const config: AnnotationConfigCategorical = { + annotationType: "CATEGORICAL", + name: data.name, + values: data.values, + id: initialAnnotationConfig?.id || "", + optimizationDirection: data.optimizationDirection, + description: data.description, + }; + onAddAnnotationConfig(config, { onCompleted, onError }); + break; + } + case "CONTINUOUS": { + const config: AnnotationConfigContinuous = { + annotationType: "CONTINUOUS", + name: data.name, + lowerBound: data.lowerBound, + upperBound: data.upperBound, + id: initialAnnotationConfig?.id || "", + optimizationDirection: data.optimizationDirection, + description: data.description, + }; + onAddAnnotationConfig(config, { onCompleted, onError }); + break; + } + case "FREEFORM": { + const config: AnnotationConfigFreeform = { + annotationType: "FREEFORM", + name: data.name, + id: initialAnnotationConfig?.id || "", + description: data.description, + }; + onAddAnnotationConfig(config, { onCompleted, onError }); + break; + } + } + }; + const annotationType = watch("annotationType"); + return ( + + {({ close }) => ( + +
{ + handleSubmit((data) => { + onSubmit(data, close); + })(e); + }} + > + + + ( + + + + {error?.message} + + )} + /> + ( + + +