Skip to content

Commit c72311f

Browse files
freddiev4freddiev4
committed
Clear our non log & detect
1 parent 6526cd1 commit c72311f

18 files changed

+3
-4782
lines changed

‎quotientai/async_client.py‎

Lines changed: 0 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,7 @@
1111
from quotientai import resources
1212
from quotientai.exceptions import handle_async_errors, logger
1313
from quotientai.resources.auth import AsyncAuthResource
14-
from quotientai.resources.prompts import Prompt
1514
from quotientai.resources.logs import LogDocument
16-
from quotientai.resources.models import Model
17-
from quotientai.resources.datasets import Dataset
18-
from quotientai.resources.runs import Run
1915

2016

2117
class _AsyncQuotientClient(httpx.AsyncClient):
@@ -363,11 +359,6 @@ def __init__(self, api_key: Optional[str] = None):
363359

364360
self._client = _AsyncQuotientClient(self.api_key)
365361
self.auth = AsyncAuthResource(self._client)
366-
self.prompts = resources.AsyncPromptsResource(self._client)
367-
self.datasets = resources.AsyncDatasetsResource(self._client)
368-
self.models = resources.AsyncModelsResource(self._client)
369-
self.runs = resources.AsyncRunsResource(self._client)
370-
self.metrics = resources.AsyncMetricsResource(self._client)
371362
self.logs = resources.AsyncLogsResource(self._client)
372363

373364
# Create an unconfigured logger instance.
@@ -382,51 +373,3 @@ def __init__(self, api_key: Optional[str] = None):
382373
)
383374
return None
384375

385-
async def evaluate(
386-
self,
387-
*,
388-
prompt: Prompt,
389-
dataset: Dataset,
390-
model: Model,
391-
parameters: dict,
392-
metrics: List[str],
393-
) -> Run:
394-
def _validate_parameters(parameters):
395-
"""
396-
Validate the parameters dictionary. Currently the only valid parameters are:
397-
398-
- temperature: float
399-
- top_k: int
400-
- top_p: float
401-
- max_tokens: int
402-
- repetition_penalty: float
403-
"""
404-
valid_parameters = [
405-
"temperature",
406-
"top_k",
407-
"top_p",
408-
"max_tokens",
409-
]
410-
411-
invalid_parameters = set(parameters.keys()) - set(valid_parameters)
412-
if invalid_parameters:
413-
logger.error(
414-
f"invalid parameters: {', '.join(invalid_parameters)}. \nvalid parameters are: {', '.join(valid_parameters)}"
415-
)
416-
return None
417-
418-
return parameters
419-
420-
v_parameters = _validate_parameters(parameters)
421-
422-
if v_parameters is None:
423-
return None
424-
425-
run = await self.runs.create(
426-
prompt=prompt,
427-
dataset=dataset,
428-
model=model,
429-
parameters=v_parameters,
430-
metrics=metrics,
431-
)
432-
return run

‎quotientai/cli/entrypoint.py‎

Lines changed: 1 addition & 176 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
console = Console()
1919

2020
app = typer.Typer(
21-
help="[green]Quotient CLI tool for managing artifacts and running evaluations[/green]",
21+
help="[green]Quotient CLI tool for managing artifacts.[/green]",
2222
rich_markup_mode="rich",
2323
no_args_is_help=True,
2424
pretty_exceptions_show_locals=False,
@@ -30,181 +30,6 @@
3030

3131
app.add_typer(list_app, name="list")
3232

33-
###########################
34-
# Models #
35-
###########################
36-
37-
38-
@list_app.command(name="models")
39-
def list_models():
40-
"""Command to get all models with optional filters."""
41-
try:
42-
quotient = QuotientAI()
43-
models = quotient.models.list()
44-
console.print(models)
45-
except QuotientAIError as e:
46-
click.echo(str(e))
47-
48-
49-
###########################
50-
# Prompts #
51-
###########################
52-
53-
54-
@list_app.command(name="prompts")
55-
def list_prompts():
56-
"""Command to get all prompts."""
57-
try:
58-
quotient = QuotientAI()
59-
prompts = quotient.prompts.list()
60-
console.print(prompts)
61-
except QuotientAIError as e:
62-
click.echo(str(e))
63-
64-
65-
###########################
66-
# Datasets #
67-
###########################
68-
69-
70-
@list_app.command(name="datasets")
71-
def list_datasets():
72-
"""Command to get all datasets."""
73-
try:
74-
quotient = QuotientAI()
75-
datasets = quotient.datasets.list()
76-
console.print(datasets)
77-
except QuotientAIError as e:
78-
click.echo(str(e))
79-
80-
81-
###############
82-
# Evaluations #
83-
###############
84-
85-
@app.command(name="run")
86-
def run_evaluation(
87-
file: Path = typer.Argument(
88-
...,
89-
exists=True,
90-
file_okay=True,
91-
dir_okay=True,
92-
help="Path to the evaluation file or directory to search in",
93-
),
94-
):
95-
"""Command to run an evaluation"""
96-
try:
97-
# show an initial progress bar to indicate that we're kicking things off
98-
with Live(refresh_per_second=4) as live:
99-
quotient = QuotientAI()
100-
101-
# Create a progress display focused on status rather than percentage
102-
status_progress = Progress(
103-
SpinnerColumn(),
104-
TextColumn("[progress.description]{task.description}"),
105-
TimeElapsedColumn(),
106-
)
107-
108-
task = status_progress.add_task("Starting...", total=None) # No total needed
109-
110-
panel_content = Panel(
111-
"Initializing evaluation. Hold tight 🚀...",
112-
title="Evaluation In Progress",
113-
subtitle="QuotientAI",
114-
expand=False,
115-
)
116-
117-
layout = Group(panel_content, status_progress)
118-
live.update(layout)
119-
120-
# Kick off the actual evaluation
121-
run = exec_evaluate(file)
122-
# Actually fetch the current status from the run
123-
current_status = run.status
124-
125-
while True: # pragma: no cover
126-
# testing these three conditions is NIGHTMARE NITGHTMARE NITGHTMARE
127-
if current_status == "not-started": # pragma: no cover
128-
status_desc = "Initializing evaluation..."
129-
style = "yellow"
130-
elif current_status == "running": # pragma: no cover
131-
status_desc = "Processing evaluation..."
132-
style = "blue"
133-
elif current_status == "completed": # pragma: no cover
134-
status_desc = "Evaluation complete!"
135-
style = "green"
136-
break
137-
138-
status_progress.update(task, description=status_desc)
139-
panel_content = Panel(
140-
f"[{style}]{status_desc}[/{style}]\n\n"
141-
f"Run ID: {run.id}\n"
142-
f"Status: {current_status}",
143-
title="Evaluation In Progress",
144-
subtitle="QuotientAI",
145-
expand=False,
146-
)
147-
148-
layout = Group(panel_content, status_progress)
149-
live.update(layout)
150-
151-
# Fetch the current status from the run
152-
time.sleep(1)
153-
run = quotient.runs.get(run.id)
154-
current_status = run.status
155-
156-
# Generate the summary after completion
157-
summary = run.summarize()
158-
console.print()
159-
console.print("Evaluation complete!")
160-
console.print()
161-
162-
# turn the summary into a table and print it
163-
# print the metadata of the run
164-
table = Table(title=f"Run: {summary['run_id']}", title_justify='left')
165-
table.add_column("Key")
166-
table.add_column("Value")
167-
table.add_row("Run ID", summary["run_id"])
168-
table.add_row("Model", summary["model"]["name"])
169-
table.add_row("Parameters", str(summary["parameters"]))
170-
table.add_row("Created At", str(summary["created_at"]))
171-
console.print(table)
172-
173-
table = Table(title="Evaluation Summary", title_justify='left')
174-
table.add_column("Metric")
175-
table.add_column("Average")
176-
table.add_column("Std Dev")
177-
for metric, values in summary["metrics"].items():
178-
table.add_row(metric, str(values["avg"]), str(values["stddev"]))
179-
180-
console.print(table)
181-
182-
except QuotientAIError as e:
183-
raise
184-
185-
186-
@list_app.command(name="runs")
187-
def list_runs():
188-
"""Command to get all runs."""
189-
try:
190-
quotient = QuotientAI()
191-
runs = quotient.runs.list()
192-
console.print(runs)
193-
except QuotientAIError as e:
194-
raise
195-
196-
197-
@list_app.command(name="metrics")
198-
def list_metrics():
199-
"""Command to get all available metrics."""
200-
try:
201-
quotient = QuotientAI()
202-
response = quotient.metrics.list()
203-
console.print(response)
204-
except QuotientAIError as e:
205-
raise
206-
207-
20833
MAX_LOGS = 10
20934

21035
@list_app.command(name="logs")

‎quotientai/client.py‎

Lines changed: 0 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,6 @@
1212
from quotientai import resources
1313
from quotientai.exceptions import handle_errors, logger
1414
from quotientai.resources.logs import LogDocument
15-
from quotientai.resources.prompts import Prompt
16-
from quotientai.resources.models import Model
17-
from quotientai.resources.datasets import Dataset
18-
from quotientai.resources.runs import Run
1915
from quotientai.resources.auth import AuthResource
2016
from pathlib import Path
2117

@@ -361,11 +357,6 @@ def __init__(self, api_key: Optional[str] = None):
361357

362358
_client = _BaseQuotientClient(self.api_key)
363359
self.auth = AuthResource(_client)
364-
self.prompts = resources.PromptsResource(_client)
365-
self.datasets = resources.DatasetsResource(_client)
366-
self.models = resources.ModelsResource(_client)
367-
self.runs = resources.RunsResource(_client)
368-
self.metrics = resources.MetricsResource(_client)
369360
self.logs = resources.LogsResource(_client)
370361

371362
# Create an unconfigured logger instance.
@@ -379,52 +370,3 @@ def __init__(self, api_key: Optional[str] = None):
379370
f"If the issue persists, please contact support@quotientai.co\n{traceback.format_exc()}"
380371
)
381372
return None
382-
383-
def evaluate(
384-
self,
385-
*,
386-
prompt: Prompt,
387-
dataset: Dataset,
388-
model: Model,
389-
parameters: dict,
390-
metrics: List[str],
391-
) -> Run:
392-
def _validate_parameters(parameters):
393-
"""
394-
Validate the parameters dictionary. Currently the only valid parameters are:
395-
396-
- temperature: float
397-
- top_k: int
398-
- top_p: float
399-
- max_tokens: int
400-
- repetition_penalty: float
401-
"""
402-
valid_parameters = [
403-
"temperature",
404-
"top_k",
405-
"top_p",
406-
"max_tokens",
407-
]
408-
409-
invalid_parameters = set(parameters.keys()) - set(valid_parameters)
410-
if invalid_parameters:
411-
logger.error(
412-
f"invalid parameters: {', '.join(invalid_parameters)}. \nvalid parameters are: {', '.join(valid_parameters)}"
413-
)
414-
return None
415-
416-
return parameters
417-
418-
v_parameters = _validate_parameters(parameters)
419-
420-
if v_parameters is None:
421-
return None
422-
423-
run = self.runs.create(
424-
prompt=prompt,
425-
dataset=dataset,
426-
model=model,
427-
parameters=v_parameters,
428-
metrics=metrics,
429-
)
430-
return run

‎quotientai/resources/__init__.py‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,7 @@
1-
from quotientai.resources.prompts import PromptsResource
2-
from quotientai.resources.models import ModelsResource
3-
from quotientai.resources.datasets import DatasetsResource
4-
from quotientai.resources.runs import RunsResource
5-
from quotientai.resources.metrics import MetricsResource
61
from quotientai.resources.logs import LogsResource
7-
from quotientai.resources.prompts import AsyncPromptsResource
8-
from quotientai.resources.models import AsyncModelsResource
9-
from quotientai.resources.datasets import AsyncDatasetsResource
10-
from quotientai.resources.runs import AsyncRunsResource
11-
from quotientai.resources.metrics import AsyncMetricsResource
122
from quotientai.resources.logs import AsyncLogsResource
133

144
__all__ = [
15-
"PromptsResource",
16-
"ModelsResource",
17-
"DatasetsResource",
18-
"RunsResource",
19-
"MetricsResource",
205
"LogsResource",
21-
"AsyncPromptsResource",
22-
"AsyncModelsResource",
23-
"AsyncDatasetsResource",
24-
"AsyncRunsResource",
25-
"AsyncMetricsResource",
266
"AsyncLogsResource",
277
]

0 commit comments

Comments
 (0)