Skip to content

Commit c976bf6

Browse files
kiranandcodekiranandcode
authored
Drop cacheLLMhandler in favour of LiteLLM's built in caching mechanisms. (#510)
* drop cacheLLMhandler and update docs and add tests * added fixtures
1 parent 6fec67e commit c976bf6

14 files changed

Lines changed: 553 additions & 60 deletions

‎docs/source/llm.ipynb‎

Lines changed: 26 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": null,
5+
"execution_count": 1,
66
"id": "5aaf649f",
77
"metadata": {},
88
"outputs": [],
@@ -13,13 +13,13 @@
1313
"import sys\n",
1414
"\n",
1515
"import dotenv\n",
16+
"import litellm\n",
1617
"import pydantic\n",
1718
"from pydantic import ValidationError, field_validator\n",
1819
"from pydantic_core import PydanticCustomError\n",
1920
"\n",
2021
"from effectful.handlers.llm import Template, Tool\n",
2122
"from effectful.handlers.llm.completions import (\n",
22-
" CacheLLMRequestHandler,\n",
2323
" LiteLLMProvider,\n",
2424
" LLMLoggingHandler,\n",
2525
" RetryLLMHandler,\n",
@@ -129,33 +129,29 @@
129129
"output_type": "stream",
130130
"text": [
131131
"\n",
132-
"Here is a haiku on the theme of fish:\n",
133-
"\n",
134-
"Silent stream below, \n",
135-
"Golden fins dart through sun's glow, \n",
136-
"Still waters, life flows.\n",
132+
"Gentle fins flutter, \n",
133+
"In the depths of clear blue seas, \n",
134+
"Silent dancers swim.\n",
137135
"----------------------------------------\n",
138-
"Here is a haiku on the theme of fish:\n",
139-
"\n",
140-
"Silent stream below, \n",
141-
"Golden fins dart through sun's glow, \n",
142-
"Still waters, life flows.\n",
136+
"Gentle fins flutter, \n",
137+
"In the depths of clear blue seas, \n",
138+
"Silent dancers swim.\n",
143139
"\n",
144-
"Swimming silently, \n",
145-
"Scales shimmer in liquid light, \n",
146-
"Depths hold their secrets.\n",
140+
"Swimming in silence, \n",
141+
"Beneath the rippling waves' veil, \n",
142+
"Whispers of the deep.\n",
147143
"----------------------------------------\n",
148-
"Swimming silently, \n",
149-
"Scales shimmer in liquid light, \n",
150-
"Depths hold their secrets.\n",
144+
"Swimming in silence, \n",
145+
"Beneath the rippling waves' veil, \n",
146+
"Whispers of the deep.\n",
151147
"\n",
152-
"In oceans they glide, \n",
153-
"Silent beneath waves they swim, \n",
154-
"Fish dance with the tides.\n",
148+
"In the silent pond, \n",
149+
"Silver scales glint in moonlight, \n",
150+
"Dreams of deep blue sea. \n",
155151
"----------------------------------------\n",
156-
"In oceans they glide, \n",
157-
"Silent beneath waves they swim, \n",
158-
"Fish dance with the tides.\n"
152+
"Silent ponds shimmer, \n",
153+
"Fish glide through liquid silver, \n",
154+
"Nature's quiet dance.\n"
159155
]
160156
}
161157
],
@@ -180,15 +176,15 @@
180176
" print(haiku(\"fish\"))\n",
181177
"\n",
182178
"print()\n",
183-
"cache_handler1 = CacheLLMRequestHandler()\n",
184-
"with handler(provider), handler(cache_handler1):\n",
179+
"with handler(provider):\n",
180+
" litellm.enable_cache()\n",
185181
" print(haiku_no_cache(\"fish2\"))\n",
186182
" print(\"-\" * 40)\n",
187183
" print(haiku_no_cache(\"fish2\"))\n",
184+
" litellm.disable_cache()\n",
188185
"\n",
189186
"print()\n",
190-
"cache_handler2 = CacheLLMRequestHandler()\n",
191-
"with handler(provider), handler(cache_handler2):\n",
187+
"with handler(provider):\n",
192188
" print(haiku_no_cache(\"fish3\"))\n",
193189
" print(\"-\" * 40)\n",
194190
" print(haiku_no_cache(\"fish3\"))"
@@ -856,7 +852,7 @@
856852
],
857853
"metadata": {
858854
"kernelspec": {
859-
"display_name": ".venv",
855+
"display_name": "Python 3 (ipykernel)",
860856
"language": "python",
861857
"name": "python3"
862858
},
@@ -870,7 +866,7 @@
870866
"name": "python",
871867
"nbconvert_exporter": "python",
872868
"pygments_lexer": "ipython3",
873-
"version": "3.12.11"
869+
"version": "3.13.9"
874870
}
875871
},
876872
"nbformat": 4,

‎effectful/handlers/llm/completions.py‎

Lines changed: 1 addition & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import string
66
import traceback
77
import typing
8-
from collections.abc import Callable, Hashable, Mapping
8+
from collections.abc import Callable, Mapping
99
from typing import Any
1010

1111
import litellm
@@ -77,34 +77,6 @@ def completion(*args, **kwargs) -> Any:
7777
return litellm.completion(*args, **kwargs)
7878

7979

80-
class CacheLLMRequestHandler(ObjectInterpretation):
81-
"""Caches LLM requests."""
82-
83-
def __init__(self):
84-
self.cache: dict[Hashable, Any] = {}
85-
86-
def _make_hashable(self, obj: Any) -> Hashable:
87-
"""Recursively convert objects to hashable representations."""
88-
if isinstance(obj, dict):
89-
return tuple(sorted((k, self._make_hashable(v)) for k, v in obj.items()))
90-
elif isinstance(obj, list | tuple):
91-
return tuple(self._make_hashable(item) for item in obj)
92-
elif isinstance(obj, set):
93-
return frozenset(self._make_hashable(item) for item in obj)
94-
else:
95-
# Primitives (int, float, str, bytes, etc.) are already hashable
96-
return obj
97-
98-
@implements(completion)
99-
def _cache_completion(self, *args, **kwargs) -> Any:
100-
key = self._make_hashable((args, kwargs))
101-
if key in self.cache:
102-
return self.cache[key]
103-
response = fwd()
104-
self.cache[key] = response
105-
return response
106-
107-
10880
class LLMLoggingHandler(ObjectInterpretation):
10981
"""Logs completion rounds and tool_call invocations using Python logging.
11082

‎tests/fixtures/tests_test_handlers_llm_provider.py__test_litellm_caching_integration.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"choices": [
3+
{
4+
"finish_reason": "stop",
5+
"index": 0,
6+
"message": {
7+
"annotations": [],
8+
"content": "Apples are crisp, sweet fruits that come in a variety of colors, including red, green, and yellow.",
9+
"function_call": null,
10+
"provider_specific_fields": {
11+
"refusal": null
12+
},
13+
"role": "assistant",
14+
"tool_calls": null
15+
},
16+
"provider_specific_fields": {}
17+
}
18+
],
19+
"created": 1769462344,
20+
"id": "chatcmpl-D2OHAYV2epNFazAphxZKLpJglzrCc",
21+
"model": "gpt-4o-2024-08-06",
22+
"object": "chat.completion",
23+
"service_tier": "default",
24+
"system_fingerprint": "fp_deacdd5f6f",
25+
"usage": {
26+
"completion_tokens": 24,
27+
"completion_tokens_details": {
28+
"accepted_prediction_tokens": 0,
29+
"audio_tokens": 0,
30+
"image_tokens": null,
31+
"reasoning_tokens": 0,
32+
"rejected_prediction_tokens": 0,
33+
"text_tokens": null
34+
},
35+
"prompt_tokens": 232,
36+
"prompt_tokens_details": {
37+
"audio_tokens": 0,
38+
"cached_tokens": 0,
39+
"image_tokens": null,
40+
"text_tokens": null
41+
},
42+
"total_tokens": 256
43+
}
44+
}

‎tests/fixtures/tests_test_handlers_llm_provider.py__test_litellm_caching_integration_1.json‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{
2+
"choices": [
3+
{
4+
"finish_reason": "stop",
5+
"index": 0,
6+
"message": {
7+
"annotations": [],
8+
"content": "Apples are crisp, sweet fruits that come in a variety of colors, including red, green, and yellow.",
9+
"function_call": null,
10+
"provider_specific_fields": null,
11+
"role": "assistant",
12+
"tool_calls": null
13+
},
14+
"provider_specific_fields": {}
15+
}
16+
],
17+
"created": 1769462344,
18+
"id": "chatcmpl-D2OHAYV2epNFazAphxZKLpJglzrCc",
19+
"model": "gpt-4o-2024-08-06",
20+
"object": "chat.completion",
21+
"service_tier": "default",
22+
"system_fingerprint": "fp_deacdd5f6f",
23+
"usage": {
24+
"completion_tokens": 24,
25+
"completion_tokens_details": {
26+
"accepted_prediction_tokens": 0,
27+
"audio_tokens": 0,
28+
"image_tokens": null,
29+
"reasoning_tokens": 0,
30+
"rejected_prediction_tokens": 0,
31+
"text_tokens": null
32+
},
33+
"prompt_tokens": 232,
34+
"prompt_tokens_details": {
35+
"audio_tokens": 0,
36+
"cached_tokens": 0,
37+
"image_tokens": null,
38+
"text_tokens": null
39+
},
40+
"total_tokens": 256
41+
}
42+
}

‎tests/fixtures/tests_test_handlers_llm_provider.py__test_litellm_caching_integration_2.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"choices": [
3+
{
4+
"finish_reason": "stop",
5+
"index": 0,
6+
"message": {
7+
"annotations": [],
8+
"content": "Oranges are a juicy and sweet citrus fruit, rich in vitamin C and enjoyed worldwide as a snack or juice.",
9+
"function_call": null,
10+
"provider_specific_fields": {
11+
"refusal": null
12+
},
13+
"role": "assistant",
14+
"tool_calls": null
15+
},
16+
"provider_specific_fields": {}
17+
}
18+
],
19+
"created": 1769462345,
20+
"id": "chatcmpl-D2OHBpQKOkF5lxOZcOPvnuno9XCZS",
21+
"model": "gpt-4o-2024-08-06",
22+
"object": "chat.completion",
23+
"service_tier": "default",
24+
"system_fingerprint": "fp_deacdd5f6f",
25+
"usage": {
26+
"completion_tokens": 24,
27+
"completion_tokens_details": {
28+
"accepted_prediction_tokens": 0,
29+
"audio_tokens": 0,
30+
"image_tokens": null,
31+
"reasoning_tokens": 0,
32+
"rejected_prediction_tokens": 0,
33+
"text_tokens": null
34+
},
35+
"prompt_tokens": 232,
36+
"prompt_tokens_details": {
37+
"audio_tokens": 0,
38+
"cached_tokens": 0,
39+
"image_tokens": null,
40+
"text_tokens": null
41+
},
42+
"total_tokens": 256
43+
}
44+
}

‎tests/fixtures/tests_test_handlers_llm_provider.py__test_litellm_caching_integration_disabled.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"choices": [
3+
{
4+
"finish_reason": "stop",
5+
"index": 0,
6+
"message": {
7+
"annotations": [],
8+
"content": "Apples are a crisp and juicy fruit that come in a variety of flavors and colors, from sweet to tart.",
9+
"function_call": null,
10+
"provider_specific_fields": {
11+
"refusal": null
12+
},
13+
"role": "assistant",
14+
"tool_calls": null
15+
},
16+
"provider_specific_fields": {}
17+
}
18+
],
19+
"created": 1769462584,
20+
"id": "chatcmpl-D2OL2w9GHCV8jAbzqup98G9KU6yRq",
21+
"model": "gpt-4o-2024-08-06",
22+
"object": "chat.completion",
23+
"service_tier": "default",
24+
"system_fingerprint": "fp_deacdd5f6f",
25+
"usage": {
26+
"completion_tokens": 24,
27+
"completion_tokens_details": {
28+
"accepted_prediction_tokens": 0,
29+
"audio_tokens": 0,
30+
"image_tokens": null,
31+
"reasoning_tokens": 0,
32+
"rejected_prediction_tokens": 0,
33+
"text_tokens": null
34+
},
35+
"prompt_tokens": 232,
36+
"prompt_tokens_details": {
37+
"audio_tokens": 0,
38+
"cached_tokens": 0,
39+
"image_tokens": null,
40+
"text_tokens": null
41+
},
42+
"total_tokens": 256
43+
}
44+
}

‎tests/fixtures/tests_test_handlers_llm_provider.py__test_litellm_caching_integration_disabled_1.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"choices": [
3+
{
4+
"finish_reason": "stop",
5+
"index": 0,
6+
"message": {
7+
"annotations": [],
8+
"content": "Apples are crisp, sweet fruits that come in a variety of colors and flavors, making them a popular and healthy snack worldwide.",
9+
"function_call": null,
10+
"provider_specific_fields": {
11+
"refusal": null
12+
},
13+
"role": "assistant",
14+
"tool_calls": null
15+
},
16+
"provider_specific_fields": {}
17+
}
18+
],
19+
"created": 1769462586,
20+
"id": "chatcmpl-D2OL4MbMdKpmNB8y9bjtECBqAgZzX",
21+
"model": "gpt-4o-2024-08-06",
22+
"object": "chat.completion",
23+
"service_tier": "default",
24+
"system_fingerprint": "fp_deacdd5f6f",
25+
"usage": {
26+
"completion_tokens": 27,
27+
"completion_tokens_details": {
28+
"accepted_prediction_tokens": 0,
29+
"audio_tokens": 0,
30+
"image_tokens": null,
31+
"reasoning_tokens": 0,
32+
"rejected_prediction_tokens": 0,
33+
"text_tokens": null
34+
},
35+
"prompt_tokens": 232,
36+
"prompt_tokens_details": {
37+
"audio_tokens": 0,
38+
"cached_tokens": 0,
39+
"image_tokens": null,
40+
"text_tokens": null
41+
},
42+
"total_tokens": 259
43+
}
44+
}

0 commit comments

Comments
 (0)