From 3986daf425515af49d6813770cfd939adc8806c5 Mon Sep 17 00:00:00 2001 From: Rohan Agarwal <47861399+roaga@users.noreply.github.com> Date: Thu, 6 Feb 2025 18:44:13 -0500 Subject: [PATCH] feat(autofix): Add prompt caching for Claude (#1884) Iteratively cache the last message in the conversation when using `AnthropicProvider`. Also cache the system prompt. From evals, seems to save an average of 15-20 seconds per run. It should also save us a lot of cost. --- requirements-constraints.txt | 2 +- requirements.txt | 3 +-- src/seer/automation/agent/client.py | 20 ++++++++++++------ tests/automation/agent/test_client.py | 2 +- ...ied[create_flaky_anthropic].yaml.encrypted | Bin 4679 -> 4757 bytes 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/requirements-constraints.txt b/requirements-constraints.txt index 4fff18586..fc91739d7 100644 --- a/requirements-constraints.txt +++ b/requirements-constraints.txt @@ -99,7 +99,7 @@ chromadb==0.4.14 google-cloud-storage==2.* google-cloud-aiplatform==1.* google-cloud-secret-manager==2.* -anthropic[vertex]==0.34.2 +anthropic[vertex]==0.45.* langfuse @ git+https://github.com/jennmueng/langfuse-python.git@9d9350de1e4e84fa548fe84f82c1b826be17956e watchdog stumpy==1.13.0 diff --git a/requirements.txt b/requirements.txt index f9e0fa6aa..115aab515 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ amqp==5.3.1 # via kombu annotated-types==0.7.0 # via pydantic -anthropic==0.34.2 +anthropic==0.45.2 # via -r requirements-constraints.txt anyio==4.8.0 # via @@ -729,7 +729,6 @@ threadpoolctl==3.2.0 # scikit-learn tokenizers==0.15.2 # via - # anthropic # chromadb # transformers torch==2.2.0 diff --git a/src/seer/automation/agent/client.py b/src/seer/automation/agent/client.py index 0086b89d5..dbdebb295 100644 --- a/src/seer/automation/agent/client.py +++ b/src/seer/automation/agent/client.py @@ -437,7 +437,7 @@ def generate_text( max_tokens: int | None = None, timeout: float | None = None, ): - message_dicts, tool_dicts, system_prompt = self._prep_message_and_tools( + message_dicts, tool_dicts, system_prompt_block = self._prep_message_and_tools( messages=messages, prompt=prompt, system_prompt=system_prompt, @@ -447,7 +447,7 @@ def generate_text( anthropic_client = self.get_client() completion = anthropic_client.messages.create( - system=system_prompt or NOT_GIVEN, + system=system_prompt_block or NOT_GIVEN, model=self.model_name, tools=cast(Iterable[ToolParam], tool_dicts) if tool_dicts else NOT_GIVEN, messages=cast(Iterable[MessageParam], message_dicts), @@ -560,16 +560,24 @@ def _prep_message_and_tools( prompt: str | None = None, system_prompt: str | None = None, tools: list[FunctionTool] | None = None, - ) -> tuple[list[MessageParam], list[ToolParam] | None, str | None]: + ) -> tuple[list[MessageParam], list[ToolParam] | None, list[TextBlockParam] | None]: message_dicts = [cls.to_message_param(message) for message in messages] if messages else [] if prompt: message_dicts.append(cls.to_message_param(Message(role="user", content=prompt))) + if message_dicts: + message_dicts[-1]["content"][0]["cache_control"] = {"type": "ephemeral"} # type: ignore[index] tool_dicts = ( [cls.to_tool_dict(tool) for tool in tools] if tools and len(tools) > 0 else None ) - return message_dicts, tool_dicts, system_prompt + system_prompt_block = ( + [TextBlockParam(type="text", text=system_prompt, cache_control={"type": "ephemeral"})] + if system_prompt + else None + ) + + return message_dicts, tool_dicts, system_prompt_block @observe(as_type="generation", name="Anthropic Stream") def generate_text_stream( @@ -583,7 +591,7 @@ def generate_text_stream( max_tokens: int | None = None, timeout: float | None = None, ) -> Iterator[str | ToolCall | Usage]: - message_dicts, tool_dicts, system_prompt = self._prep_message_and_tools( + message_dicts, tool_dicts, system_prompt_block = self._prep_message_and_tools( messages=messages, prompt=prompt, system_prompt=system_prompt, @@ -593,7 +601,7 @@ def generate_text_stream( anthropic_client = self.get_client() stream = anthropic_client.messages.create( - system=system_prompt or NOT_GIVEN, + system=system_prompt_block or NOT_GIVEN, model=self.model_name, tools=cast(Iterable[ToolParam], tool_dicts) if tool_dicts else NOT_GIVEN, messages=cast(Iterable[MessageParam], message_dicts), diff --git a/tests/automation/agent/test_client.py b/tests/automation/agent/test_client.py index d0fa6c690..5f5436e25 100644 --- a/tests/automation/agent/test_client.py +++ b/tests/automation/agent/test_client.py @@ -301,7 +301,7 @@ def test_anthropic_prep_message_and_tools(): assert "description" in tool_dicts[0] assert "input_schema" in tool_dicts[0] - assert returned_system_prompt == system_prompt + assert returned_system_prompt[0]["text"] == system_prompt @pytest.mark.vcr() diff --git a/tests/automation/autofix/_encrypted_cassettes/test_bad_request_is_not_retried[create_flaky_anthropic].yaml.encrypted b/tests/automation/autofix/_encrypted_cassettes/test_bad_request_is_not_retried[create_flaky_anthropic].yaml.encrypted index 0c1b55cd1d67ca320a8fbec1801f787a12a21380..6dfff299201fa20fcf428890af26821cf343851f 100644 GIT binary patch literal 4757 zcmV;G5^C)L0046eBmgZ(spqw*M{U$c*#^21^CozY`KERqoQ4T%+&wy=Eqo}J-x5mz zJP0T|Oeo|0+R%2`rn+(P2@H_M7{wAo9j&&AyyTnU@+`!Zp8}TYl2N`0N@YuXP0U{D zU=DkJ=S{_2<)jNZuN~>mV&bI)J-9#;x^7-ZE2=6Z^r@JIf^dmWeP*CaYAX&yfqNMd zFR|=-+T01#tX(IAfi&JdBoM7sdwB5$VxfQ2g%(CHMJ6{LVb3u|hduGMOu^B#A*GE4 zgld2)FX5D2j8{9Y9%I4p=6Im%IcL(bV1fnfj%T!hFS|1!YQb;>c!4JQ1sGRSf!2Bx zIhXO27L;0$+p?DW@PZNnG)qMp?jXxebEdzZhY8#?UjJ;M_n4%Uo1XRIX@%)~6HTGX zc@q9#)*^KnkW`)J=O4-L43n)nsBq*G?c}&TY;Xs?99LdFbgPBpwJUL2;BLjhe7KSh zaRDCY289_TW$J;ydp{R=A*5s_F^b*l)2u3W&62uND1ks7Alc2n&18l=a< zsHD0!*~P9)cZKVJ64te@fZ$jY#Ijy2eM5!A|7;9V-$zm=!xbL_!2=T4JD-;=BmuFx zv^in?*`Rswehtkalh-|==PHy{_&*9EOVlxn2PE#|gcFgx=eeP8>UF2Qj=uTD%?b@s zu=?C9xP06vAjKDh6ozuMtDY4c#>J%uiY1>^TywilTk^0*p<}*!&(=wf$&jf_L^lBu zT>E11|1qWyrJa}YBlZ|eamCk_%m>&^na|n#4QTqAJ`KlN!kdokW_ zqtAYF-fP9j7W($X-W{mzf&|ZA66d~`BkKg)n;`|S1j>Q3Ww4sDjYOwS1ORcQEGBQ9 zk-+n+MFKS)7!2#n?BgW&57OGg0=6B}SMa3@hSNv>m|W}!eiV-aJ! zs^7ZSol>(n06Ord3D1lNTJt!DVHopk88b2dQ1QW1@fy`ZgK)n=QfqQ?YF03L+o{8u z$p|1a`}l8_pPC%FwJLjRPq;>>R&3tHC~4N_r1&R}5G@RKVN3O{J590M*pqM1D7!&A zU*I>7UxuKF&G`_(upqGXpG+<}O5&5(?_!kE>2@rr8;X$-X;b(dh-Db>B^E+11C^$M z;@dt;5#;#_h3rj)1lo4*aF;(;vuQN^@fH{R^#%R4mMOWDAXPin5JdXDTWd(;EA-%m%Jb7* zlz5z1#8NOwKbpV5bQKSN$0Ti;&csJd$(56>EBj63jb(mM^*u83nXTZ#{N0?P{nNdt zFN{qU@CM+n6SsxVxD5umFEPHCe_0Uti}Yo5)_WO>M6nVoBY`2`^bb1*Zp|1bi+Ls< zGw$8@==L1|b9kTU%w38?Z)#Ie#XcA?aaN1pOL{3`1NurYiXQT;osQ?No=5%+9-$LB znv&wM9CaygGhl)9a`@gC#b9Dw4(#V|zjkz9N6Ff)+YT)Q$L}SE|6y2u zJ49RvhqpBq_iQx%>S@r4{o5(#0FuyumspYgaByv%Hj5!37j*Qr5$xWCawOM^OH38Th7zs z(7RJ=Nx0gFhtL+o=byR3b*{EU84|)m@d1n;0bxU+EKu1Mva(Mw9r>mr&Gh~j8nR-R zG`Ec`%-uFjHpe*4?UI>bce7_K|B;Xjw6E?7^%s%QXE%4UlMU5UNuQ{L`r8Uk(8}Bg z&v?Q8lZYSx;S)prK1?e29LuX(l>IFb*r`yui@@>fY}EY~Ke}CP;Zgx8q#AG5eFuS% z+P&m!sLLHmHsz>V9W{_RwJt7rljR`(8?(6G`YnT;2=TvXaVMTVl5*1Qe@DEzip?MU zl#GRaNDCVJKw-PX{5`8tXYQ(1HnA3eu8xgTTsG0-K^gKT&2!(uguma@t)UI-x+sOT zvH_L3;ri7)n;v@0bk?_05Hl{3_QSbcikcZ;t&^25F_?0HbQIeMCpguADOLI=ZdF)> z^@wgvn_;JOx2U;Yc1i6o|EHHh_{lMm3M15j!#209g&|YZ-@h(&uF)SOXlsvek6`<| z{%hVNXdez7FORwMcqHJjiV{U+2o*Tu)H3nE_&(JwA~DvWbcp2+DI56Qvfv>pk-p-^ zaF_5ci|Wh70^oWX%OnhgvwPt;OLqd1HG*4E<>4=cG4KxFH9JmLQig0NQIXyfmmn{!FUxbT)R{!_BwhZf;4oA9%9>#$JBMK)?daJR6RZRFUu)Hd# zj^-$A?n~~7pHGH7mD>!Dn)?EIex9n*2=0{+6&1QB#{eJ7{RhN~;eenk6%;Q{oUQL7 ziU{(Kk*sgg03Rn05AsN#7l9-&E1jOO-g?#%4LH7-0I#D*YmSvnf)gtO&E(SDlG`b$g0}|u?Lt^Q>}mCc)UkvH-VVUTsImk>ThExGUtsnvv0Z{>vc{w*E6D2 z-&DW+N}1`9aZU|#wm4TkP)=QQ`LeIKt0rDVMlGt1t&EjJSfx+-@$**B5H|1+V1PdZW=RI~8kdmb@?0 zz&2N1h!9v<{;bP-xlU@!q<5CX&NO3he37eraEz= z^(bH?nZ2Vl?M4=(buca{XCYhK8RFSTDTQtEM$XQ`u|98eLQykc)j>S5(}sS>CeQCm z=(|SfBqg7dW(C7R)oElts1L<=Vf}AeCK}+Oi3ibgGAobg$j1_vJbNp<%SX_&1|4N< z#2Lva0uSWydKEx_cE=R0w<8%i4P{{yqNVC;lxTA#-oYX`+e|P$7qjY4Z zPDG*TO0dGqrMcnb=aZ@x;QY8FAxG&lOtEand`1jMrUmAJ{0SN&FvI#hcRppdbHZsZ zl)wK@^ETJOz_R_xLS8p7{fvSVLq=){zQN~KAClsiK_OhWw!1sbs#a$;u<5Lnk1H=L|RG# z;xsxyG6E)LmLgmUqy-E7u8#61ZH1n>nMYnNm%o1hjTaOG0sIv6V!t^ncqzp{pN!SRS&VZ?{3l1G|C& zE*(qI%T|l_(%N&iKlgrOIj`7ZYHFUDUkA84Gc{8+?X){F#Dne(;{$*vKyz=Y>e2+z zL&iXioHGynY`H%Aa*_XcQ2!?Y0hpvKA6!5hYmd@s0fU9DfFh<6=&7=zN|R!v=az;m@PE((+Q*G$fXwfLh`Je7SgK|5NvN zUCVG#z59VD5*_(_CFQWmMtnwy)I$?i9Fmby9wa?NinvYj;K2ji1t}ta`m|0aj0pe1F-CjG45L0{d2(kmB?s3%-=tV7!|xoaH8NIjHfC!> zoG{)V2iVR~pkT3GSHhUBQaUByopaq27u{6txEQnv~!Os*ag=-dQJYp09vd@D^ zs68X~!8Q(+Cc(r=RKrK-khR78dT!%`1p&x5rOF+TyB&H_;G?W+bN&ssWD(C-neDUUssQ2@XU@;QTLjw{saaO4go6Jx;B!+ghs1lIsrPa;VsJ~mgz zm`qpP7Zfl)dUj5DxfxGR=0ZZ_U>{O{j`91v!r z6%u^vux>zFcGUh&3ev0d#ceFTPFaR7T8b0E(#IXRiFazjD1bLRFR%aDDq;5L+^)sC z#uJi%OIiYSNIr3ib~#0QR;IypYK+8T1hV4rj~dE*NH`I1&t8L>_d`qFN0)^5j1zBw zFnGBt#u4}{q1hR2?K$+;(Q~vA&;=om zmx;M)XM*rB>+!Fv%!5RbkPw{?1tXIr11(%|syS3a!k~ywGp?W>6OakqGe?>XE&3aq;{F#o^G2H6EmX$;7=UTDRi z^)$dL_~#RS$WKbCTR`HEscv$u2I3iXJ$%yXi=``NOanz14LZF>5XP;KlUo7yf>I1G*4}~8?xHS8e60LdqRZO&kr3uo?n=UTA|*#-3I(zkO2w zQX}wXlj%Jmu3XOA8vX5n-_y0~S|Pd7hD$Qop4X{i>n&bOqie!~=Y^zjYUKY45f22s z<69jp=7725H5|Gihh#9{*F_1)h~o!4u4SGp88o0fr;`J0+W#Ts%&*)orvl?N;uY{^ z?d!S=a7Dt0MyLQcnjBqyVCMXugU-!6wx(AI&GXgK?{$4jzlK@v++qc6*mh9xj|xY- zl7U~BDJ8-JRqbe%6VsMn^YVn&gP$CD3*8=W-M_7cD#HL(h@S^;+;TGb*Y%d;s_KlX z@~V66VkkZWe0k&MyEr$iekzindJ`Tdi+XA>dN*=Sv|I(zR|;TIeH3QxCyy_Jk)Knn%az?9{rh-=uPJ>AU{r?~Q>%^~)0oy6gG(ESejwq@On{HD;{1Vcm|G0Nv$78P)dp+#q$AYV0bw5_SOCm`ocpHd#dVe4%i70X9J zAZr`|@AWwe5pmB*{_c{2ZvZk^8*877ciGPGs6B|cp3$P?HD}%l_3lqc9uyZ$wwsO9 zT|Pd;2KbNz{;P6`Rp|!T#H={CV3AS(&!`&qQPaEIVSxBq54^7rwB1Hm(0>=A^V5t! zn>MUp|8Cpl4rr6qzV>(vp6S8z?S3Y;*G!@9@jcob;7i37=81W`_Ut3U0>ALq zQgzMBAI`-hSuvyR`rk;p1~LIgW=OW~KHEv_*4H+_w0^0MZ4LF9h-(4Mu3pR0q6%^L zqeewa-Ls*BqSq}KV6KBrAF`O4wR$iL2I~e6Vn{^#?7grt`fT=#nZqPxz_0(hZ23UY ztQ7QjKDFI{r8-12B{{Js5FSek(PI{nJ5_#g9^CcSVOwYq%=1LxaLlTcf}<{TG4Kh{ zGX$0}i05cX9yQ^d4#rsb^{tH1zCKIVl8&&*)rlkndT_As)WiZs z7qxD$0+ha75fa!h$D6-_3tb%$wNT&%TqU!I=`jFFLZ z&mzDBd59QUK2f)yZ3P!!h^r9defkI&FPuMGT&ifC;_xC_aZ0cG&HxZoPHj~aGh)W` z40vQvL^hTwuM`J+Gm$`(Iq--P=kU&sn51oeT{#%9P(jB!gw}_YKYlblha(0`Mr4yD zIh9Edyf|EcXF|+!p*5XkCaOCgzf+vA3BPV`>j~j@Wx_bJ)ImE-0mPiNHbz>@!E3n( zS&mL?6kw*8@KT}ziB$q!L=htxkt|ddq}Q*K#=fN`4Q8@&Edvj&d)}Dz45XgtqM)~L z-VSsNms+j!{T^A4a`v@qP_mG^Cr>tH$2>vsxo9o-sI6=n>0ROAH6Ko4#ojY`|-TyaS0BXowTi*9X$V()6B}`y^zAvV* zTE$zFWb7sk`bwmHqX-A;V7y&`^yDBnb@i-57x`tcQ;b&%?6ln|MRA02YJkbZ-Y(RxryP`h+is=W z^8xI`C(rRX+_`aFEi9dyz4`m@u`73{nE&v|U)5yy880F}zV%KCdQJ!@;HAAwG-Ie> zoL5i7wZjc3^rn%-ud2IB_;>fWMBO|?-UAtZ1nbD&Bx5tBf@Oy{D4ous2o1J&<3^GW zS-%Ig_~4xMtx?D-94vAW|BHWt{=gp8H-I4N>N?&Uyf2dYB0k6d6?6|pn@c8RuJS=Q;@9D@1ux_3kbF+ca0!U9 zfD|^w%Qq96Wn_geOC4xz-<@X{QZc&V)3vJt^q%NZHF1wRgHhQQvx3%0p#AhUW&6z; z;(zA_M4CEws#qtVHBmG<-SQY!jDskQNaA2hKXHNZ6b5*G{$~7~dCNZQ;ENKFXbpo`>JEu})d zCXOy4p<$so7wF^080Nth3-0f`I`hx6Lg=vf=1Nle?65D={!xt{M&7d=8jbz>sh9mv zcH@ywit{vLixvQ9grDcAI7hT1q9!Fim&y>e6066N%w;vOrvqPKSY0ah`9Mw57wqXV z);ypS&gYfAsMVWaIXK_+TigDDu9D0i>BCvpRc_LqLlz}2nx1QZAunTWIX$E=RDH>< zzz-&K2T>g?NqI@3RYPABI`v`vMM7Rbp?-F1KEr&mk|JA$f=CJ_pFTC|unncZ%s!Uq z9^L+;7Y@dNzh-g=Ve>ddH?>fXt`xYf%>$B_yAQXy`lE!lN-zTn*y&HjH)-4PQUG!z zKy1K)j_LbH3;M`jjrq+ed_fY`Rk<-fb_d~TL*?R>dy!?~6%(5y71U)v zXZUPN-wW3eGR+ALrYwHq1nvrGCakwaRd=1~(}5YQU^41;Y*Gb_9`eMl!jLM zl4J^zF1H zFi# zu1Vr&*88IU6(LWxOjjB0*%_P>*qIbMEy0VE$s%7Fp@euGi<4gUXbP$1OHQQ~>Zw7d z;thG!j72Os@NxWTtSnJ&p|(JS3j+I!^@Ev6j`g_}=Z$l$CCzt5W|e7Hk>>7uqlH}2 zMvR3-*Brw8&EgVKPT z6j%P5bOY>`oKrCvkAH%vnUfn(Kfx<64dZ6)-08L8W0X*15Knh-{@q~POCQ}fInz`` zeomtu4!536q(VxE_vgJtDVlR1E!rJ=w>37)2$){;OmirFDT3wpwSlE2AC{N$rADbe zO3#zqn*2D|^xD^_3VTC2u%A8#0(JQg@@Pxw7Eh{kHD^cZ=eP$cmkd(RY^(N?C&*KE z5`5~%s6V^d_)5}GF~E;wowq;IcG+7 z1{aQH?K|*HDGx*aEn)57R3X5nfufaPC+A+y1;e}$yGhLogvEkkE2gTF?D1xB+5UkR zRR-#2J`pDJlxRPolx2rERUU zbmIhR@#253oJ-wBZk}YkpAh~=1F<0*`ui)!)_&jjY34C2M8&Vpk9`QU4ZY7bv3O5+ z<{psl8rxr69K(m>BRjRmu=%M<>Lk;KPpUqpMnV(COi@FE5p$I`EM1xjbsudZUie$L zZ(>(veij`KyB@zZ#(|5jOy<&jjgFt3L=hHdOy1dcECCu`=)Ryglyi1A_>kz7AsdzL za~6(yHh9$VNX+ZvQ7j9C=63}J#K(;{;8`gofpJkQMpINU%`;U)a^raccwX?Wd$L5- zvK5&;?SI>Nblv=Vw6CUt_{$L$Z#RXd_Kv z2{bpTfUX5Q63^&Qw^@%2IDIC*{YntPd6>jMZt?Ucl*rd$UBGyc>S@+%lQ$)O9JIw^Vsn&H6N?HPWx*Cn2Yz|}`7!xmPe_D}EXkKEE7P(-y(L_j@vWc^bs#Qf25{5vqPjS- zQ~pj2I1ftHRnM8{6W<7Hvt=uq#uG>y2Wm+fBN|W3nPOeQxB>lC{_5Ykwh>wD64aJT zyS8fG`Kp^Y)XqZu^#Q13U>6r=a#>#6GtWId7s1R`2eBy~&DoAEz6o9@K-$o-UrgZr@4a-j;v}Tky6K zzU}u0R(prW5Gcm<>we5QO;|US9vS|FVs`*Vp|FpBSGXE_CG9c{0l%khhteiGIdbev z``A4Z>4qmq)q2pT<5g4t5heKkR%uyDFw_)CLq_%}a_JQl^nr6w#q^e8=?tJhPQ0{M zqLGM<>l$dv3?WWUa1jV)I%#Wi=|XVu#O>I>6UFo2!j7G9$dmhc>3!vwATVc%K zgGt;WDHNS)g}RVQv6RDnSowJ71rvyKwB*9!%ry&{y_gayaZmd7toC$82SQu%N~-Pd zD+f-C$7Xd&MkBryvywT?@Xwg4=>RUSHPOg{4C-c7r0IOORq0W$X?OwEu-+oZ82SAG z+V4Jggddxf41n-mO)nKi1l=zl4>DIMTZ$K@pxC5HfeF($2yW4eqJka0_GA$Wh%oM# z`MS987MjE-p*K8fD6BRfk}ZL1fg=E)Y3OfwH>%_vj#C)B1&65EGbEzYZr3_qkXY-a zNjr;Sm@Bor@YMoGloRuaP25K58KwWqUk@~>C{Oefw~zoqGI-9ne~zs1 zw)zJNa9Ha%NM#5l6jnF_j>u)1nj2oGYNp*=@K~H>VYnq4_prl=mLzB|;P|P9t~%woMXI^Wl5~jpmt*e`6j=TcF@LBhF43rl zCyVME4uG)HsKM3hJcV!1y~OyQ<>d$J?Yh6@jB4|R!gQb^FQz(_5Mggd3f!x4wh#Fl zt=&=5kQ-d5CTo}y9X~mK8UzLin(X+Sg2gh5d+b;^>Pz11>=_(O_(uY2B z)KU8~ugd{gqDL-=IiP>{Sbgi;)&^PtHNR0i3v)ZD$_)DvlkS1K6bIuwZ?vp$8i_ST zM4iHa-@*6W^nj;N@WZUS@#0DGir@^}|J}%5X&uPNEd+`>1kR|vA<2opPi|GVvaWw9 zy{hwH%KF2NU}!3|*>sTjIl~F)>CDlJmBXumaDy&QCf3_JeOnx