From 0e445e6e6924622ea3f7de8ace1299bb62075296 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 1 Sep 2023 16:56:07 -0700 Subject: [PATCH] fix streaming cache logic --- litellm/__pycache__/utils.cpython-311.pyc | Bin 82105 -> 82054 bytes litellm/tests/test_caching.py | 3 +- litellm/utils.py | 43 ++++++++++------------ 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index 47fd545e697281c89b57e5562c085b02f2d81dc0..6e56ae6b94ae3671a765faa7f2f6f13c3ce87c1e 100644 GIT binary patch delta 3296 zcmbtW4Ny~87QW}byaXZ%|3u`+Xo4$Yfq;n^1sVSTqQ$?(AE*K${OfLri1tCycG_*% z-BPa>XZJ?9ARt zPR{+#ch5QR-uuqUA9gY??_?Cm6$&{bb9<9$YiY@zl9Jh5k%6wo@f`vN|jQ466sGV=D=)n`8VzU8%2%L~5Fq-9LS7)D zM^u{pTzX_NBO4yc}1 z)l;6JId+XVJ5kz`5U%Wxc32{ZqK*MM9P;&!q3FPQBMij^BHdC}`oj+(#``c*8lXZ(2NLC>05LZeglB|sNG0|>^eNZAR$y`PxV+qdC z1wGTQ_q?q+n|ICJXA8^~mtA$y*)!UmjLuXTdC7Sr~_pW}n8lGb0(> zHsS>TJ##5und5|5EMK}l^=$&#dWg79H#XR9-QskKC=8-V6b-?(7R$!^+WMvjJgYcf zDYZ3pY;CjKwu*!!IKwapkB+PG$)!437TNlai!?r490x``R-BC0xk31XF%HL;%)`yO zJ00#QG*dm0bwj0dsdWF8qIk{0oD+pN;`FXK{h8H*(R|ZbdBa%Auc+fKR+q6+FgD&a zw%#zd@@;m$^97fY6^yJKp**UHdVtWvB5J}TZ>41lX?cA9Dj{w4{^xGZPCKI&GK=`) zDj{<%KU@6V(fmZvI2Q6*rGlpHYPz6V&u_2@ng-t5C}^7aW}Bet;O*UlriZvMpv6jm^aV4&y({8^cb8!xM(-~?wf z7Y4y`u4na3fKR#pHGLB5d%(%-$Kf0|V*?9xsy!OB)_DBfz7X736O9L}n^Y1ONP$<9 zQYM)$Y!(L3ujzmge6S`tVY=@yQ}vo*B`j?LvG(AdOJd?#X1c4eG)u8ED?KFjb|IWQ zl!*Jr!bD=4T}zfs4HmI5MP__tpRGHH4H^jVcg4Y6e0WS6MM}^-pN(90WV_!Tj6WFD z;=eo}DwnbHX7H3N9KInNWMZU5l2VLs1slxDJxNJ?ar~3g+tYkRgnU#nIjhu1_u2i@ za*)3;89rQGX9)`Ix;BG7bJ}i8^`EuO@>zVX+?K!wcR@?sQ#|B}K zLGiY!`x#nmiu2T1@dDUZ5^+126k2aRwVlMz*S-#?@cr5ZID$j!YNg@iuSG`!?x<@f z74(a`GG$l-pSZ{sW)i|oe3*$#s&Aw;VI{NCC0!Pgm*?hZlb1z?Rs3~BPI5LAz03RE0C8NCbr4_;7tvHtLp|5os;9ug9dB0y z9K}~R55TKj^OnsrILY1Wd`Sw~xTgC)$=%#j25H=zJ>SS-klXd@GJtg)|JwTzGyGm( z{Klcefcc$64zhI9;p6~x?_qw%330dDNIS6&;w^8L5SH^TE7_>T0W*0STL&s41C|VP z*#YyQbA8c+q6_iRZ7+&`mWkSNeEQrnQZF8#TQTF52uwAxs`px;k{fyNYbkueojgAQ z@D)Dz(Flxl8!z@y;O4#@euZ>w&U$H#0Xb*5rk8?=YxwNt2&m)kjX#o;pRewnr=f)# zaKB5#F0S$+L%|^a*&{t1;9hx@FV>*Z$!_>PF8y`_KE#W^>w=Fk_gyc+4Uh9+nCpDJ z9>@yUzq;UKuKeF~DfkoD{zOW_P2Bz@-q{Cm4`AHcO~Z=-e{%u@dVqg8Go|ni4G)}& za(E&Gr86-C95k5v4@biVIPI*CfiEe@b1sR6eGIH}x)Nc4hOJK9Tu7&>L4^N$eNO-7 zBq)a-XXrfmSPFyvpZ^p-fc?(CRH$Ld^nND14a3eo3*bWv-f=Ep2r)8!sRT&He72nM d;1B(+Yy0=+z?_H@3L@RePp)@f$$Ec(87ECmCrM|jlP=8t}`=brQX z&iS2l@BO~-+`YS)4|Xw%V+w^Fk#Vg3rt!OX75xd6SeOi zN^lwg(`FemdkMO0w9e{A)J6*!@hP&PR)ib4d5OJJ|L2tl5 z0Mx0BL1yVJh}6g=w=&o_F*JAp`u`Gx0GX#3oXf=+X7<2!y94mU=onnL#FGv;YjAQ* zBu*^|#lr*MQb6m_0M(y~KUz``5jZmGJZ{PM$GftZ!A5*iy$mnX#zP=pt*webL@2M) z&5dT$QlgpC1hGKCWc5a4Z9{#-rbg3#V#mqwJK7km)rUyBWyDsg@E^5F@?VG$+KHSZ z1F&An34p)ShvCYUTAR}g1*`0tmzD7o%6KOg7^1o(Myl9fT2G2DN2X0groA7)>nc9i zRbJLra%-BnO|28UHeT2ExvuN7u8Z5;%k|kN5z5Y^Zcj@mLI$dYx=D-@N|-32zDUX7 zQw&^D6`!)!?)^nf%1I-iQNooz&u3KIxR`44e=RnFkIm+mmh-VI$GiC04P5OeKDLEx zZR2CxxsI)TY!4yc2Kho3O#wxWVyZ&OqNxxJK~e--OC1;Snx&k6C9hdEZsRqzTwM#V zY3175c})k`*~4pkiF+Ru2(`r(QZ{0d%Qem79~kEHcS%wUi6#09YTY#7I>B&3M7lD# zkZ*NGS1PKm)HQRPw{dlS6Lo!jT_0Dm-8NCNov+x=J!9c?)(M@J*I7ArhEt4cVGGy` zd37e2Rl=)F$8#st&ws0meM7;kGT#^RS!G;#4WC7xQzxI*MRG)y3%QBpx_d}nW+;1P z&!3x``ctIFiRcj3jEE$v7=^12q`Cg4$Z$%C1G(Ota!Roy#Uj2ZfK}Lgc_2O-AB{Ux z6}A`1%6c&`soWc#NeRtYptGJp^s@?ifrvh)FoLe71wEA_5f{?){L1CbMTwXg7rhsjXVML19h1}J zN~#a_C51m-un>EuwNN$p*==cRSk(V2*`J|;hij1ix!%f{XO9=M-=*dVJb5tyhp3|Q z&Jjb*i2STxTqFma4%ZRMZ4hC;7h@|DS(TfJ(QG*OT@DbfO z-dwmG|0}x}d~wgx_4tS(AXaROBYaoIrpE3U+RY~6d6|w0;(#CqiRcy+f!FBMC96n% z-G2xCiardY@dr4qvWM&ot2Jj4a=$Ae0&n0b-1X$_tk*s#}H_28BO$a^g=@d z`_l5QG`xvLg{2-xg&JKY<4Z}wcuV1^__%<*g|iJMFpT#YYCPudW(40b6v9?Exu}DJ z*YV*M2{42|Sy8p57#LqC`VU!2EfR2dQ2}^$iIQz7siI(nJzZ8Pg|~lG84PfV4OufF zq9_UvulpMIvoEc;K*$2~Y=0(){OpO`D#y#Kw z&6^Uvl_7-zv zdwWN1dslOFOJ{R!*Tz=dS63trGd)_&<8`_qp|6^RRdQ8En(L!RdIX2pzYHUIq&@h6a81w2guWZ=PGP_5XPoAy#=tA{d02z4NYu8o0ftXS#yU9 z$mV*j(+>S?MpvfaK+Up06><3@pm6OZK8z0ip!FD)* zO~a4bh}#SWb{un83qx$>-F)GI8l1AiyEyRPBz%k|_j};Za6*@ffnGV;~@jrX}IIKE`x^> z@O51G1se_e!L~s72u?cUg5WC(3LVlAc#VPQ977AhPD8IFHyV;?un%@Egm&2F_;Mkf z6BF}Si{W>0&{2~JH4HINr@-ql=BQ1D(-a(cgk?aGL|ZNba*RG%PEO;u=g-y+*5^Q! TUl9fVPBc%hb-bDbH>rOExV3LR diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index 8c1611c0395..b08678931b8 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -12,7 +12,7 @@ import pytest import litellm from litellm import embedding, completion from litellm.caching import Cache -# litellm.set_verbose=True +litellm.set_verbose=True messages = [{"role": "user", "content": "who is ishaan Github? "}] # comment @@ -176,7 +176,6 @@ def test_embedding_caching(): def test_caching_v2_stream_basic(): try: litellm.cache = Cache() - # litellm.token="ishaan@berri.ai" messages = [{"role": "user", "content": "tell me a story in 2 sentences"}] response1 = completion(model="gpt-3.5-turbo", messages=messages, stream=True) diff --git a/litellm/utils.py b/litellm/utils.py index 29eb2d0ebcd..d10a742fdfb 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -292,29 +292,6 @@ class Logging: call_type = self.call_type, stream = self.stream, ) - if callback == "cache": - try: - # print("entering logger first time") - # print(self.litellm_params["stream_response"]) - if litellm.cache != None and self.model_call_details.get('optional_params', {}).get('stream', False) == True: - litellm_call_id = self.litellm_params["litellm_call_id"] - if litellm_call_id in self.litellm_params["stream_response"]: - # append for the given call_id - if self.litellm_params["stream_response"][litellm_call_id]["choices"][0]["message"]["content"] == "default": - self.litellm_params["stream_response"][litellm_call_id]["choices"][0]["message"]["content"] = original_response # handle first try - else: - self.litellm_params["stream_response"][litellm_call_id]["choices"][0]["message"]["content"] += original_response - else: # init a streaming response for this call id - new_model_response = ModelResponse(choices=[Choices(message=Message(content="default"))]) - #print("creating new model response") - #print(new_model_response) - self.litellm_params["stream_response"][litellm_call_id] = new_model_response - #print("adding to cache for", litellm_call_id) - litellm.cache.add_cache(self.litellm_params["stream_response"][litellm_call_id], **self.model_call_details) - except Exception as e: - # print("got exception") - # print(e) - pass except: print_verbose( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations {traceback.format_exc()}" @@ -356,6 +333,25 @@ class Logging: call_type = self.call_type, stream = self.stream, ) + if callback == "cache": + # print("entering logger first time") + # print(self.litellm_params["stream_response"]) + if litellm.cache != None and self.model_call_details.get('optional_params', {}).get('stream', False) == True: + litellm_call_id = self.litellm_params["litellm_call_id"] + if litellm_call_id in self.litellm_params["stream_response"]: + # append for the given call_id + if self.litellm_params["stream_response"][litellm_call_id]["choices"][0]["message"]["content"] == "default": + self.litellm_params["stream_response"][litellm_call_id]["choices"][0]["message"]["content"] = result["content"] # handle first try + else: + self.litellm_params["stream_response"][litellm_call_id]["choices"][0]["message"]["content"] += result["content"] + else: # init a streaming response for this call id + new_model_response = ModelResponse(choices=[Choices(message=Message(content="default"))]) + #print("creating new model response") + #print(new_model_response) + self.litellm_params["stream_response"][litellm_call_id] = new_model_response + #print("adding to cache for", litellm_call_id) + litellm.cache.add_cache(self.litellm_params["stream_response"][litellm_call_id], **self.model_call_details) + except Exception as e: print_verbose( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging with integrations {traceback.format_exc()}" @@ -545,6 +541,7 @@ def client(original_function): # checking cache if (litellm.cache != None or litellm.caching or litellm.caching_with_models): + print_verbose(f"LiteLLM: Checking Cache") cached_result = litellm.cache.get_cache(*args, **kwargs) if cached_result != None: return cached_result