From acbb611e8997b5fb00015dbd3c0ccb9546029254 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 16 Aug 2023 21:44:50 -0700 Subject: [PATCH] adding exact match caching --- litellm/__init__.py | 2 +- litellm/__pycache__/__init__.cpython-311.pyc | Bin 4491 -> 4528 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 25606 -> 25606 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 42837 -> 43479 bytes litellm/tests/test_caching.py | 27 +++++++++++++++++++ litellm/tests/test_completion.py | 2 +- litellm/utils.py | 13 ++++++++- 7 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 litellm/tests/test_caching.py diff --git a/litellm/__init__.py b/litellm/__init__.py index d47c8e7b873..f81a9093a6e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -15,7 +15,7 @@ openrouter_key = None huggingface_key = None vertex_project = None vertex_location = None - +caching = False hugging_api_token = None model_cost = { "gpt-3.5-turbo": {"max_tokens": 4000, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.000002}, diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index 07167d14986bfd4239c769b160eb2111badd76f9..f76206ecdc8b8551fc786f48f4b19a8d87a71008 100644 GIT binary patch delta 432 zcmXwzJ5K^p5QXnuAFPWD@qzMwE$^iiP+oS}Sy@d(kXcvEu7PNwCI$*RD?({wG{J(# zWNYGIu&|&p@uxVfI?4IYJu{Pg?}PHD$gi?2F{;_q7ru{VA8(A(#|{BDki+Ka=CQve zup@@vAw!?WF18oDi<&GkHYNQQcm$?-CKRUsEe1R2A2Qgbi@YQs$xjN9f}{{BtTPNe zbBJJ2=Mbe7q7)+!lgBZl^FAmPqZAVu!?-R$5*194r>LK#ewsW*p3x#&^p2j_Vm9M8 z6`Kj0X`4x#DVv!EA&Y5xK8G318mg&+${C$=M*f81at2jn>=QzJW&sLj5sH7q5)Bti z`{WGs#(`OevRQ$OS%s=ugPK`~x^*h_M1DB_l)gnryVXALTre9ysP!$<^R1bEkVCTK8_3@B?61Wt;#2 delta 400 zcmXwyJxjx25XbLHml91yZMA8ezNK$zA|2|` zMaX9m{16?wyZH&cJnF&!_rL$$!M)qhw(({dI&gh{ujn-}OmP}CFB1Z{gO^M&(}UU5 zWJiMwpzIXL^kBRjU&pmb04!O*g=r1E^n%0ke*~DJ+!Sz+1n#r4ER&UE1;V@}m~D$nhQNpd*0$w%@*rz2hf4(nVJuU}$QV`?ow)%eH^S4&?gB zvWs2W!=CJ8Uk-2}hd7MJ+I>=-ge!d=er()DXPa}O-|$btdWga=<5sMr>(sf3=36Q4 E2mI_$o)GR08Uy56#xJL diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index c8658d0785ac8fb4732a9947849c6960d6124aed..fe9983c5b45b9f6ae2c83f507ed30a015cc80981 100644 GIT binary patch delta 4092 zcmaKu3vg7`8G!HEyZ2^ad0=+4d6Lb8kPRV*NFI`akc32Du_b`IIxNfHo6X`r!r2=V znGGQrH9^pl6Vwiw5)q_WALz9zK00b0X6)1sC=ATDv9`*L4r=ky)@m#L|Ff7-XX@SE zZ~ycE|M}1RKlkh(4Cmf8q#aF7wFvO+J2Is_GQKY@n~Xm*x0Bow6g<_6WmxbmPy~g9 zYCJXkU-C$bRlTmupxB0uo`nkGpVT1uD5NVzNgF1fS|uIo)hQXkdZk9W26PR|b&4JS zuV<%-J>M)S3zaM=E(!=rrefruRD~D>wUK>C=HAw%)GE2^bxI!m^5ItizuD?yrBGem zWm8y+U%yq(Wb1J!c}VdWn5hCQ<##+!F>e!~zAs z&np!LVZZp0pa=s7_=#r0AdFh$LQsga$--vQph@iYO?fOkzg#jZ;-ttvzuh4jhlRM2 zZ3-2xk~9;}eDc{MeJ@dGT+(dcmjq#(K+~=+$EBQUK`0gIodYHTle zlPk{TS0#&AlV*c2gq1ZP4X7Hu6^is)fIB^5Qhb`K1;c7Wj70nBBG4v`TcW`TorPkG z-S6{v2P1)mm3PC7MU{=ZR?bUJ#_^z=B?h52FOMTrHP#!_=w>v?Jv18bVV}D??H%1$}TerY`A zs6Q?P;5*VbY*lQIIRxu}C!r6ODUoxjpln#LS zvi?%}39@oGh;ETqqVE`jp~67Xo1&jPC2719?V5#D{95htO7wcoZ=o(T+IGF3)q5Bg|d7L^=kFgdVE28l8`$XbJoJy;}4RH4>wB z=zNsjUwu27U>B7A%)u z2k|TYy9G`$?+9x7NX?PnFznh;D8@VPm)X`GcD=RsI4QtLa4d{88d7N&*5G3mX%J*) zukRo}=C5y<#kVr$4e_{-vS;e!t*bN`6Y(Nqm{_;cl$JNPc){jN#;)a723W) z6e6Zr-um2|PNT1rXC25#xbtv1{JkNc7Nzu`Xf=nT0k66ZvMv1wR?I|Yx>xI_s!#Fq zMBJ~@uhH}h8(!}&jiWd-2A)1K?a_ZD{)6}zJH7r!nJ;-iE8I|T*@jxgq`qmx*)%!n z${N)L?#pQ6%P8aIBo_sB0u~8EL|kP1)tUw#(~9L!&?gn8G{mP^OhM^0l+sbkK+Hne z*=4nQFcU=wSEA2a1aCP9bxuSsA`dY=*EAc8e0urPF2ZVm2jC?T@Wy)meprI!j;_U4J2xM#yLEWK@%CK(SuG zq32l%lHwP=ZDdHle%o7w)G@KYoH$u!eBi5N+;ACv5lHAB%8!9KQ z?Bk)EEG^gy;t9QZ*ICiB5zB~o^rCxSB;+Vd+}lXHSk_1j`H*==##3)c51va24X|%V z>d4>pno*I+zrc=6u*TiF_Lbn1kU~+P5~EA8uSReU?%qni(Eqr5BdHI8eY>exQ~R~z zMaA0<#RDH-hKEj|uSla<-Pje3D14RNG7Vs(l=bY{%Zv{c=(&4;N|s>|8b(AAA7S%+ z<#-fG{HX0gV0Y+N0K`s4LWRbjxc?S|QDm3*_UEd7_x&<-aG{qJu)=eIdY;2#O3qR@*AFu=C zL&UEa?5`rEjjevLDQ^|p_{tb@WoRqb!u&{`?9m4|OZ&iHu3H{jZLG!16|M?CZ+w~w z*5%XVL1_pvGf}&+co)0xK<&Iv6mj+vrl6)$uP<_Eayg|~PbjF-HE2(+2fKKn$j-0v zlo+HOcw0$`y);BynQNld%;$}4f%l~~6N{xoP_03KU}A+>%zMGthi^hLVa8Wy%C7+4 zIm*5{SRx&P8tYl{u4t!*825`?ynHh4NZ+)@WVZjpnIVUHPN z9N=C<%fL4wD^Yl*-r7ZCcounqLOu^Fw-(9O+9FQv)ap#__dkn-IvqFJ-~G;a zzVDpxe&?L;>>IDC7k;azJ(8O0Qt0!Y6>sUE?Ae_*f$cg~ehVwsU873ZbY0O|a7Nb* z`P8~JJ;kW)QT1`7_O6-YD06qz>FH!wuV)Zv>9u;+DC=s_XX-ihY)n@E-=AKm=Th(N zUPaH=?eZ^GXR2Z}2~{oMGDol1^Nd=3B7G;(H=n)*#$0`}F}G)&UO1|C&C`o$wr0JU zIA1RzUayxD7wA)nEqWQTRiCAo(=2WJRBE;B9^yh(S)}L{ql#X6LRFN<>6`d;Ez+x~ ze=*)NE4ZXqZNH)`!)mgkD)%Ofj)KK@5?sPRc~J%=XIu#=adw>TU`*WDIKC?8{5lFn ziBnJ97FXl;9_gAsex#q&DY15~8;Mmz8hDRMSy5 zpJi--_aLwd7!oVXTQaU`L%Bh$sCJ8=m8WYX)I4UMFW+d>c9QtCxvcUu>$rnNkBwK+ zgWm)E01B0lK!%+sNp@O}ejW++$1Inh9zn`qBs}K8x5E5hnyXCPOXAC7sQMp89PZ=z ze)QD?2Y_l|5pYoKsPV7|#Pc=H+7VK`Vt!eZVjuq?iIywm8}!B^>x^)e&w}wGkw5dh zY?m0CIlSy~D4qb00}lgF0uKRC0VjZ;0FMGs1E+vz0QtH#kRB7Qb?w@-Bz|OWuN!Zh z_yDwf2v%}E%`snVc$yVrAlh7Z9tj#&l0OoT8Q~b0uWPf^n(5-` z?B#5g_+)ltuH_6F(WtN2h;lif9R7f4Y>HQ2oryn(IguR86*Ho-E0dmub{!BBe{7n- zel9+5s?vT#iWR13PI@vsFJ`yh!c4KfZBoVs*!>=O8+c2cZrfIHoe0M~TzfR8DG%^xL6GyxvvQ_r=at z?Rj!YrxDh@15tzXYYfGuRb6ygm#nT{^f8=pXsitXT3^iDABn~~5~=2YL*HM4$pCUg z%jD~i4RFJ<4-5o!O9SipIdNuneINDfANy~YM4>yr3*b)Mrgd{2p#PETnB;#jP* znsh<;8SqcwQ_h1_L(1aUPyB!OvTyW8J>TkORm^|ASIFy>2u)U zz?Z;Z#H6*gjdDyIvh^;M8YBns9-3bP?*mEDIe}zgoVaK0^erh6Q-Ms_+ziN>Goecd zvVd&hYEF3unlg=Y37g>dcY>9IEJX+Weo8gJ1hpHc$RN)Ka?wnrj+&xAP&qFM{aE&{ z$r(B(!C@kh4-^370eMKzio<~t*IF0?%=}d#gRYOF)*LEQXGpU=aqd+9A~nRzMFnC( ze+6q6!TzSc0k~}dHWH}x+xsGcFt50({zkFfj1Kv(aY(~(a$bo83VAfPK>CK`29dr# z*Oh~)fw|_i^=XW~FPeEwt0u)#^Ev*EM%Alta3LEoPYs@9Y=+o8G?k@`<3n>;t$1h1 zmK`*Ajr>`4$c3IS2JhOMGPdUFB4xCSwTi~kscgRRjjnJtV=~~!<_n`2Y_2tE z1HUv6ZhfAyL*nLbbC^%;*w)JaAYR?JH+3~EWcg#fTSV`h#ojd^yU)hz1Y+AIgS8yG z;g9Hs<%-gs9E)xY#Mbh5a@L2X}i~bK!4qb?d2MKg3CI*8dB1(66ih-T^=E2=xsCn?>KLxM@uiQ^k5>M`}Z1=&X z9~dN9POmpiFW~j^7TCy5Bv*Nedhy0EN*lXYWep99{C)ZKV{Oj9B4#sh+qZ@}k!4RZ zpCexUVL^kR8r%S`PQ)=D9|5oyd;*XQ+zuUZQDpDG(*eoPaGO1~-_6(p@zR006Boip zu7@4#!Q0>{PjtGNc(7Z$pZfF7?;l)duirymbQj1}$;`@%C6lreqr`!+^bDhUhp-)~ zuUrlhIkTL!Rk_y}-jrBGE!t0;moG+tVhM!*NKp=EvXY`yYv|{OWgFl@K3^O-Qkonh zLyPtCJH;=K%+n52tJ+L@aEYy0Mj=;4?nj%I+-rmly7eNQ%iuzSKUAU}C6A@zfrq-M z+==&8zXq+kRLEffPs+mOe4i*fmS=1yAHDmZFJklaBy9fxW;HzyzKpShm0xx(M-OgRcVm09l4* zl|xbEvMPDefodU3L}II9ukhUfR)!yc! diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py new file mode 100644 index 00000000000..895e9e0fac4 --- /dev/null +++ b/litellm/tests/test_caching.py @@ -0,0 +1,27 @@ +import sys, os +import traceback +from dotenv import load_dotenv +load_dotenv() +import os +sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path +import pytest +import litellm +from litellm import embedding, completion + +litellm.caching = True +messages = [{"role": "user", "content": "Hey, how's it going?"}] + + + +# test if response cached +try: + response1 = completion(model="gpt-3.5-turbo", messages=messages) + response2 = completion(model="gpt-3.5-turbo", messages=messages) + if response2 != response1: + print(f"response1: {response1}") + print(f"response2: {response2}") + raise Exception +except Exception as e: + print(f"error occurred: {traceback.format_exc()}") + pytest.fail(f"Error occurred: {e}") + diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 5bd78b33f70..8c094613140 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -213,7 +213,7 @@ def test_completion_together_ai_stream(): except Exception as e: pytest.fail(f"Error occurred: {e}") - +test_completion_together_ai_stream() def test_petals(): model_name = "stabilityai/StableBeluga2" try: diff --git a/litellm/utils.py b/litellm/utils.py index 545138313b1..6d828b26a9a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -28,6 +28,7 @@ supabaseClient = None callback_list = [] user_logger_fn = None additional_details = {} +local_cache = {} def print_verbose(print_statement): if litellm.set_verbose: @@ -138,12 +139,22 @@ def client(original_function): def wrapper(*args, **kwargs): start_time = None + result = None try: function_setup(*args, **kwargs) ## MODEL CALL start_time = datetime.datetime.now() - result = original_function(*args, **kwargs) + ## CHECK CACHE RESPONSES + messages = args[1] if len(args) > 1 else kwargs["messages"] + prompt = " ".join(message["content"] for message in messages) + if litellm.caching and prompt in local_cache: + result = local_cache[prompt] + else: + result = original_function(*args, **kwargs) end_time = datetime.datetime.now() + ## CACHE RESPONSES + if litellm.caching: + local_cache[prompt] = result ## LOG SUCCESS crash_reporting(*args, **kwargs) my_thread = threading.Thread(target=handle_success, args=(args, kwargs, result, start_time, end_time)) # don't interrupt execution of main thread