From ebd763287a936c02dc37fd8105516eb541e37658 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 21 Sep 2023 14:28:07 -0700 Subject: [PATCH] support default fallback models --- litellm/__pycache__/__init__.cpython-311.pyc | Bin 11116 -> 11116 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 47622 -> 47622 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 119632 -> 120297 bytes litellm/tests/test_config.py | 24 ++++++- litellm/utils.py | 70 +++++++++++-------- 5 files changed, 61 insertions(+), 33 deletions(-) diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index 0bee58842316ba1befa0ac49a468adbc926a2b4a..5f5a1a503d3374690de68d31c6f30a5a760f0842 100644 GIT binary patch delta 19 ZcmaD8_9l#LIWI340}$L^w~;GJ8vsOe1}gvn delta 19 ZcmaD8_9l#LIWI340}#}&+Q^lp4FEzn1c diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 9ad726ac531f3a312d869d863ddb7567d5829471..420ed270f759e742e8f30cd89701486f2eaa29b9 100644 GIT binary patch delta 21 bcmZqs!qoPKiEBA8FBbz4++Mel>(5RAQI7}5 delta 21 bcmZqs!qoPKiEBA8FBbz4OkT2)>(5RAPpt=a diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index 7c768446dbbe5382e38572b490e2fc1303ab43d6..5286ad6f39686d4f8c4856785cd21ae649344e7c 100644 GIT binary patch delta 4302 zcmZu!3se(V8lHcWNyvi)3=jw{1TX{?0Z|0O$BK&5T13TGTr~zJJPIUqGKktB)LLtO zVxwNQ?Uwdbg!(A9w&S+7TCG*P+cvoA%5n}p?Y^tKZQAOtXZP%O|2vVQb|;f>{`=qm z@!fxB?!6P}QGazrZ8#hmsb%mWC-tt%)ZK=7c=v?8Q<3K8vo^ugM;ibs{IzM2;(<)TBMSAXj3TuCPtIaJz$Z3n+TH> zFh|lR!yzSnEA^*=LIED>WIDvMz)K%xzy$?7A$4VeLkWc>lnd?-cY=@Q7dNsrqhR8f z`1ETS!6-gR+u?(+>;Z;3g8%Tb5qz)_(im~Uh=yUtGQ!jCMseY&Mywc>eM%qjOP^cO!v~PQiBRZ2dQmgQYv3OdD@QsQ=F#yZG7_33R*kfwR^1)jqeTZm!o%h+!jCb5dWAtP#HtJpz#XGr#Puxi)Q zF;F-yEn5!hEDT8SLoQ8i( zLFRxN97}Adp%~1maf;s z8#M8YkHVr9*&%F8jW%G`KE>_iVXx>$cmR^66^*cndL3_ql$3X{E0YuMac-uRX$0Xk z2{l2b%1(JVQQmg)STk1Z3`<^eA=h9toJ^L-@HS0?e@5_9pn-*VDV!#U-LMRD-|_%X z6S8e4zUp^b6t_URs*onBq)Bqfdo6G+K_Z@KV1$=wcx z@JrIL9U2;5ph3GSyiDOozG0S>pm2_Y%u8k?d-P9cM`&ZIDo0--tz z*QDYdFdcx9bvxltFhm@?po057_M9arcfmUCkV6wQGDD-k5*`2_L-r z4moO6ZrA#>xXNk#?fn+jM&Oxt-Dp-ALuTxQ@rE&X*y}VThd@dOO|XQ)#V=wcg}nH! zYS)XS){IY2&hCSRv`7qW@acq9U!+fm%UF_6=ZhS*Yd89gv3Nc0&< z27w%-euI7q29=4TOlk>a-S-m``R#h2TFCOLyA8Bj#B2d$X*(w-k5a`Vt9tZ4w=|xNcQDYZ@KR9^TdDC0YC7<=mTe zg0N$*zv8cx30N24c}m@F?o9}$ObR#eMDpML(3q(ZoaiESAPi|n;(9Zq!mqoH_R)t3 zU;M`pz%+%RXGz^LxL=7D*T_2uU`EA|am4L59bqT+DyLrGQTU$18#IBzwYJ99A}^h8 zXR}MVfjvW!^2+Y0ce?o|m*9=^!z~ke;28K(V#Cs6_-VstU=8UUxh7{8#48%|6OCQj&|8%Qh~NGu8_7Ij4o zT2ec@rv@z(dYwT_Zhu11Qrx8;G+9CnL@$P6OIl!T#eii|(6T5Hvj~YMrEV!7{C`B| z4j4;=#!~c)Tnd9JdHn^!l;Tr!gDE9}*-L^cm0gCAGBPqgOrci{B+lqxgMeoTvM?ui z8E*}n?h7&6$oNqhPO)`6UoyX94yH`(G7P5J5KJi{MiIRby4|M?f%H;52gm01&j^k! z>3Zl3Ti#XMlqi0IS1p+ zUFutSgkq5tUT?-TF^NHwty>J3ru42spr?MytzlhEh*5G;qcEt$3bLAd7>r4~8e_i_ zWABB5m~p|Faa-mNo3i@z@$NiQ8!*)knCd!$rn-Rs-gD4k+!D^N!N{21`!GIh=krUZ z%r9m>%of~<7YAk6y z3Q-XP=i}xuE59UFufmYOs9ekPYT`Ho_RK9f73p!zw-5Xnc&J@18b{g;!z+F2=NO(N zuN?uGS+yQ}dezs=yso+I100(6EEJ$qCdcQzuWk;VPS{Hjlz2qCEpS zs(30Vk(p;;Dp!c|pwxZ_>`HD9$|kArJUj<5k<7h-yAl=IZ~+#>4r$;5Zpd*%6@3x@ z039xRD+(x7P^hFJf1liATRGO5MV<% ziV~=3U~MJ{+Hk12C}0$uOXK5+D53}JhO;ec&dHG;TLo-&wc4Vkwg21@0@T^N-~H#G z`TzO;nfdR`wU>BZ3_R=&>J~up7U)ai^6u4-A^BB(A>7_C^$7`zynGiNJLDC+gz}ae zp&D`-dD$s%@}@}zEu`EnuW=SSJmpXFJ);pTVxj6Ei}!R`r?zLILXUiQB^{|Ob`nZwc<=~D3hsqNfeFGWLD}7GbuSs# zxJ!m*kKO}$`C-vxJ+XoXCSiWf*r7&OxLoF5K2#Hg4daI7&4M!7y=tgNEm=ddyL?Cz z1dnw?)KIM&CsrO41>q(750=hs7P$VZMH|*Q#n2t)8 zU4RjAutfzpX5B>A3I~T#5L{Ec3sH#p1tC(#fF;&%8+&p>uVq@mGA$^$jHX0ZHUnq% zYe-TPe}>8^DQFg~c6PcH7vt)3l@#Hd6tg==_s!T@qI;a~+i{XRM{QpUBhaQkw+x+n zX)le`q0W8)-Fob0Ka`^st!&|gNHaHaA{=^D_k$QM;<9>Z4Hk&#V1KQ^99(7N9>Nt0 z!s5VvIK&n?FwO9tiHx3QdmMNZH`Mu+SP9g!XRFYRn`)AbwZKuf&54wFK2!br>Y934 zxkj^fFme#7B;R=FM@AeLp-VE$BYt-QdYm_scG2if=Q z9Q9k3H0~)9?y{Oti-m>}l+Lay zW^crFwYwJ4-2THl*zG;!I#W{aaZ|?`a9_$BEVLeFv6t!}5Q-`udt-MGnJ=4^5+UUEaL-ocyo z^<48YcRaild`gEFFogT}u}`+60@dn$9^{)_E^!??#3v?AF0E(5iGJAF{U9Vjq2gu*e|7Obo;17f7A+hLt~L^Ykk!Mw^MPrYZ~g6 zLu~OrOhY$&Y#(Y`4)cM2&EW`#+wl!X#RD3otHq&F(}MYsH)t_yOgS56fCK;DDVZGj z_f2W%Iu4&xy1H^Gpq;mWK`}&9>z>3Ii0z_x|MqBh{3-O}Pi%V|mP&snqc@qq4O`V^ z`!OEa&YBNk26nQZ1Mo=KsQF#B(2F=ET_w4SM$-xnAsFayrT;(XsdZ1|9hfAKezVX9 zC7V6RkjQqlBhK6OmeruFmMxtj?1+q5gtE8WX-^s1%PU~3^cYWs9V1^aaerEA4)KJ@ z;lqcVY`jCZ4$I9Zk4dq*Dc+$XN8Ih{5y(Qb`3@>4&RN-EhUCb|JKUH}f;-xqHZzue z!eG|X=w6(OmVVJ}aVM~ui4;ak2P~M_$XlHxIYLho=9?g?j;D4Ik$N*4Gg zA3iY1CQM24gfUMCa;%memKB1>q7fsR^>iRh%#3EW2Wi`-JcA?FFL~aNp@^>1Z>Qk( zH^-a&BZ>zf_SG}k7!|H)xv6bpzC4trj~>DdeWq2p%04}WnX{B9$$?VB+rHuu$qHw1)Tn6p^A{%dCK978#nBp)Wbx zw7=S$rD;j`jYJg*5u9d<-&GSq#BzzYyM8pNG_RcAO zYpySkfIlI*F1{I=aQL^#BE5)a1xI0%ZjvTcU3nDc6250$K71zL6j{e{q~arX<~XuR`}uK< zG77pu)2J6wEg7gXT5Wz27Ll$NdrshSOk@!!G1VCBk>&_%zGjP0qTeR#J%+n;27=q- zopd%9AuRV4#*FWvt6@0Bb*s?Ta7nZAuEdu-#zTT^@EE*bO^Ia(PoX5vxQ)7dbRNSD zdao8JdyPAcPh4mJJ%wdcV?F#jYbqJMVVJ5NwY8Ows*SE|6R7hxTb*NLP5rvGs)oA8 zS|?AIv?}Vss$WJ?gmwUHXN-2XKhM;cF<%VPwRa(9bbrFIO~sZP_xg(4t`#dy$5IO= z(kCpt3+uz|rr4lB&^kA$Pi05Duro4(?t|G~Qg1|FAR^BnkvG6bzCst9e0KR2*mLGV z2#wiU>ar8Zv*|^lv-8Dwa*FiyI+t&r8!o+zBHdh*^qxtgN|nw#>BksW@G4G;KZ@%0 zR}ls&jwm#hEHK%Yjhodb13?5BNCecNEL`X{;JdQM19DHi`8N3EE+}ML(nWMNNhZCy#bz~W6 zgXNz^!DvoQ;an7Mlxa~~$eFe{N;KDIbI4)rEOHDnTux*c&tkeXh2)P^^E()$m+mJy zM9n*oL%;-f>3uqT1!lQ`<#<9}dx0*rSg$_!0X{{G%r|&Ghov0KIB1vj8+=rLf7Wiv z$`PIy+R4ffNk#kC^3S2d^FiU;TU%6Rn7uY%+UyKYCUxo8S2i@%Dx9gpDJo;zFCyN& znnZr&xlXI;7x9HLeq#tSg2MmdT9y&O-bq0r6cK}vHU*P?8D09WsxDJtTtQHv`n}Ky V_O}40SjU4tUALi}<@^CX{{yrA1PA~C diff --git a/litellm/tests/test_config.py b/litellm/tests/test_config.py index 0df5928f9a3..6a2cbc238c6 100644 --- a/litellm/tests/test_config.py +++ b/litellm/tests/test_config.py @@ -14,6 +14,7 @@ from litellm import completion_with_config config = { "function": "completion", + "default_fallback_models": ["gpt-3.5-turbo", "claude-instant-1", "j2-ultra"], "model": { "claude-instant-1": { "needs_moderation": True @@ -26,12 +27,20 @@ config = { } } -def test_config(): +def test_config_context_window_exceeded(): try: sample_text = "how does a court case get to the Supreme Court?" * 1000 messages = [{"content": sample_text, "role": "user"}] response = completion_with_config(model="gpt-3.5-turbo", messages=messages, config=config) print(response) + except Exception as e: + print(f"Exception: {e}") + pytest.fail(f"An exception occurred: {e}") + +# test_config_context_window_exceeded() + +def test_config_context_moderation(): + try: messages=[{"role": "user", "content": "I want to kill them."}] response = completion_with_config(model="claude-instant-1", messages=messages, config=config) print(response) @@ -39,4 +48,15 @@ def test_config(): print(f"Exception: {e}") pytest.fail(f"An exception occurred: {e}") -# test_config() \ No newline at end of file +# test_config_context_moderation() + +def test_config_context_default_fallback(): + try: + messages=[{"role": "user", "content": "Hey, how's it going?"}] + response = completion_with_config(model="claude-instant-1", messages=messages, config=config, api_key="bad-key") + print(response) + except Exception as e: + print(f"Exception: {e}") + pytest.fail(f"An exception occurred: {e}") + +test_config_context_default_fallback() \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index c754c26ea97..51e0ebdb0ff 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2795,36 +2795,45 @@ def completion_with_config(*args, config: Union[dict, str], **kwargs): models_with_config = completion_config["model"].keys() model = args[0] if len(args) > 0 else kwargs["model"] messages = args[1] if len(args) > 1 else kwargs["messages"] - if model in models_with_config: - ## Moderation check - if completion_config["model"][model].get("needs_moderation"): - input = " ".join(message["content"] for message in messages) - response = litellm.moderation(input=input) - flagged = response["results"][0]["flagged"] - if flagged: - raise Exception("This response was flagged as inappropriate") - - ## Load Error Handling Logic - error_handling = None - if completion_config["model"][model].get("error_handling"): - error_handling = completion_config["model"][model]["error_handling"] - try: - response = litellm.completion(*args, **kwargs) - return response - except Exception as e: - exception_name = type(e).__name__ - fallback_model = None - if error_handling and exception_name in error_handling: - error_handler = error_handling[exception_name] - # either switch model or api key - fallback_model = error_handler.get("fallback_model", None) - if fallback_model: - kwargs["model"] = fallback_model - return litellm.completion(*args, **kwargs) - raise e - else: - return litellm.completion(*args, **kwargs) + ## Default fallback models + fallback_models = completion_config.get("default_fallback_models") + try: + if model in models_with_config: + ## Moderation check + if completion_config["model"][model].get("needs_moderation"): + input = " ".join(message["content"] for message in messages) + response = litellm.moderation(input=input) + flagged = response["results"][0]["flagged"] + if flagged: + raise Exception("This response was flagged as inappropriate") + + ## Model-specific Error Handling + error_handling = None + if completion_config["model"][model].get("error_handling"): + error_handling = completion_config["model"][model]["error_handling"] + + try: + response = litellm.completion(*args, **kwargs) + return response + except Exception as e: + exception_name = type(e).__name__ + fallback_model = None + if error_handling and exception_name in error_handling: + error_handler = error_handling[exception_name] + # either switch model or api key + fallback_model = error_handler.get("fallback_model", None) + if fallback_model: + kwargs["model"] = fallback_model + return litellm.completion(*args, **kwargs) + raise e + else: + return litellm.completion(*args, **kwargs) + except Exception as e: + if fallback_models: + model = fallback_models.pop(0) + return completion_with_fallbacks(model=model, messages=messages, fallbacks=fallback_models) + raise e @@ -2924,8 +2933,7 @@ def completion_with_fallbacks(**kwargs): # delete model from kwargs if it exists if kwargs.get("model"): del kwargs["model"] - - print("making completion call", model) + response = litellm.completion(**kwargs, model=model) if response != None: