From b28c05589628e7880977b6455e2baa56d7c2c2e8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 12 Oct 2023 18:24:09 -0700 Subject: [PATCH] feat(proxy_server): adds create-proxy feature --- litellm/__pycache__/utils.cpython-311.pyc | Bin 159871 -> 160512 bytes litellm/integrations/custom_logger.py | 19 +++++ litellm/llms/bedrock.py | 2 +- litellm/llms/prompt_templates/factory.py | 1 - litellm/proxy/api_log.json | 85 ++++++++++++++++++++++ litellm/proxy/cost.log | 11 --- litellm/proxy/proxy_cli.py | 71 +++++++++--------- litellm/proxy/proxy_server.py | 83 +++++++++++++++------ litellm/proxy/secrets.toml | 29 ++++++++ litellm/template.secrets.toml | 50 ------------- litellm/utils.py | 19 ++++- 11 files changed, 246 insertions(+), 124 deletions(-) create mode 100644 litellm/proxy/api_log.json create mode 100644 litellm/proxy/secrets.toml delete mode 100644 litellm/template.secrets.toml diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index 99e08f1adf4d6c7e15ef562b90a543156ee0b624..e2888030c26bf0db34deb08c63b4675bc9a64c8c 100644 GIT binary patch delta 21771 zcmbt+34B!5_5a+LnaMKQNytVv5+FlJAnYI@0m2%>j_kvbHz5N_Cfu3GVjv)>b*tc2 zpH@&P)IzmZj2g946|MfQMN#qhqmR1VsZwY7Hff4=8UW|FA=_w)IW^5wp}o_o%@ z=bn4seSH0P{O;e!C;vDpDZxfR?+?m+wl5DPXNz+uhg=X&&E^xEW*b#sitIRDc<0cc zxoVg;I$@*DyTVuFo90`o4r`fNtMtrC)SQY}`)#Y%P|0-PN-Dcc%j&mE?~gLdU)wXk zW0js2ZnKWsc{JNBo1VSTrsw$9-xFiA-AjM0-zu-3yU(_2gP!MG*`VnI_Ssj}bM1Vp zUEmwjr?$>zg;ZAbjWQpX6;s*3@!u$H;KC9rEd55=MlKsfWrNkan5>Q=lq&m1Ex+z2 zLO1C)eQ49>=B8Cm!p7~&srE2mQy)T`xvYZ9D(}%~!t}@bd0YI%N?kO>=)?DkRRMhj z_1dae5pL0U=%e7(_L4Nj^( zjcQDw8f~i8o~?GoZ7z+tE)$ho;xi!;W}c_kVgvcz~7E=}LVj{h8&v zLZ9usq9IYAL!GwkyQxO+;ph5xHrVxfRJw~wYl+YfeLgX3w_ZoMM_)jArM{4GufB+I zACIxvx3YtpEg=G~(w9>9)%qSTSmxWQFQRdkDYJvRzNI_4bzSs+;r;RK8ya(KP>h8vcN;`)2rd zLc60(C0Y5X>}I`zsw=;ZL^M&KNCa=B0dLV8Dfd?0Pk5WYiO|qD6W*>j5#FIU6aGMN zA-prf*}dQ3TWgW;puWX-h~y<{mUrnvYH+tNi)jxL?QQBMXMWP*aL(nFvlThTN4Ul| zdfwh=c0bg`5}SKpXJYJ!qBA+ZS~!Q2e#&ySEMc*zP{$KWg|5ydY@zg}uFTxwR2UQl zQ{@QM8`2H+s>N09s){_?jOx2YhI+}hC7&}<+f4Lq^tJe;H{|p9bvc=`n$()aQcB_(LAg1=9;Luy9O=zeV?Lv-n<4x!n_hbbbC9sE-!Ts+3UC#({Ox{WmM`wi$|x7BncuM%{VPBT(;ua7CGxE)^+cw34hwlXdWn^z(Bo*p7(g+Anav}Q#baN!VZs16tA_r|Em^D}mC5P6(0{*{ z$}&hZoq-LTeDxu@LS2zF!pZZVtWM^PuV>9)gN`wTJHb++24vI&{D94XCIZtL3`w82 z*>p)?ur<&U^m%ASMCS=#f|o_lrJ-tb^TeUf#$1=6l^4o;M@&-72ULm?>Z$=V#MA0m z11iQd6EGkx6eej4hD64R7$(USx~0VM4kF8LiyEF^DxOqJ^Dmvv@{T2)t*e_^cf#41 zm&`%#^;_A}Qa9ChYd!eThCcdCd|6SMcuDOl8qAATZlx-6 z8=$X(>_8^qS#_ppnKiQ=s;oFf2Ki8NRSsL!q*m!~33)E}$qj*^Pwu7auc;4;CzHSi z4V*J`9X*=~^?~NrCSS-OXsIJJ)JIoQMP37d%1IU|oA#^m2r$({1Bv4A@xeBTfC?KQCdPY6%Lv*fQPvOW#1?G zM&&W;{z2JP_4z@SLwI!irhvc2OjwR_YNQNEmZTZiY#HsKY*qsX&o8Y+kF4+tDHi0d zNZkUs4PdAn2M-*;WI?NTq}B#DZuGZoj2O{()r*6Flm`~nI|#aePD((Ee|c41Kjh(z zcH5@d?jk+rprhU9iBq%7M#VY8Dxi7YEf)&BMHevc)k+=VM(QaSytGC z_`8tFZBQ`?GFSm=FN|}~a(;@;%hfWsXTbT~f`h{^x|}5m7i|RJv!9DAJj?&fD)pi} zSF4RP-qExSBjqWPpyI1Cjh8ZnORb)rtUef))H!wNQ(C6BcGAv-cw2jdp12uOv&rj1 z;^yDvs^Jx%+xLr>iq1imev!Y8D&&MKRE5T)qV#J6X&FcNCdkW!T)bVl&>pkyx5lvPn zMqZ!ER?Th+-q|W?1%~7`YR;&6t6Nbs0|}nVW1K2 zxM)?`Ng0+_N6IaQx>YS2-8>i@u<2;>UG8ggTU1Ca3)8uIt5Nlso}x2RXgOms2z2T-I`*4T-c zfgCfvi44IE--fo08+~$mli$}8l30vpvRBtV8)z%i>t$Q>1~aj~Ef@+kVg3qrv^BpoHj{R!zl|-` zraM&rxGJ%>bHTU^B56!g#b-8x{c#ty(Q4d;Tg7LcXC`Ebj*B*XPVN^8w%mb-%U;PT zKbKQZ>nSt$jqK9@%^v+~_UQ8i2Ok@Hevtdv+Vdqt&Xr7jW9Ybx@wS{XpV=Haxt{|r z0{;Hx7m2p);)}NUw2COak&%BcW5_ER?sFOLvl;F;GKw#1ww%S<7i50KbnSDSO(k3s z$m~arzWnmbiz@|DcoAfNd9g`-|B|b{3tYC-X}Qye*-n=Wq=zL;FNrxlwq$xi%riL- z%6X>1K{?NqOe?2{=Tc@)o}a9pNz~}!Ome~kr{hd^b;$zZc^kAf&`!CJ*r=It@Fo7KBRorRU;loGOWkH z?M{ak5s$1Wk08|rcvL-CUFv)c$;Z{j>QE+>Og;r*?P8O7r`kSk+&ormHsIXH%c%1j z0eM@j20U7XVvD?UsC1qn2m{S_6Z??*$FySM?aZIPK1R!Rsg9Y~q{xtqYJ`8fW)$Bq zGSY;tU2KA}=bs*vqmG>K=ai$#;KvzvjS+>aZdSTcG(%+CIZM1Gvef=rP7x_hD{FVi zgm&5zHYG;-u}`98C}IaG>f#7xsCQnEVXA?W=yi4qN*2U#@IGxD={}~T%;iyb$4=K>OFC^2hbnuM?#6y)4~Le z0YS!F^R(RTo=n*knegFUwP$Y5xV-Mx_x0<2-sRphtlstuS>LOd0qT{xS+0oSh-&d+ z|8&2tcNf-VCimB~dgtrek!}lULiyb!=s6MUY&+xI<58xL%*%+o+$L}Deo*nXN%Qie z6-oKJ+uQU3J<>g7tEYsyGQ5w>!vmnt;r~^iP|%jGJ!u*DU$xt&ZR-&L0Tt>{ZH_u! zn>8|29iiW*r`(s{+ZVX6p!f4eabKbOx;9fVXNI=N^OA|^2Q7^O-Q8^}?oRcfH?g!q z_-&LC5#BNJeZj&m3NvXuc`7faiTs!umRhPfPof&MuwUW8d&tSUm;P8kZd4L(^nOUF zQ-2r$c}9BPiNzmLS6QIj)%>~~V`x8-HoD!e+oQtTDd>4{k7pM>59#ro$V-EiOg&wf zNw$O*Nmf~3ecPEFnrV?7*Y0XhY)|U7nzF2Bc{w3?CE;IaUU!N(H&|^c@2~D&;O^({ zyRt$Hj1H%k>y+-$!rolT`36_^$J}t3Jh8=U(Zc+cp`m3xMkuEdhO#}dmxoq_Jue(> z!$0!8R;u4EoS!nRPqXSp<0!5)$(xtIq^+Jp0l}i_-lnD^C!)a%D42N^~2nN5gW!QUVHBAx(ePtD+^Ni{vSkya%`iz@ze} z_5dDFs;x^ZCT>O&-WZ<+PJF|w>(e|!^p5rHuMJqbZE}UR#bOLIj!w69xNG*AF)e|u z){eUey)&s(kh&f9oZi+}@-L;Xe!VnT+^s%XI^1&JUP6^yRq?Wn@fgcY@V5Hrds`?V zCQVmcE4ug82S_H&=ij3?Z_aCw;*i7))x2!T`U!Z3i!Uz$JPkMtI1P9P@GO9L@l-21 z;Iw4igImnOcTn$t0RIcXmM(8o|5}!v8D-u0Iue;SRmZ#)8lP0{8nt8VWh~rKI`u~;Gs&lDTXH%;>&n}w=d|k7W!a9`)18#{dbWeIUeAv~df=et2{FCS zjI_&IXU0i(aySMlwYEu|cyQfSBBj0E5jky5u})hHrlzPJiu!tCY;5eVSnB1dI=g-l9nMg5&Z6O>A1%_dj)$lZ`7YpL z0*V&0664%R=O7RA*CQQ1Z}=%~I{5s+F(^xYYABbYRZ%fw!S-D-5tR?GQ&f4xrrz_; zh?O7F1FwWSwRJ<|_^jwkkRPMo9|1anncCp>H?>JpUNR~hy!D$UYdR0zsIuzwXE6O2 z(SW_rGf45+mU+MYSy!Me^AJ`WrdL-RBYY=hfpxvKqSPVA3){6)&RpmlqpU zUc;1*S)*)EC(M{UGhTZxR-=dK;uB_7I-cuSojS|yc)r9*S7f+MXjY!kZ#awl1T^}wn2 zYA*RtXob11?YwbgmvDZK+5Je(^_MJs14$fjC^F1u`)Opb{n~?6#E8+coexAD$l(s| zMmftIt^~`R1b)h&<**auyq&-CpNSO>>c?BkvmQcKEE~5bOOQ4IkE)NhxEDNzAqQMHdpXE$!v*ScPvd_1RL;qemyqDk^!1ZGT= zuf?LJUHw8u5nf2J#+}GE+_ZCefsHtm!SpdLyx2gb>7rvk z;%>Y}Q1xJWWl_9tx~n z?$>>ig3XZ%GM#2hK@ZH@j1T#me9gX)+%DrN+sp`h8^|E^9*u;V2<|Y|z5D47z{PjR zGn19EtyKKH^NDS5#=6*6K#+2?`qA#@BFt{-3C(m;xOpQDieCL6#!xw{NVyuhXIRI_ zsLxia&xQ#C<;^H%f&CJx{{w*UD|0}8Ke#F2+*W_6(bF8zeH5tS9eE(?oB=?_W;_i~ zb`L(yUy#Yt$1*n68+(RlA}rolib54%S+%a5O57dl!!;r= zja?lxzInSR)QIp7F9m7Cl#&6dfE4w>mDwcM7p@#6Uh4ex%D=>norLl1G6ls$desiv zsc3!C)TtauHl6iW7^#dZ?NQm+w1}$CYpz)+#Bp{0d#})IA0jEyde!G?_J-;k<2$?3Aj`#v5ka2P~z!Rt)`KG=*ApD@{oKdjDWitNub* zrE(8E;^KgrC2MuzK$VD7NjFaxqtv{c^Th<^zqvROj5h5d-?k71)^5Am7moLb0-Jp; zCzU{&8{ni5t*Q=Svk8?+$_hfi=o>rgq zaIew_rgX;Mo+qp~8%}J{MNw8 z#H`3!U@fXs(~p!`3p-i?f+4fU{K4!bBp2S#$viaq>*0=uH6PMos%eH}k>-By$3Sdt z*mnI4kI@0vp`M2z$S=KeEb$Ur?<-zFZNT?C|9#|;7P~y*84ngvIA`)w#;U1 z_d6Gu8#9i4X3qvM1^91MXO52Q_yFCoZl}r?jEH_@0975{9psc)=DigeiH#H{#iDNT z*8AipWVu2CTEMh;JzhWCGr0ww@CuYj;RUpWTgM)7FqL|bOoS4uWtxGGPBdwpH7C-tt7|^uV<-350=o{yZyoOqFBB1pd!8Zb(V@YbzSG};VTB6We*({ zsk_jMces5>-KZ`+yu9@Llq9Rn+tpfRvv#|58EEx{U*``|%N`k)dNm3m0Za6|)PYAv z)3DDyGDDPiy1L3mC3Mk@_qLE(3bgv`saIHOSq*4M!<~Q*0y@PVQ2L{z-CTjokS(u_ z)Y^m8Za_7f?@~`cnmqslBX^-sBkr|YeevjJx!+o1wQqQ=V*S?t&>pogsF`FPiLHei zq+QlOvWmPcQ7DQ9vg+$odLtp5IZen+3le-|%lE|HEciEwYsYvwvn5}ljL z&E9SP=CHy$-00p>3HJ-vkzTs4k z<5|l1qVp%G#wMq-g^zZ(rSsy8|HnS+I<&YRa2wzbg0IBHkEWt?wrdFi3PDJ+p`=tN zQRN=>@!8ze$B}#j&^rZ*=Xoa_Lt@N%rWV)>)mVrS$kzBCSOrE}AcFp3NNahjsCo?Ap%rn#KY8QH@ z*DhE#BT~9a6@Th;gr6;7B>c}#7-aIEqIRhPd7U`q<$POvsV(}m3fI~jQm@^h;5 z>(0r)9ix%$4!-&KvE0cTv;_a?_z(VJBeI2?C!;y7&hW~Vf1|Q2^~1LorD01p<6CIa z;w&DNflm9|FNjO9G{_s6OMmsac@$&aB@iCk27Pc)`0Z(!(%s$H3gX@^6T?*1yA=bV zi)4dXn!N4ve!v@MK<#*U{D7}9x)ad=pYC>;4(V;7*j6J()zwSy=KFdpf@BBcOH7nE z>;$I5BAok=_=jdvB+v-o!18(jO$~!d)Ok^l-&FD+7rvM2h$FA4pmWoENlAInVF2Fv z;3+MYtVVAlvrXN6ar~?#%9hE16acTNw@?gSknaJorsVGdFA$jNP`||smMyJW9MOW4 zD(y3O#}U+dAHd3(2xW{^!3HEh06;6{I{;{*d>6oL^LLbTXUbnAgVo>+rA#dlG-G`& zm-}U)g%6abliXFr7+*%kvjAScnbGz83Nl{>yry3IEPLWPN}B0C+@0<dgB4@Hp{E*DN8v5aL5)a*QYu4;bDU zF+e;Lw8V}Jfbw~$#%ln6?O1~1Qb0Lis4>GXeByKC zCw8%-gV|Au{BeLjXS%1+NebXB;4}ekbtOpi@tDI!1=P??3*sd?9iJq5fgaqEw1ymt zUe8g3KCvWpksxfbjJ6NDzUL5cr8#lLlfr1u5{sQ~v^;M7GE0ne)q#V<%@%vb zSmXL^G3Zj(d{#o%?7ySNSAc&IoCuppE_x5S5zV(;voXe}*`l1TVZkLuwnfLx5(@JMLQElj6u zYr##@K4d+B3MT;%0{%d-V{i>FG88SSuWysa2RWjk7FI!)5|9OogmK(OoFz|zUwcu5 z<%<}J6+nphkrm@Btd^*!f~+!QQLgAevl?ku8|>N=YGcOMds{>B_oIPvHa!1EkF(I= zB2qJvf(FVN#!qua{un!Ya1pq(t>A$T8PSPG0~VIZQF`9^I#*0DVME&m3toO<#oYwu~ZE` z)uLcNU;$vHaqTcs;#`9CDC5ClqIAh}q%R{dV_+)!N?n3fxe^eSHV^V1`j2v86{@g^ zMjP%5QIh!)%EusgDPS33wc#Zr(ZQh=W-kP8NtXr75ip*|I#lnodM)9CH)~PK^5qey z(<6tmJVEQlm7IYJEXo?BSe~ab76jSBl8iEi~>5W9w)^7qgAGkBOYF2SY#>-y?DPm# zkt2+EE)kiIT{P^@u76x2k_By?36sU@a43PiVt>PSx}X*EZKMr~LIrXk)xFNxJ6TK@ zRb9_d7VpM%{1vQ-kylZPyc%!~0cF5rw$v;=9Kp35JCE+K4|VVFy<5qDQJd}%Y&b$= zW=r2je~|Pq>==6J4At5Mg}jaiy4fh3Cgu<^q!|P4VyAKIOfghQ1 z$yFuK#S$2A+?7V&ERkCS;|y7CX(QK@4%)4F@i`l$&jqk{{Q@cW88{a64A}WB0iEsG zyFOs-m?b8-b|Qa0IB=u!i&>&58>ZGwWFMRCFd}pEFcm-A_19S<&hEN_s{+1n#Mg?U zsW&0LA8-J0voXC^tnL8)(g3iwS;XU!_mx9P?ZL=*AoT-4lze6Y&ml^*-Rk`=)MS#6 zQi_AGmQsLd%d>|bdlH(6`SKXm`Jijad{I`A3g1(1L-RJIt}{HF#V{9^h_wXEjoUU0 zx1bvWn?=<`UNg)lEFAxSYT~~g@KZFtk${wUC>r;!DK{DAP2zIVY5cHBj1vzUW|PPh zCyl>0iF$Frv8q`F#2v=3n#FD6h~aG!%QK^$M3$($$M|)NSdbd+n%&!aqdXvXy4Vq> zd60KubOywO;#WpPtJo*Hx;}0dQ^WxF68N+QVfs1Rp8bhI0 zPcY;SwFMA0JDv^rN~?}0lYqq zjEI+DIiIW=te6Gpwh#aflf{(kjsoA03>=Znw5YZBgzts^)L0Z0g<`g`B`7jV**w6` zKohK=b^Pi=AApqjt`k9VU3N!5>ToHo@;r=*jl*>6USD&dg;qAtJVKF~)*X(FzMCng z2Hm!ddCNyQXk%|hl$q!raFvV7YN^6*+uLdV-!LA$LCm66|J4oR<{0;6j4!sjlO!*8 zd#jhCf7oAW^V$%^lNl5AwaQt>!}~?4E9#m}Fy7xUW{VL<#Q~A)c#$S}V^{3~(P7VG zb!PEGnP{TiXs7Y+tzt?C)+p_lw}2~HY{Y7`?c+|E@&0DK-)i!2;Lt6Hkb|(Ik)>*; zO0SzP$D)A;muF~o8eP)7n@?HY84=%-n? zMOZRP#`HVI#K~`AB$h4|nn*mF)~$66m;y=VO9PO+Oj zfmH`ZX*Nh@&w*kL=&?J-VBB_4l&AlRB>xv2S$5mbr}X=tOIoe^M_S1@iGWC(q(h>@ z^%=F1Z-cHXqvnvjt%icAcK8T#G=C2 zS?K~&V_VB+GmdO4h3n}fIkwUtqwW-=`7oKf5yrKLNyoo5UO6mYBUS&=5wTVz{vI`$ z99d^19~D_M|3EoAoU3OnsH<7K)CyGNcty+W6Dl$7M48TK8IFZ@e-ZO3`sq35=A&W; zjrqV)F;9z6H2!r|yi0?Un>SADhz`0Vmg$~YBtJk0A7X45*iy797-*qvJ{dDtV!IC> z;K)*JDb}ZeI&I+elB*hg0_}XXBesBA%fJ5i;xxW@LY(92n69WX48my)Z-yW}%ta>dk5qKRVr37KKn$IhS%sJw zjNX%-RNjjidN$JRDL?&ysCF*Li1+h|T{PmoqJy=WzIH31L|!f$b2{l1)7aJCDJF{H^`M_f z2uqVEDR@G3Ac)92YBGNJknoD_#;k|MW98AdE78Fm0PiD{kV=6h(KCDA<&RLij;)o{y08$nLmO;Is*$HopUuG{dtl7^>N^JM{fU8b|1U&M>=Ed4 zf-t`Ng~+ApWj-ku8H=72{e~2j-QDtomg8$5o$*RR^|^xT{jT%<3J#aP+HdgLeuL=) zll}9~mk!@Q-`Mt~*a$28q-aTB-VDa^J{r{=KGXIv)2KqkghRpXz#f6rIev>+ue^_n zt-$W}czoPA_9_KLs?mnuz;HCV6Gaaad{jUM8-H*IekU%q-pSmL!Y%;sBjKwy><6$f z2^W|@khTK(P;Pg*T!RrJUVv$Dfr!{O__Id0#(LaNO(GSMUxCbj1N!4JhQN%qgd4Qj zAB+4{0J~vNV^jwoPh+ItqUi+yG}=sB5(>1=p);X%OpQmq0RUbTjSEo65p4326Ue@7 z#5c}rA?Rfhd- zQP-h?mdtOx&i5Ah@-EX5rK$7aXURT0#Sv zNzL?~7!ik#>3+jey01#dJ2;YqjS~oY+B{m3aR~QW` z)7LuT6}A>LV$;d%?{9d#48VcA36Klr!I=T!c!POowu=| zb)U`Ry=63%@wba&g%b>w9%I>OVsPeBY9KjGaVdzJ3OHx%|BU_A*FO_QVr$o5KNIgb z3va?4!iO>(1WczjjJz2IKQ#XOr5N17`pdTKHp-Ld5a0)z?IpG_{RroGlO59a)2dBmz*fjN#Q5CgzBE_ zdgkw9k|U{s#-~$?)vuBGwYaV0e#)XR`?pgtj9#H&tabo+MjjEb6OQu5BA3l_KYC?C zPG_Lml5q6ki;*mo+>Uud)~1usg~1*+kNpIm{wGvxvK2>v9t}1EST7)VI{z}Ak6~aNyYTZ}bv zT8{Mwgm_cU(wXr@xc6ragNYe^zwwQ)lt!BtptQ=vPflH_~;?H9AMzjI!Fg{Jw3bZPl`s~H*t|95#CNT>x z%~JSLwz?f!WhLzZT%GlZw}HJ>hcT12tp9*xZ3I*@AAs zd?uSNKXX$EQtm_URe%noC{HUC<;K)JZK_2i%%lX|-E~u*wjrkWU+5g<(bnI6Z3JP| z9r4w;CRkBhJD<3Koy&?iV`W-;eFmvY0@Kk(U-n4(i7~lQ%cq0e>Ow6HX3IJYU6X*BJvi*F9mNy5#V1&L5X&p{L<%3 zv}RF&Ylm)Ors%!~FWvmbwZ%|?E?9qo-j8=J9;7un=q1pTLuvTqhB;L08ibxHm;=|0 zHa3Zhyu2X)a9m%8j4i{oVd7Hby>e{`T{$`MPL8p@T&uGF!HA!Mm-E%p9@=~&d}qc|d~4Kwc%Gu>gl!r!e${zZ?~#>sJ7 zqvdse*i|@Qdoor$VZ3!I?L8r5_*Bx+kTG|v)<}xlIaNEK1OM1`(Zw=<{dD#OP5Yd> znG57&#<6N`gp;?OM#HQoMV(;8PSZvWgp}ztPPd-CA-svA8)Lk|#D?D?$%Dq4Y1;4{ z@T#wrF@a{|*fgzJZ0ve&nwIP6ajpDKE7dmuTx^E&F)^O?Q#{L@z8!Wi+kUMy>n>a@k_@t5}y66yaW9 zK&$yEXCuq)*MnYuJ$_u&(ERQJc78zfz)-Dk{jL~bfw`sr0T&qd8P~-&h zoUoQ7#=tndRvRO3HvVU=HiGQ{?Metny8HFjX!d!aHIi39TUG;9@X_cjNM_|_1usUu z=K;J19H_@@;1$$+6VOMsPsQL$4xw#j*A45mae}Gg47XM%M-ew@qk+MvkFR2^V*>SR zo#NdoGwH@WRhd~glYabfBGpMg*g3dO4~R69ZfZ_U;XRIFAu{ZBaxYOav!l*^R8FGW z^g<@U?>yOXJcBXWiJgc^F&o$>UPr^zn+iHQhXMh*CUO}j$xLH)`55ikAYP;bTKxV1 z-gH^qdmLE_yjjzER|c^`#@J+y@$7o7C>NSwrbpg^RMKZTUXHGu>H7D2?NYpKp0YvP z-jPG~&f#H_dC>VNlJFTD4;D&Fhvx*rJZx*qFSzj~9IB(VE^|51v5-PP6A)zwwi z_4={n^Vpw`$0pvFkPv61-<(Y&e1AW+J8_UWet5)tB2{C2dd4KGBh1x)+scawv)Z@Hw`#2x z!HJXzdmyk0)APSz)?cU(3K3pI#J!qrrcKY>Y16ZObqAwtwnOyC`mLUs`_ z)d%miuk(txeXa5Z+e3L$CC$ z(x=k6y>vyd>ibMXO`}1!>7*z6^nPu2y@r!*XATzJdlmpW0*e3yAF1M0Vv);k`y*MR?cx zZEJ;oAq{h#ewn_S{=Qzni0W_9Jp^ym*AU#rE9<3|t?Rq8D|hSl)P9prOn1=ybsApj zKHqd7*y**UsBJB^-K?)8^jrKkTB7sx^N8W~G~lhepK1+#1HnD|MuNBL4FrF#Hxj&E zZz6by-c0b$uw-}km)l5vlkYBF`u5U#1!$K0^dNP(+c%JTZzbM0TfD2ovoq3G-V)uE2k2^+tI9csAJvVnYX)->IA*+Oov+C!y+NPHugfV^)u_hD7m6`zRs5>b$@J_N zvXcH{!g);COlkDCc-H!x{DJiz>GKAfn`AYiw5ZSG^Qq?@8JQ|C;muU;?UcpTBx?Xl zlrwS2tWp%`0L%rLN5FKpc%`>7U`@eEHF@~$u;zi;ZcCo;3k1CDd?g+a4IK1%wy4I$ zS>>3CJ4()=zvN5;welPkClKf@3LYL{v-&)7h9!!c5ThyT!lYmLW>cfvVKKV^Brs}( zO}A4<_y)@s^Yc++xu2Ek#Oc04h#s*Wl7h`0i(q#mUpzp`vu^Fd5>_m5_J&6 znM`xje_&c|NzWBGt6S5(vsuwsfJ(F+3yR|a#slO7u&{aLbMe?;ZJ03trKV;qx76l> zt~)Y1wPcokis@{w+u*AY$_v!8%rYm>cZ#|;bHZrW`qdx`^6ofGeb%8u2iO460MMxZ zmYI28WIl*kSpEBNfsFahy02leFVNE56o_nuySk=jxdiRI+U$?SWL1_^BFdE~XSz75 z_U9B&W)WaW+9u4D#Gt6?jAEutp<619$RfPyfVFLiYRMTQo>e7-D`$aDXEdp8ZS9QO zPG^5{vJjc{StZGm0Ju{f8N4PI#3UY#4{sM4YHaRf=hvfZBLUN{dxKtw<0R5Ha5>x`bpzh0CW+}@h>e$tpDm#A_8SX9l<(X`A6I!IdDd^ek zlXcAjpWI1Z-c--!Po^dODS!414?UZ4_05ef4Zfhixv7@SQa{}Vio6P7G5|}HP5za5 z1b9!~P>^r&a3vd$R|<}Zk-+X0rem|Wq1D%^$pAf@c7M><7{D&hv+TQURfh`Qs%==j zI$k)4xcX<|lpG%DdP0&n5;zlGe5Trm4JG8ehn0vc)oa5tlA+yZOrvGD1F}hdGi-ig zDM+y1-;5G3;H@a#0$>2_QR{{mIC(xL>c-*e;uCdX_lCf-h$D7A z>JGc|rVdb(M;6A|Lt`la$iiHAjJ%WZxXQG*lFa2@c)kN*r#donNH4bbqLR^p6Tl#> zQnFlqJ94J;1S&VF8nL z(1M0hhQ8D>iKaOj?T4EK-_dTG}n6@ z0@81k+Q7m)j?Lf)a~jGyFWJ-tO;H@?EfBj1Fko7J)L72+-xf9{}}&|x#t ztLvUR3W@Z3+1glVrqp{|g00f$@m*Z+Yr$rA$6FR{Kl&d4U~&i1TlnNH>Xvg$2V!Rg z;t8EvU)hzIP%f5roipJ*kx-mae9}f>SIh~mNM%>tB2IK2u1FIZ|NQpv zakfGECu}h(#Sx&&Cta#0UlfxtS5T)D==tr53)OEYU9o2VINQ;btZ7BIqoW1NMRCgCZ6s(QIB3gvNCsS6Ui9-Q1}cUsd{Wzz;aF=ct5nm=ux^A{){QG2JAmpzJNNE`mp zQw}XB9(f(QQTjE&V>eY7x*kXA2?E>dczqkI%hR!z@>zg=0I;#Nl3ms3&SfQGi^H^D zMaydhC_-S{#G^)*huJ&^q_+qJq6tJ6WZd+8aY5IL>1(32Z{pNf7Y$P%%&c>)X%ib9 zD&^I*fg7BJ&ObS}&9(uZ@iQL!ml$Y#uvesFuV$#sHSxyPm0F_uYF2d2nMfns95SxW zK?Gb@e@Z(-`j(!>e==*HM=T3A+-n3X4{aUE}NaN{yw@e${8G@c7L3;usu51 z3*ln1A#6~B1A28#=_%8|VSL@uCUj?JPm`@J`u=G3bkPuD4epSPB=z8qL6h19QOWA* zcYkW%Ga1tvVDY_6$of8HvQ_E4fx@M{3v=W2G`}r$%A>~uJzdY}OGzIXrjkn|<{10F z5@{vvv2C$<>KR|pj7V@ZIlR7Y>g2qE>hW5a%6v6mMbA%A-{0;Ut|x}XTH0>``jGA4 zHQfn4Y1;Ci2N>YISi88_DH|iG@8@SGo!7%K=^6z9#+uhfT8Ex^f6hVjIETVs=l;Qc zpVx`|bNkk`$F;@vI6s9I~EC6U``GEBrR95ATZo;s(J> zIV2>j38aE|p3KpL(^$cbzgBATYUiTV%KU>w<`78nk2@8FI(^z_TMg|y5yGSa)$RqE zB1#=vkg3`hq?S|Y5t&ZAXtV3XdOf@7d3dkq_%_yaO;1xp7N)DZMd=y(h#s}-;ix?^ zIGuS`I~F>D)3YrG;vq*L6r9i$C9x$^1W(m$Wl4G7%nEN0~N7RZX zBgLa?$CAp4;kbd~0C;~hUWN`u9y*R$k>u;>1V_@_d8lufEX+i2(^2oOU+=TxJoteb zOQoT_OVcxTbV1A@ml2@k2g>x9u5Mg9cBtvxxWy|eG&l|n4q161_AiG~_bc_u(&7qs z>Tu&6Xm*|vuJr33=5yN`@?f=U&$9IsiY&%4h8aUAB038+`pl@N<}H>JWJW77#BD%v zdRtm3W|BTNd3l!Dr_Ntqn#Wwc0u+u;VkxNcP#n}0lrN*wQ1>k#oW;WY1kdalSEGd0 zmNn{wh4iCrq6BVfkYw+4dEjq{q7;dOZ3y-f z)E7Yc0N@RPHv!%PfC$XgCf^oMBTl)LSK?!Cxvy8G7mms$9C83|E98sAe^Hcl|B7Pg zEo7SlH>nR-xQrOacB}F;K1cmW-v)I*xWxA#ob|al{_o$HXKBNA$FkzfR-bl^iVK-KEOB@oG<1 zqH$%hNK$(%qobp@M-$Zz>bbgMNhxRajBKk^+Eybwju2(}ApjyPN*u9L;v_-=`x?}H zQ4SrKkd!bT90AKlG_rEcpb{l8B07ydCyxU_JLOro8XCH7>dpG;8Y}e_wOYS;V0uKU z<(EMDGk{P1RnK-ZeZNZZ4X$7&|AwZ00G76*6p7oQX8lx;tU;-2@)eGVaP=0jP6M#U zAaLkW71ET)e8tgeB#S`3rjD=37UR^9E8OXy&uX0CM$$3MS) ze_)=vfAwt%HG*0g=i3v@m1q5wju{oU=i;W@Ym&7W5;b~wAvtcwIL8ZFRmn5T950S? zQq_xP4yt-_TolR`!)6vl$wkCbr&e1_nU1kwa~!~UfD(Ya0p14qBfvKRHq!0>=l*MK z2~>FvH6N!xfxDF3pIaL5pwnp>K#b{4oi-R}5)c@@K1v;)oYr;1f0uCni=O2{b@Rrd9VUv1@+q0ert&#du;I8ArLgs&Gav`t zcc7LD>_dqc58j#=FO-B1pQ&(ITjy_DC!42i#QzJAe*yRk;0FR`RD-X{;-y{1v=(n-NxOuMK+J*m8o^9C$ z+?^L8h4^>dtpjpw(AVH=^abTs8LjTvmg9C|ay)3?4!u(+U@1X;w5?G5vdekN2hlF} zcvuG6p#J;P#ys%0v{N&kl=)jn#6i^mC(7Qrm9IO`ky@cSbO zsYTUY-6Y0z9lLs=5S^;v+GA;>sM?IT-obbpy}|nR@<|naT`sA{@au*;%BeY7)n4}s zrS}G3e?`d{47Gs43-fD5WtbxnjBf+@*9Rb zCJ?m|>cSi5CXb;e$#Ea_m2Anv5ouk&zoFEgvkWbmt-PN=IKPC7R+XWKi}TcEf@74f z9(CP>YD*i`u$#+8v|4fV-_}5n z!(2lsv}n6-yDzI8*onaNwD_d;o=@u3!`rp!=h>9O?`ph#QM9N~&)hTVKhJpJy(1~a zn|kl?5iBz7qgu;hTf<>Y%!PbBYvX)%{k=mwxEMjeDpJyerG+_3EVH%8lclev!C#NJ zf;_o^#3C`TG_P?eGqHy-5Ze~ER1f1ZGQeu!T|^)dM-6$I)~XYcN-<&EMwk`ium=d>?u5Jz4bonN0lqsOh$l?0v@klVgT6!z&m*pN)iC?5;;CT zBj|KuR;0a;VNrFb{SAlKQfed;m40m5Xmb0h$0n#=t!XmsyC193B4Wel8@di zlH0){JFCl3x=s!I_42~&sYouBJvbn_*dgD4$?frF8^5tmc*)t36>b+l& zrD2mFn=Z1tmOVCFlt9tUSZ@>AuI3hhJyB(!Hx0tR1ijk;wiBSk(k}Jj<6~Dd0=J=B z{w$2O1EmfC?he+?q^8!!o_EodCk>c_-jYxQMGXDu@Icd4_R5O;*0QVjlnozvqMdOvMwt!%}5uXz;^Z6ljAxf%{*P;n6Y?e-RRv& zW@8GNiLzuBL5Qv+^VwX#k>tshOoWLt3rxgG_XD<(!AfZKUc3~S1FB((DB5KI z_Yx-5@QBtraVK5@UJIW6H1*(9xmjmJWSnX0^QTG*Bh8WNoj#ZC<1{j>P&_pZ%tmF zdhVIDA#CdhAiQYiyg%LR2vK~V&_C@u{ml48%il#9Xz8kW<+t|LSA*y^064|UJp{fJ z6<}x8`L(Lq#(&)V;_0Ka2L~)TX!6D%L_u1M8c~V$y4kMp9p$ zLhl~LOTi?&-lJ5az1wSc>Y2B)i;huspAZ1#k|C#thDHy)9op>IX!Y%)WmZWJ_OKO(Z)P`#zbu+Pg!K@WEi)+HQ`ul=Pg(^J=_B(%uHEavvA(*>Z+v?{LSjp_lun{$MS$W^8Wr3lMv;50PtU- zm(Z4{G+mGn+#UrM@)Y4b(RJkq<23RGc)R@e@#Eu}&?@wX-07GPxx;DMLL?GFg6v1= zc9Hys+MMdIA1z9O4>x0Bed#3_y#;IPs{QzPg0o~4KFp=oUYAXx$Rm|~nq$43ZB>gu z9VxQaC7%}Oz&(%+0ImeH~4h7!*ob*6D5!QAn#LyJ{xRB zCy`zpTKoeuVpkp)^ikMkame(;0i~G`PW~Zku%hZ8sbheS#Jnf5FY%jd#llI5`#emK zgBuyG*=z}2k9?Mpko^J%WG_l}wxyDl?|oDT)gQl}Ff*R2WdcAV0B_I_&t4_H)piJ@;09M#|C~O!7u1kIn0L_)h0idb!BLLpj zpHj)4E`N^-UjLVUV( zbGy1vcRDFV@_t_$Bf7hHM~S}+@tJX@Lkty%jYAHRBaRrqb%@DM4qOHsE~i-F9Do7R zj5SV?mc_}RSD`xvs2vJ03czjL;1oXbwQ<@h&Y#XwDMtNxfPRO@=g>b1;1s}70^}$Q zQRdV0TPO`Sc14SUqj>~3h_C^Rp(bb)epWCW3!wnK-TivB_%Ow3rD}}sOtIKG5*<5@ zq%3i+Yd#RWLAt4<({-$ug9e z6ELG-Jo;;a1jq7xfXM3eAjd&JB8Urs!b^9K5hxWy(?3W1Skx{7SPHPxI8-XiI&gZn zq#RjwUU1Hp@jO-oz28O-FS@)Ct-Q`W;xu~XgqJ5^y!S>$Pxp`kM=DMFjUBT^TG>o6H5-66=P8uZsF&P? zQaDXZ$3+gNcNxE#Ehf6QL!KU>-C)Gb5qX2ksltrskebXb53rY-@9UmAN5t5ZuLbIL z0M`>BAK5WWWMx}9z6L+#_-;VkjR5pLgH7%>uCEoVIxs7FD*$U&cuH1?dj}dW1t)t@ zx(y&Q9Tp4EDKZJWMg2}-@)Yi&5@&TIi*E(>{JJ!bS-Y2T-tOMCK#a^yM)WDSpgUeR z%BzjeMp5L#cCfZvk#W3HxW#BAu1Sj;pNjz;Icn6x&n zO=7e7mGRdmaqi&5gt#qzNl((FrsvvQUU8m$*qGWZ>cs=b{$|lE_84g`;#RTGIMgDR zr$;EWwlO0IjEqfUL2_g~j(Y2i)=i?_#qlvsjzSLO$4%l_;&;ZwQtT9uc8?2)DI$kM zh=XY7z=9Vw6SEjDFE!|9(vSjPCPcobYke@-;t2%3!PbD}&Cd?TG+qmef+1|J$AVG; zAzRbuJ<6*#+DK^?gF3L5bjgqo76HqFWue0aNdG8G;c&rD~4<-mmn}B4-#s-6RTKkua{pC{bco=rDraPPaP4-@3H6QhfSfH2JpfTb1ar z59EEpDggx~AuyrsM$sNIWd=4o`B?>tLCCa1=-V$2F=PFWEzL66;IHF+G3TuVl*
aKGeo{yIELtTAFyf7?A{nCxOG@x$frM#Q@E^#>#6HZ(M(;sHpfO&{*@K z^0elqsQ3f;;Z14b5}E!iml)zMy1n;86av>Bws(^TT;-yu6TLFRz2y5eZn)2iy`tE4ochQQ!Oj@t#=T<0FqVua@v4nq zkJAz{%dn&Y!n46i8oB$(%6($Y+9xLWihuU*6Bmd|ton@a9QfX#;!6iTunRU_bi1V` z^kvi1&M0;_rpq4)ee*;a+e+YmZ=8FNn7xGUco+EM0|K&Np}VAX%V>RT(?&CfyoRU2 zPv2Rwk^eJrLBWj0$R50n>U*)&c;g;1Q+#1$-z(lASNh-giq)dyGjwB)nasAB0=%dy zs3_Q`6|{f(6aD0ugtE=KYWjlOn$=6Kd- ze9Of_`qGNNA2O}7@%saU&J@OX2gF?Gx7fO-QSy+;Fxq}8;*G|G;>lj(t-ep>YLyX= zw?(rgD)Qv#80-u1>B8jmFnOwm_qXI6-E#n{H1>6h^fl8UYD|m|mn2zC#X3=Zh0@u? zANpA9EEAI_s862+q>=thF;~QQdw(fbh{zO-*AIy|S>~oIVtxaBb-;4SkXT5_%-&y! za;GKfUQ9t9==jcx$^3wUe$wv45A-5sy|+nk@Og}34~TMSVLY+i24RwlQBz-FFc0VNQL-cAP=%4yj+n*I*K77l(TRd-zBOWa*PCKYgUu4u06e^8z5)G-L5G(y$=DP#~`to}aZ( zh+$$jbks;Qjy);Tj3=I?cs|(u`IF*=5LX!c_K6aQXcLs*Dt}N6G76p%mx~zV7te@_ zIO)=3+U;$c9&HpoOL`k?EPYnwYq6Q??xKOlhj)wEj<`Fbme6f>`XJU-E3rUQ8w1SqtuVyM1-eC-&VmYdtiQY z`e|<8T{HPPQB$xByYFM7L9Qw}-H5#@cF%}o$pvpF7hL6d*Oh$2mT2sMPAp2@7x!Ay zDEj~1f%*H2-W*uE%Wjk&73;_!?)_Y3c0YVnEK6O^ucp|WMs$M@zkSWGrGI{`CI`Cv zP~!A2QpNHRP1ec_UxUZTj8$(@_GB9IXuS{OWHi6;IZW+e=2FADpWlXGXQlNr=mGS5 z6oCChR4)(DVJAXm{+`x)0RZ)NkLC~dBABMV3CqF(!b$p|KlEWEyT*9Z9QjYM`6ECE z9yJ1Hw6&lCOYTn8CjoHi^(;oUGgQVtq#Vj`6oo#q&8c(HB`33M!(|o2Ac&Z>90_XCli=O0GfE z#}Ed>DzYJL^Tg+51?Cy<&qb@XFHTQDd{4V`0O4~|pni)kXHcExLT90Llqk^cB67n` zYyIo!Ju0051{lBpLX7w~uM=^*imvN0HeZPwOQKoQo%<)TFvhyzcC!&YA(o{e{o=P# zrb({AK%F+DY=WrHKZH55r0&NIh&7slC9nsmyv=RwK23qo)UU+=@iSxnNpa2q7U?=7 zLYLUAEMJ51;7O4YO0oTak-WoL`n4#F0ZF<`vER7vYvH$Ek0A?W#;&C_ah&}}Z6fclZ!pQ_060L&iG*j}UJ7j~_9v0^?vsRRW?BZ<2%Zwz z6p`*aMQ0c`U^g2No)YIfAws#vh(9fcr|$>VJHbpPSeOcM%$RvvOsBZ(Y2$Xt#DeL{+9Uur)vz z4_NafcR0vnTCxCRPLO6xIe=b)PypB5Xe~0Wrlr#grAKKC`2DPzZ8Qok+bLJmsPA@P8>L<2 z%=!Yj%=0v={jwO32LM`((l~9|YUV%QN>yR~Wefldo;hMwgM!f%DK$Vhv%>7K%9&C0 z2_!Lwam<7!dTC2}yim^KPUE9Et$Z5t=VoHF^wXVPx)#9;FTst)7biL62nRO9`dM3V zI#2+#?xikmSCshP_$*QLik-&HByFsvhz-WilC*^doZ`!%+XVQe#A7X;C-fn*Y$6gB z#;GLj60$+f$y&2GWqg>d6_m0ZS)U;wGm%b9;IHTVub62dG z+=<%D0k#|O4AzE-BI6%}wW$`Duy$)_e9;%Y=FAwnpx;@SaJ>$~evV(2H2*{g>+23{FQ=t7u%N>iUjRP1DW}`hGy}91wkp?ET-q<-zo6z5_ z--;%HKN)Wg(|$oKvVFMLC~}cdp?`8FN%yVw(v4(XgA6v)Mefs}-q{^HQfqMJR1i|_ zwgJ^vpzYK1q;FFzZREV66yxtM*1CtmNKje_Q5ku=06O!og&y@d%2uPXL@T0b=jSEb z2z;O}7qeUvoRrN~%Ts*M-}_qQ!BFW}uwECieLEYq50g+7mC%zHs) zKLFbxa=4}1z{*R}1lBBT$)@WrAl(kYGlf}|_*bXQI67N;=#wh1KWLpJ0*1XzE1txM zgi0E zzM3-+6|~!CqLp81aLn@?s_r?o@(B;UO&4+`l;x#^+B&(IYWi5AQRB2iG1*u!PJ4VD zwAr29KX%*;#(5X*LkZSfI>0?Uq0`2c@mi*vCH5$q9s$4-_xg7w?m->EH;KlM@!FT9 zQ5(+H{vkx0Q8!VWD*VR16Sb=cv)G;i?z05?Tw;ojda4z#n=(Q8=`fD+b81`JWHtrwlfhHCBIOhmt?i|(lT>#I4+Gwri$XDpD9 z8>^;kWlr{6eq;A^(!UDh>FL@z1z?O$&~*RF8zeX05)4=#it?(g+wyi}K#f+K3H9h7 zzC#iX#;O`EUo7r!uhFs`E{^sfZ27)%&m3){IMDsaIoh%q*IeSv@~46BNlUb0!g(pM zJ~7rUB`m*j_foBV2=m9gcMsw_T1IF}a+9NIwepv5s zH=>qnmsq#4?8eUJ+HiUSa%j1BQM6@z-$U1S=vXw^Am->5v@!G|?VJmq;#zin5LiMAee1 zJ84F(+(;-hYG=@o|BaklDbUBkeR_d#Cpn6`lTcWV7%W1Cy;fdET+CQk>pmdQBW!vJ z)6B0**;YJ{F*&rIh)J;+*!o>fqtP1(Ivxj`o9X(<3QUrP#_IAV`uPD)Py@X?@i*hG z7is32dWvGE8<*5;d2}?pqh1>%Ug|zluT={1knwk2+uAXRNXr81YYwJYK7kVJ-|(PU zKlCru&;^4)Nsk*D3LN^th)!Q2o+pFJez&zMQOB1wrlIsTm^uY89go!jH2{43=J#|h z!zN!rb0Gk)F`F@5zLCRFHynVE1qS*Z#p6hTDFD>~H2@0$mH=D;uo~bJ0K^rh9ao9u z<#@gl;2HqyTF3|tRSqx#fMXGSx4~a8NZk6D&jb7c;1mF#wAke0a-pSrnJC4gH6MU) zcr>E!M~s9U1+pFBGJvZA_~3GHE{dGd;5TRd=84}DeF-w}7@yHUiZ`^cI)4{4iGq>V z2HzCvvfFHgN8bc?H>}e>&{E&ERlU~t|L!>(v~^l}qeIXqerJa#x&CzT);d@B#|_$v dsIrS4f<7obJ3Pttr+c?", ""] def llama_2_chat_pt(messages): prompt = custom_prompt( role_dict={ diff --git a/litellm/proxy/api_log.json b/litellm/proxy/api_log.json new file mode 100644 index 00000000000..810c5b826bb --- /dev/null +++ b/litellm/proxy/api_log.json @@ -0,0 +1,85 @@ +{ + "20231012182157625128": { + "pre_api_call": { + "model": "anthropic.claude-v2", + "messages": [ + { + "role": "user", + "content": "what do you know?" + } + ], + "optional_params": { + "temperature": 0.1, + "stream": true + }, + "litellm_params": { + "return_async": false, + "api_key": null, + "force_timeout": 600, + "logger_fn": null, + "verbose": false, + "custom_llm_provider": "bedrock", + "api_base": null, + "litellm_call_id": "902640b5-4a26-4629-932d-35d6cf4e1635", + "model_alias_map": {}, + "completion_call_id": null, + "metadata": null, + "stream_response": {} + }, + "input": "\n\nHuman: \n\nHuman: what do you know?\n\nAssistant: ", + "api_key": "", + "additional_args": { + "complete_input_dict": "{\"prompt\": \"\\n\\nHuman: \\n\\nHuman: what do you know?\\n\\nAssistant: \", \"temperature\": 0.1, \"max_tokens_to_sample\": 256}" + }, + "log_event_type": "pre_api_call" + }, + "post_api_call": { + "model": "anthropic.claude-v2", + "messages": [ + { + "role": "user", + "content": "what do you know?" + } + ], + "optional_params": { + "temperature": 0.1, + "stream": true + }, + "litellm_params": { + "return_async": false, + "api_key": null, + "force_timeout": 600, + "logger_fn": null, + "verbose": false, + "custom_llm_provider": "bedrock", + "api_base": null, + "litellm_call_id": "902640b5-4a26-4629-932d-35d6cf4e1635", + "model_alias_map": {}, + "completion_call_id": null, + "metadata": null, + "stream_response": {} + }, + "input": null, + "api_key": null, + "additional_args": {}, + "log_event_type": "post_api_call", + "original_response": "", + "complete_streaming_response": { + "id": "chatcmpl-1757e5ea-71f2-44a2-9d8d-1ba8238a7c99", + "object": "chat.completion.chunk", + "created": 1697160117, + "model": "anthropic.claude-v2", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": " I'm Claude, an AI assistant created by Anthropic. I don't actually have general knowledge about the world. I'm an AI conversational model trained by Anthropic to be helpful, harmless, and honest." + }, + "finish_reason": "stop_sequence" + } + ] + } + } + } +} \ No newline at end of file diff --git a/litellm/proxy/cost.log b/litellm/proxy/cost.log index 365bc3e70ee..e69de29bb2d 100644 --- a/litellm/proxy/cost.log +++ b/litellm/proxy/cost.log @@ -1,11 +0,0 @@ -2023-10-11 15:02:23 - Model gpt-4 Cost: $0.00063000 -2023-10-11 15:02:57 - Model gpt-4 Cost: $0.00093000 -2023-10-11 15:09:10 - Model gpt-4 Cost: $0.00135000 -2023-10-11 15:09:50 - Model gpt-4 Cost: $0.01626000 -2023-10-11 15:12:57 - Model gpt-4 Cost: $0.01974000 -2023-10-11 15:13:35 - Model gpt-4 Cost: $0.02415000 -2023-10-11 15:14:04 - Model gpt-4 Cost: $0.03291000 -2023-10-11 15:18:16 - Model gpt-4 Cost: $0.03669000 -2023-10-11 15:19:12 - Model gpt-4 Cost: $0.04806000 -2023-10-11 21:11:06 - Model claude-2 Cost: $0.00041534 -2023-10-11 21:15:34 - Model claude-2 Cost: $0.00054606 diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 644641f978f..19b105a1ad8 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1,6 +1,6 @@ import click import subprocess, traceback -import os, appdirs +import os, sys import random from dotenv import load_dotenv @@ -8,42 +8,33 @@ load_dotenv() from importlib import resources import shutil -config_filename = "litellm.secrets.toml" -pkg_config_filename = "template.secrets.toml" -# Using appdirs to determine user-specific config path -config_dir = appdirs.user_config_dir("litellm") -user_config_path = os.path.join(config_dir, config_filename) - def run_ollama_serve(): command = ['ollama', 'serve'] with open(os.devnull, 'w') as devnull: process = subprocess.Popen(command, stdout=devnull, stderr=devnull) -def open_config(): - # Create the .env file if it doesn't exist - if not os.path.exists(user_config_path): - # If user's env doesn't exist, copy the default env from the package - here = os.path.abspath(os.path.dirname(__file__)) - parent_dir = os.path.dirname(here) - default_env_path = os.path.join(parent_dir, pkg_config_filename) - # Ensure the user-specific directory exists - os.makedirs(config_dir, exist_ok=True) - # Copying the file using shutil.copy - try: - shutil.copy(default_env_path, user_config_path) - except Exception as e: - print(f"Failed to copy .template.secrets.toml: {e}") +def clone_subfolder(repo_url, subfolder, destination): - # Open the .env file in the default editor - try: - if os.name == 'nt': # For Windows - os.startfile(user_config_path) - elif os.name == 'posix': # For MacOS, Linux, and anything using Bash - subprocess.call(('open', '-t', user_config_path)) - except: - pass - print(f"LiteLLM: Proxy Server Config - {user_config_path}") + # Clone the full repo + repo_name = repo_url.split('/')[-1] + repo_master = os.path.join(destination, "repo_master") + subprocess.run(['git', 'clone', repo_url, repo_master]) + + # Move into the subfolder + subfolder_path = os.path.join(repo_master, subfolder) + + # Copy subfolder to destination + for file_name in os.listdir(subfolder_path): + source = os.path.join(subfolder_path, file_name) + if os.path.isfile(source): + shutil.copy(source, destination) + else: + dest_path = os.path.join(destination, file_name) + shutil.copytree(source, dest_path) + + # Remove cloned repo folder + subprocess.run(['rm', '-rf', os.path.join(destination, "repo_master")]) def is_port_in_use(port): import socket @@ -60,23 +51,31 @@ def is_port_in_use(port): @click.option('--temperature', default=None, type=float, help='Set temperature for the model') @click.option('--max_tokens', default=None, type=int, help='Set max tokens for the model') @click.option('--drop_params', is_flag=True, help='Drop any unmapped params') +@click.option('--create_proxy', is_flag=True, help='Creates a local OpenAI-compatible server template') @click.option('--add_function_to_prompt', is_flag=True, help='If function passed but unsupported, pass it as prompt') @click.option('--max_budget', default=None, type=float, help='Set max budget for API calls - works for hosted models like OpenAI, TogetherAI, Anthropic, etc.`') @click.option('--telemetry', default=True, type=bool, help='Helps us know if people are using this feature. Turn this off by doing `--telemetry False`') -@click.option('--config', is_flag=True, help='Create and open .env file from .env.template') @click.option('--test', flag_value=True, help='proxy chat completions url to make a test request to') @click.option('--local', is_flag=True, default=False, help='for local debugging') @click.option('--cost', is_flag=True, default=False, help='for viewing cost logs') -def run_server(host, port, api_base, model, deploy, debug, temperature, max_tokens, drop_params, add_function_to_prompt, max_budget, telemetry, config, test, local, cost): - if config: - open_config() - return +def run_server(host, port, api_base, model, deploy, debug, temperature, max_tokens, drop_params, create_proxy, add_function_to_prompt, max_budget, telemetry, test, local, cost): if local: from proxy_server import app, initialize, deploy_proxy, print_cost_logs debug = True else: - from .proxy_server import app, initialize, deploy_proxy, print_cost_logs + try: + from .proxy_server import app, initialize, deploy_proxy, print_cost_logs + except ImportError as e: + from proxy_server import app, initialize, deploy_proxy, print_cost_logs + if create_proxy == True: + repo_url = 'https://github.com/BerriAI/litellm' + subfolder = 'litellm/proxy' + destination = os.path.join(os.getcwd(), 'litellm-proxy') + + clone_subfolder(repo_url, subfolder, destination) + + return if deploy == True: print(f"\033[32mLiteLLM: Deploying your proxy to api.litellm.ai\033[0m\n") print(f"\033[32mLiteLLM: Deploying proxy for model: {model}\033[0m\n") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 210493a7420..186f462520c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1,9 +1,10 @@ -import sys, os, platform +import sys, os, platform, time, copy import threading import shutil, random, traceback -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path +# sys.path.insert( +# 0, os.path.abspath("../..") +# ) # Adds the parent directory to the system path - for litellm local dev + try: import uvicorn @@ -76,12 +77,10 @@ user_max_tokens = None user_temperature = None user_telemetry = False user_config = None -config_filename = "litellm.secrets.toml" -pkg_config_filename = "template.secrets.toml" -# Using appdirs to determine user-specific config path -config_dir = appdirs.user_config_dir("litellm") +config_filename = "secrets.toml" +config_dir = os.getcwd() user_config_path = os.path.join(config_dir, config_filename) - +log_file = 'api_log.json' #### HELPER FUNCTIONS #### def print_verbose(print_statement): global user_debug @@ -98,15 +97,6 @@ def usage_telemetry(): # helps us know if people are using this feature. Set `li def load_config(): try: global user_config, user_api_base, user_max_tokens, user_temperature, user_model - if not os.path.exists(user_config_path): - # If user's config doesn't exist, copy the default config from the package - here = os.path.abspath(os.path.dirname(__file__)) - parent_dir = os.path.dirname(here) - default_config_path = os.path.join(parent_dir, pkg_config_filename) - # Ensure the user-specific directory exists - os.makedirs(config_dir, exist_ok=True) - # Copying the file using shutil.copy - shutil.copy(default_config_path, user_config_path) # As the .env file is typically much simpler in structure, we use load_dotenv here directly with open(user_config_path, "rb") as f: user_config = tomllib.load(f) @@ -133,11 +123,8 @@ def load_config(): ## load model config - to set this run `litellm --config` model_config = None - if user_model == "local": - model_config = user_config["local_model"] - elif user_model == "hosted": - model_config = user_config["hosted_model"] - litellm.max_budget = model_config.get("max_budget", None) # check if user set a budget for hosted model - e.g. gpt-4 + if user_model in user_config["model"]: + model_config = user_config["model"][user_model] print_verbose(f"user_config: {user_config}") print_verbose(f"model_config: {model_config}") @@ -317,7 +304,55 @@ def track_cost_callback( except: pass -litellm.success_callback = [track_cost_callback] +def logger( + kwargs, # kwargs to completion + completion_response=None, # response from completion + start_time=None, + end_time=None # start/end time +): + log_event_type = kwargs['log_event_type'] + print(f"REACHES LOGGER: {log_event_type}") + try: + if log_event_type == 'pre_api_call': + inference_params = copy.deepcopy(kwargs) + timestamp = inference_params.pop('start_time') + dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23] + log_data = { + dt_key: { + 'pre_api_call': inference_params + } + } + + try: + with open(log_file, 'r') as f: + existing_data = json.load(f) + except FileNotFoundError: + existing_data = {} + + existing_data.update(log_data) + + with open(log_file, 'w') as f: + json.dump(existing_data, f, indent=2) + elif log_event_type == 'post_api_call': + print(f"post api call kwargs: {kwargs}") + if "stream" not in kwargs["optional_params"] or kwargs["optional_params"]["stream"] is False or kwargs.get("complete_streaming_response", False): + inference_params = copy.deepcopy(kwargs) + timestamp = inference_params.pop('start_time') + dt_key = timestamp.strftime("%Y%m%d%H%M%S%f")[:23] + + with open(log_file, 'r') as f: + existing_data = json.load(f) + + existing_data[dt_key]['post_api_call'] = inference_params + + with open(log_file, 'w') as f: + json.dump(existing_data, f, indent=2) + except: + traceback.print_exc() + +litellm.input_callback = [logger] +litellm.success_callback = [logger] +litellm.failure_callback = [logger] def litellm_completion(data, type): try: diff --git a/litellm/proxy/secrets.toml b/litellm/proxy/secrets.toml new file mode 100644 index 00000000000..fa04cb5f188 --- /dev/null +++ b/litellm/proxy/secrets.toml @@ -0,0 +1,29 @@ +[keys] +# HUGGINGFACE_API_KEY="" # Uncomment to save your Hugging Face API key +# OPENAI_API_KEY="" # Uncomment to save your OpenAI API Key +# TOGETHERAI_API_KEY="" # Uncomment to save your TogetherAI API key +# NLP_CLOUD_API_KEY="" # Uncomment to save your NLP Cloud API key +# ANTHROPIC_API_KEY="" # Uncomment to save your Anthropic API key +# REPLICATE_API_KEY="" # Uncomment to save your Replicate API key + +[general] +# add_function_to_prompt = True # e.g: Ollama doesn't support functions, so add it to the prompt instead +# drop_params = True # drop any params not supported by the provider (e.g. Ollama) + +[model."ollama/llama2"] # run via `litellm --model ollama/llama2` +# max_tokens = "" # set max tokens for the model +# temperature = "" # set temperature for the model +# api_base = "" # set a custom api base for the model + +[model."ollama/llama2".prompt_template] # [OPTIONAL] LiteLLM can automatically formats the prompt - docs: https://docs.litellm.ai/docs/completion/prompt_formatting +# MODEL_SYSTEM_MESSAGE_START_TOKEN = "[INST] <>\n" # This does not need to be a token, can be any string +# MODEL_SYSTEM_MESSAGE_END_TOKEN = "\n<>\n [/INST]\n" # This does not need to be a token, can be any string + +# MODEL_USER_MESSAGE_START_TOKEN = "[INST] " # This does not need to be a token, can be any string +# MODEL_USER_MESSAGE_END_TOKEN = " [/INST]\n" # Applies only to user messages. Can be any string. + +# MODEL_ASSISTANT_MESSAGE_START_TOKEN = "" # Applies only to assistant messages. Can be any string. +# MODEL_ASSISTANT_MESSAGE_END_TOKEN = "\n" # Applies only to system messages. Can be any string. + +# MODEL_PRE_PROMPT = "You are a good bot" # Applied at the start of the prompt +# MODEL_POST_PROMPT = "Now answer as best as you can" # Applied at the end of the prompt \ No newline at end of file diff --git a/litellm/template.secrets.toml b/litellm/template.secrets.toml deleted file mode 100644 index 11de08fbf09..00000000000 --- a/litellm/template.secrets.toml +++ /dev/null @@ -1,50 +0,0 @@ -[keys] -# HUGGINGFACE_API_KEY="" # Uncomment to save your Hugging Face API key -# OPENAI_API_KEY="" # Uncomment to save your OpenAI API Key -# TOGETHERAI_API_KEY="" # Uncomment to save your TogetherAI API key -# NLP_CLOUD_API_KEY="" # Uncomment to save your NLP Cloud API key -# ANTHROPIC_API_KEY="" # Uncomment to save your Anthropic API key -# REPLICATE_API_KEY="" # Uncomment to save your Replicate API key - -[general] -# add_function_to_prompt = True # e.g: Ollama doesn't support functions, so add it to the prompt instead -# drop_params = True # drop any params not supported by the provider (e.g. Ollama) - -[local_model] # run via `litellm --model local` -# model_name = "ollama/codellama" # Uncomment to set a local model -# max_tokens = "" # set max tokens for the model -# temperature = "" # set temperature for the model -# api_base = "" # set a custom api base for the model - -[local_model.prompt_template] # Set a custom prompt template for your local model - docs: https://docs.litellm.ai/docs/completion/prompt_formatting#format-prompt-yourself -# MODEL_SYSTEM_MESSAGE_START_TOKEN = "<|prompter|>" # This does not need to be a token, can be any string -# MODEL_SYSTEM_MESSAGE_END_TOKEN = "<|endoftext|>" # This does not need to be a token, can be any string - -# MODEL_USER_MESSAGE_START_TOKEN = "<|prompter|>" # This does not need to be a token, can be any string -# MODEL_USER_MESSAGE_END_TOKEN = "<|endoftext|>" # Applies only to user messages. Can be any string. - -# MODEL_ASSISTANT_MESSAGE_START_TOKEN = "<|prompter|>" # Applies only to assistant messages. Can be any string. -# MODEL_ASSISTANT_MESSAGE_END_TOKEN = "<|endoftext|>" # Applies only to system messages. Can be any string. - -# MODEL_PRE_PROMPT = "You are a good bot" # Applied at the start of the prompt -# MODEL_POST_PROMPT = "Now answer as best as you can" # Applied at the end of the prompt - -[hosted_model] # run via `litellm --model hosted` -# model_name = "gpt-4" -# max_tokens = "" # set max tokens for the model -# temperature = "" # set temperature for the model -# api_base = "" # set a custom api base for the model -# max_budget = 100 # sets a max budget of $100 for your hosted model - -[hosted_model.prompt_template] # Set a custom prompt template for your hosted model - docs: https://docs.litellm.ai/docs/completion/prompt_formatting#format-prompt-yourself -# MODEL_SYSTEM_MESSAGE_START_TOKEN = "<|prompter|>" # This does not need to be a token, can be any string -# MODEL_SYSTEM_MESSAGE_END_TOKEN = "<|endoftext|>" # This does not need to be a token, can be any string - -# MODEL_USER_MESSAGE_START_TOKEN = "<|prompter|>" # This does not need to be a token, can be any string -# MODEL_USER_MESSAGE_END_TOKEN = "<|endoftext|>" # Applies only to user messages. Can be any string. - -# MODEL_ASSISTANT_MESSAGE_START_TOKEN = "<|prompter|>" # Applies only to assistant messages. Can be any string. -# MODEL_ASSISTANT_MESSAGE_END_TOKEN = "<|endoftext|>" # Applies only to system messages. Can be any string. - -# MODEL_PRE_PROMPT = "You are a good bot" # Applied at the start of the prompt -# MODEL_POST_PROMPT = "Now answer as best as you can" # Applied at the end of the prompt \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index 069cb4d8fa1..7d678aeb0a4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -53,7 +53,6 @@ from .exceptions import ( ) from typing import cast, List, Dict, Union, Optional from .caching import Cache -from .llms.prompt_templates.factory import llama_2_special_tokens ####### ENVIRONMENT VARIABLES #################### dotenv.load_dotenv() # Loading env variables using dotenv @@ -249,6 +248,7 @@ class Logging: "messages": self.messages, "optional_params": self.optional_params, "litellm_params": self.litellm_params, + "start_time": self.start_time } def pre_call(self, input, api_key, model=None, additional_args={}): @@ -323,7 +323,15 @@ class Logging: message=f"Model Call Details pre-call: {self.model_call_details}", level="info", ) + elif callable(callback): # custom logger functions + customLogger.log_input_event( + model=self.model, + messages=self.messages, + kwargs=self.model_call_details, + print_verbose=print_verbose, + ) except Exception as e: + traceback.print_exc() print_verbose( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while input logging with integrations {traceback.format_exc()}" ) @@ -416,6 +424,7 @@ class Logging: ## BUILD COMPLETE STREAMED RESPONSE if self.stream: + print(f"stream result: {result}") if result.choices[0].finish_reason: # if it's the last chunk self.streaming_chunks.append(result) complete_streaming_response = litellm.stream_chunk_builder(self.streaming_chunks) @@ -573,6 +582,14 @@ class Logging: capture_exception(exception) else: print_verbose(f"capture exception not initialized: {capture_exception}") + elif callable(callback): # custom logger functions + customLogger.log_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) except Exception as e: print_verbose( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging with integrations {traceback.format_exc()}"