From e8eb92c1082e931e2b061e9fb6b1ddf7eeda4704 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 28 Aug 2023 14:56:20 -0700 Subject: [PATCH] new logger client --- litellm/__pycache__/__init__.cpython-311.pyc | Bin 7483 -> 7515 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 29196 -> 29301 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 68280 -> 71669 bytes litellm/integrations/litedebugger.py | 139 ++++++------ litellm/llms/ai21.py | 2 +- litellm/main.py | 49 ++--- .../tests/test_litedebugger_integration.py | 40 ++-- litellm/utils.py | 198 +++++++++++------- pyproject.toml | 2 +- 9 files changed, 235 insertions(+), 195 deletions(-) diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index b15b88021181c0a14e70994b95401895e58eeb7e..8e25efb870eb9efd884e27e3e4121205f4bd0601 100644 GIT binary patch delta 129 zcmdmOb=!(}IWI340}xC)`zB@KM&5@UjCGUWbHp&#PWIuvz}UK3mP?k0NrHW{jaaSs v2Q~&)&JU~%jBE`RH(1!eFf(y5ePCc>^2niwJEeKc^^mdUH z^{!AoZY_J1Sk@UkaUy3(9m#Hmr;%w|wTYW5(^geGsTZ3j1dmf^8asB|)F_D^Pm5zc~fZ63^NZSXy9t%@{o97MEsaEpanH?_)w#KSJpWf3IRRi5Ln;TwGw5s2ZuB zYLF&VEm8v&ks4_dQWMo7O`&?EW@G-)^rj)t+#qTG6-IPM;(#}GPWWd-5(ie?Rk_AucPpC653o|ZorRGC%@F{Q9i-Yy>TvZ{T_(mVXT}%m_f?9*ynjPgQsFreY4>{y(vy=;?h9*ni zRf2u+jXo1Z^1`czSK&_#QNCCj;|pgpI@S4?`8eJR?($Z+iZ>T2g726d)_b`!1v24h zCYNO`;{|ytR(RrN6Y}IrX(L{^X()wHb!IqXOo9$$Ndh-ob1x}@zG6Emg(-8cIuE5` z2D9D9Q!`|nZLrPKqAt6vQkXnQ@{kD|KT{5$Ppv(eCsnL)i5lWL$;lq-k&;(*PO4ni zx3tiP*sgFK#eJh$GgVgw+%0^g4@x0tFP5Vu>^M=GrAL|A15aBlwbgNBs^jwOR`Ovl z^-{N#&=Rg&Y*SJ#+80hLtzT)34c9BB9OlxN>o+9SZ%|6z;hc-ffsfF1*!F5s9W{5= z@n)z##yp3{l_-3|NotAnXNAccIFVLjsY|R5=hJ++o@R%4(^BBK9wVs(eRj4ZCpKU> zKhze>BlG!?ZS`vF7)w%AUJXoMg?UmV%^N1+0<=oD|2mpP3#5$->nzu#PBg=C;b#(WA@0@##4Bw^F&A#(dbHspRD+^32968o zTo7pviZ5&0ia>b9WHIF-r{24~df+)*6P(F%^PxweIo$>(yB+nqB+F$fzQ4?QsTRiV zg%$Toc`Yrb_?}@K*@EWgmR97f4-~;|yOVqg-mxD-vuH{$vzOpXS`(olTuMv9>~Wgi zp(Al3LimE~>6T9SXWA?G|6&SE_jG~B!0cE{a%USI<3vHUQg}0CNFBwBHm8@A!I;x! zE~jPZcxh+2jFw}%D_jmG$!lyBKc(H5c}Z!$3hYYAC*aC-ks#D(y0uJOE5jAgmw8ZK zNh_wSnKoD|SE+4PRQ=$$nRzQWns z_h=Ed%yeHB5gPH0KH->7Wj_F3tqJgoIwc*z84s`0+bF;?tiU(m$2eG=*%g|c;J$iDZHzv zSU)qeN+fkr?}Lh6OY7kM9G6fx-GpY427ldS)P(D$QCbfbxfW6nPf6Bw_#L;R1jiFs zp5UVX-?UH_cm-b0t9LM~-W+Z~u?GAtV_U+FW3Azhi1rOYy!)fB9|GPuVWvymy|zpn zr?)XUfTjEtbrWrx-X7Z(Pk0j=`lgXbV!BH62FWnD99SWyGk31elQ`;yIhVzJUl+X$ z*VcRB`l1lH0=L%GTUVBG!rw8rGu*st36JGJbP3HXOK@*So;Q!|`aj^=aSzXw85kd! zfxj$pl8f*!1wBGb=nC^&>9plOIIx9od9XLx``Il&_&_s=GG>Hd6gCMlL*7vow{aUK zPC7+543m4qgV2ky&BACzv7}YD(@I<6c2Nm=3Vv18BWw*_i>`RrC zOIj=sh(MRs+j1XOfp+Ioep+Ak=UOe>BZ8Lw6xWSJSi3@K#yX_P?#hn%oai>V( z<2&J{(raqfWY8zWp0Yo~0?4R1mb9oH7!QPefe=hpY$Hy%U9tJl-2oQGzKP-9 z(B~$6%R?<{Cgkz4iBQ650i0uBAz9SN9gcC>%jwW`*@V&_64cd?SSu&?TrQ66VS+T|s} zkE?Pu%>L!wpsU_by5Ly#VZGA4jFldNzpp;_(784Ls>}_e>uQ_NvTjOWWg;zUd3f7zMcdA8csLK`?<#E>(@dW4 zGx-`W4>I?aL zeSwq1@_1kjmA3a}pFG^x??+X@%8 z%laPGgu6RbZC4NAs)-9xZ&ugK#*`jYl zMtG??+q95k{iVQLjptiG)NmR3vu`v%L)=R?E-B>)V)5%@@r~X{&uBy}o)i7^qCX<~ z7xY=PPqoaE3XwB=mN>y&_Le1M-jcf{VD=r0E0R?^6cv)_WrJ-sesD!^YXJ7+3*J8KPmxg}%P+_szO|Di47+&Aak zH}BjxJJc@l$&uX$BhIclXV<*5Yj$G$*GO_?&!LF3d(PQC@9dtn@3_Jzbo%bj7l;t) z?t^b1bit#$-*+}4bq*UT>AuV@$*L+&_4b|&u4Ux14Sn4c-7dtD2$dLhTpFX?P zDXE0W-fs2*{sZ64jil^-zINVOis?6V{;cRPN?$3BH1CncS+z3=^tuf`$1WV z9O#+-_CZPvDzy&xdzrwoL*6MhetrD$$H55{-$hGvM>hBM1Jx|~fn#enOW&$*b+Yu| z?AvTC{m;@?rT*u+Ep_bsqpY2!EWO!kXf9JSTN^hkJ#TJvZy`!aN4BncCokOWSex41 z(THiIntczT#;CeqB3t3*?soDieAHd-5D*6qc#{+CM260B3o6UGUF`4e*_OlgJUI+g zJ^R)&5J%kCpuSJ~Lx_9Fv0za%I2oG6BDDHAVE?425buBrj+Sd~VY&RbaP;UIqJ!$~1$Y$o*G7(%aV5@*%vKk0+;tNvKI;Z{S{2z&q@!nQa*}eN(7<+?=e=@ z7gyB*w|up&id(zR>X=I`G|PX^@+(lSptORL%GhPB_6Cy0q}H*1pCsW`yZl$sF<6lK z9;^BmBR^z>DN3O}iriiB^q?iTmK7_D`45)+cSdp;`3;hw9*GBT4mtq zd~&=ZSFMQx0m3%-Xq&w)gS(ZXYZruDc^PdRg^1Vl(i}sCkNTA!b+04`pUV*hSJ_v@raahkB>11R|Q?=Yr^; z&{eDCalcRYVTWkAiOzdsd_s0%0qTdc$+@x0ARF4V9MCPASn$_7fFYyb7aGPm#znX( zZisFFA#8`hqBUKtiD(`8TX1>5vF1kleC6gyS_}JGwxCd4-t(ICXXeC=1=0AF?n&Ln z^>bq8yqFmgGv6{fzEmAC6;0P5VL!15{fd}@V=0lw080AqgBTJ;lV@RM_*dln&_7az z5x}z}rwK+GCVvij4=Vk&YA<#=4FUfVah?2K>?0qA*Zmvx9xRrbJx~%aeBa?}_aG z^h|V(dc#7Rb-|dj)Jod*1ieHY-AXu(WvT6+iaT1KM-@bJN6qo#W40xo((8atMat37 r#E22johWZO+8+tXOF|7BJVxS15TYZ9tSUs*WbG1nm!!duPL%!+bEkLb delta 7288 zcmcgQ3s76vmG3_NB_sg?35k!8_zIh!!N`Ek*ZdqCjB$;B*an`A0U_{{{E6qvCZ2TT zrrwt3G-;cn3g1vmcGo+E7KPjemAU&MRKKms9lw75p5XZ+R0$^7)k{ak&<-(U#Ik}V!aict&M@*nrfsD!| zTq*GQCcDHBksyKFDl`18%B_%Z#!h^*K&rY>mqk-WnV<-oeIm@Mt*J^ubs1-RMVgJx zBB-q#lxb7p%j$g78oltAI?uhjU1ep1r7Oq@8fuR6Cnza*l89<*7L<^!$sE*D9*Cf$ zGK2|Kj!;h(2n|$;&`4DX6R8?u64f9~rdot1szaDU^$2-tK$uF62+fp3XrX3=Y19Vf zDL(ZFoJUFJWhf@Zle1ZrC@TUH6NTI}LVT{cRh3gW(jJ={5 zd+S!Pm!_mi)G4fooeAc&VX_93MqeU)V$3uZGX{{65{VIB)FUId#5Pg@Kh~GP6k<^> zew#QZE1kAM(*6`)D1*to9GMmO;<~|t&m_o7GQ&1gv!eWxTw<}8SlR0Q%m#Qm+1+Cm zDwcRev$3D##4KtTj7x?nR4y7}a-cf8VW@g1<>E~qRp9L5BsExYkup!5NzE~0UPIbY#cu1^(op;frf=s7Ovyr`y5RH|b-H6k$lD&1&goaV-Ob;1kgdbE@e z&6%VKeqin_dUzXvxhz-tYONR1brk<+*p4=%8)gYhde`}!@SY`|JO`gx_Q}=Yj&pRm6pUA;Es6f4VKc9NnYp-me5jsb_Giz z+n8^r_$utY#0%0i%i?HD%l#z|n0zDUi(*FN;_*0F*%iOoqN{8Omcd9yucDlmO;@s= zOf8S63vP&7J;*cjqEYLdm6p#u7H?>T<2kxvWMh;E;ZHF0`~gB2Ggi2hZYoC}sB@-! zl`}b6UXNZQRZ%RMi{@MU$1`P>?=%$I3&?A*%l>Ur0smpQS}V|q_r)_DtfZ9@BW`Ai z+Onn@@4u`@Gsb`9ikWft$}9j=nW>Fvw!)s711r{vM1E$vO5f`kR1d}yY2RWZ{alKk zHNlNAmwm!gCG-mewCda3cX*6(ZecLEDP{^2j+iO*S!va3QwTgBQ^HWjMN|uvjL0jBcFX!c4M$lVnIk!ZDV!U&LZ&IX!jH46g_a)p0qn8jd&%Z4cOv z#U=7qF5#XTS+RZ9k))|WacMPt?#z@`PuC-RDNvGQ?5Gh&X${tX$$=-93GSseZ2RH` zKTjx9FB5oGqUHbCrYiScfgk7ArnBVP6s$!v@OhiZwgxwkwFI|dda9jnl_aezsE?L0 za5gVy0#>WQtu_I=dAftC5B@zbQBgqJHGD=uT1VQ8J7XTZ(O%j<80w zG~%2zcMb1;=n8~opVU>^XG-S#K7{gXElaW<_wE?$3^qJK&aW5ze{wb~$!Tvy$r?wx z{_v7nS4l>hYxBOpgm(+m$yec{!aiA3;6(`AE!L(r!cH@7j)a{?7Iq#bcdi+9a#+yu zKfuc(oasSXRIgYHB8Q4%JF7!&3#aHNEVu3OQc+e&xOeWju2gwGx>)Z5g(a;$Skr&s;q;oJ^ZWGQ4lWP`#i!sRFhS~BQ zuJiEZNDRPQCE!)SPBCe-8#dSDrL4LyT5ncR(Tpag$ z7F2`dzJSLUfa4{vfU|U4Y6H@|EEkyzEvUylet-Xv#}8uZ3pTME+r(Z3E?(?oIBRgp z#RE`UmO&1}_Oj7J)-CR1?J{YOxIt{v#Rm9!*%5_Uj}6dX-hoxYGv!TfvB3-K{)yqk zfyXC2(P`Xl=;GSaG^}AavVUxRc+lftkOw@c1L6pys)ynY0sB#g4l!^Hfu9{ZWM_+o ztYH)Z952sK+=E{(qL(!lG2n*q2Df?}qAr3$RaTrbFX+7ELqo&9p~K??BQDjVNbsYI z9OW)HqzC@3;tS*eY^pqIe(!_1`|!d(=-FurN; zLy`{S>G`rPBJ{E@CDVP0WPRTkAGs^htA7bTrdsdX`R;-7#{Z> zj`cLLv2QUT8RPSeP(B>+jERr4M)t`kz6*EiTZ-RfUHss<_&!6DkDZXL{Am9mO6MKM zfFS;q5kJE~FYIaPCclK28ZsOoF>=E|zu$vVBTD)zYa||j2R`9mL)|ltIw);SNo!2w zxJmA_U4%vL3w7T-Iz90G$Tvqqj)E(#b7{5g zwrsx8eoL1;uXBbuUQ+=V8Xek~TIN$WTxhvv&7Zy6_$(>-%*>f=p=3)aH8Yg4E|l&H zYZNB!Jr2RA3|vr!6SxHXthf0lDNu4r*>j2h7v1wY75IGp%v@8?P5173_ilWf+v^X@ zIDKkZIqPd_C;17SG3D7)^Oh}h26l%GS<`uQh622!=uJdtKWOz5<<0c^`SkkPo^~0J zc=LRE^X$O3@3Qum`Sg}qyF_!X^ZNv@^=88PowqYW#;obmIb%8ASH`c8&sXi4 z+kfc&szY;CheB|+dp{DU&HkoakTV3tJ+HfPCz3YaGTG)$`C%ENzci(Oz2+Nr&(zI1 zD{h)9=S`Ji6{k=8Obgq3#o24UloToz+AtRdx7EgF}X3rZfhQ?9}X{~F@mXHqnRw5xg9NNZCo9xDpbw!Pxsj?fLsS5T!h=y}_ z-%F$ep55P0eg_}!ue9a_;=8H=nPT9jYC@%$vA)FtoE>%^eM3~ zdT9}1XG6caB=@qAcNyyfsF!`eTN0K6VI!eipq(@MI}+ z;fdJ0f*$_$h?y4*@YR7@VuYUzzi+m(1DSYS&RZIS>VQ4U5(=C|)vLw8 zZ!%>Mm%|0Jmnd^Nu`#H@GkSK$tr;|*qnerI_$r5RfxM*ws29z$edh(a538UVQ~VgI zGSZyYEUS!CIL^iT~e0*A?vGTeWsPUNlHsnf|8b$ zYK=FLazWEFHsBEitW(4vvEffJ@b~bfr*Qkn3=J{xCIjp!&ZUjyCMicO$t{U23I7SB z{51nk20lmNXKn$%^Q3oM&=vBSTkgOFJjsO(^$KmsWVxWem28!wu`QI5b=#U1a+HKJ z^FuisFf3;n!W;t{Eep^HK4ky{w3$mt0($gYu3HJ#Ia|}KJy+6R?wjemT6wkkYUO-! z1GeFPGog7tp?OZa!7?=U2-GhRIW`(C7mW9B=~BX+n#IE?Sc->=S5MBIo$KBug~XnD zbxR@ia7PQhE=gX&802HVXeP&ihtvthHyPQ%5w!8`UbRuH>=>uoIL zi7YOOY?mWyWD|%(@Mpd>Z5g6?bN&}*Z}^_$NgM1v?x@n~!%9x452d7r42j_u(xxLQ zeMNmgFSB>;K+A z8Gm#4d+u`1J@?$JjNK(a|D-(O$2y&cfhRNhBVLw$IKj##$e+p|c{LSz*^D{loZCbwkM#bh0dx*tYMuWH?G~2ZL9_|Cux;e-gc~ZFu{4BfKTNyW0T&yRFYeye)e6N$A9l+1Su!||G9G3#%QKrfp4}|txy^E320bee zsTa9BJJ;>)^>{;SPuDgN&$oAP4{2I^x_Uc3zV@E(MwdcxK?y6&bnI`ye?opF=|x0GxaPC@5Ivn}m<&KRAQ0Zpv~j!~ zriub*e><}r7)SwLC9i}8aeN%0indP`id)&UeLFO%RS5+~9&}?(pZz<)QKnDMvo|t6 zaFcYMRelXC)R33!?D9wm`J_%?sPriy!PLXc`{X}hctxM`2h4r)N1&@8i*z@0JxfZI z_JKZGAJY=e(kBOBl!QJ7_~N34F)$L!#@1*F5P{Ss!QW?gfTPTIrjOa8?q~bh4)9CQ zVeXJ-tuzFWvLVGa?#^AFM>%03#D?VU13pie7bT6;DGg0Mxc6mG^+ricOM7>_ucf7= zwzH?z-RYeNj&`pPhkf3?kkXq1u%8j_^Q3pQet6xn%~3BMy?tDf^IM4?`0uF}+Mo_$ z2iEpKBlHK;2+dnA()WaKRDN6|<_%2wB}x;QcSHZQ^G;}MXNwP}yK`S+iE}|`XU}dA z@1%X_o$Yj~=nM|YJ9^r?Lz-sn(#3+%BM5V$s*tM7_YG&tVEFdl6-8jcFLsQji&M<%Q&ZId&s`4mv{R-EuQXc+J&C(E>E`)x>;y< zZ|n4UV{GtUsUCvzysZHH8PZ~{R@v^{9JJbx>BtYP74h*9jun3PrGlJ2kxar?t&+Bw zBR?!nYl7xrEPZ%cEqwd1J7QsS8U{jPT9|@{$pj6Gj%nda2q$;g9Hb*HiL6f3CaEAg z+M^(b)V_Gq{b4pq$+PI9;X;QLst}shOdt9BQw#a2Ek(sL-Ltb8pO$P+ap*NZ{S=Y# z=}?ia)kWk&o{dc;=k79P*E2)x7SgYT>`FdABJ$eZ$>q?#IA3Bcsv{xd?ugO@O*BM8`2-tjnPtvAu73n3&tuUE z1Z{Z`NHa{4cFf0;LmesP&$fhbAt?z+nofSAFOz{OnnwE5;{V5#J!Lb?_#_*lUlXO2 zRHtZ{95+9r0&3w;dRXYrL&ezlkr0N-W`ytcD=wlb`O-0QRLL#AY>CeIsmY3#Bl=6|Eq#0Q&c`+^BxXB59DPum2f=-(7Y1UQpTiA+e7v`@3$mP<6tTS zufgyyNf)YF*9>72JAzigYZFu2#!N4!zz5&v&4FfKa*u1b2ejKo?RFv-rjp&6Iz8xU#^kU}rmF#0MUDTC zOckrk)fJy(0ESdyE{8mvwIeT3R42}F3KXrGET{^{Gq$R8jMA3!IYKxF1~MyqyDOSD zSXde;TqrJD8z@{CEGi2WEfVY32a1}Zj9_lT!` z&)GbX*O1A)QjoGp!MrMG!TqX2v#3_~sy=H`mF(4-3Jj@IU`TBuxKHWp^OjX}rz<(| zoUYb1I25mQ3k(fb#T$uA2ztY+fS@-VGIVDyNL(>n)*9s;=t7*ARD$gXJPJF60xF^) zRC;(;5=X#@ksy%iG>rA1$lc2(y)-2nR68Zn(BhMkksT@IDwkfa0sf|(^+JB8frTt@V&5MdDE z7KEh;w<6p@<`vpzQND^6DGWhOM2VpFM(LM`MIBo>LNc1nT9`Eb1>-liWH>lhzTH*L^H71g>UksGUAOIV`iu^$-+gq~ppulTA z-HJOr=e{JTyPJ1PI<3>|@i~1xs0*V+4{0MD($cf7Bc$>A+=8#g*WLw!8J7igNm5^u z^^ir1(uU0GN}vo7GE7BK#hJ=^TGAbuJ<1FrHDqh? zTm$qAw_(if2*{L|G8N?-B~=bXN}zpLr%yNr=>~-RF!d1xx;w?)80d@e&<#>ZAE9Fh zbVjE~*k_4smM)1X1r@^U2fzg=O>LaCX9esPVaAHg27t_FxtQ7PE^(lsaooN-U|%iT zS5Jb!VEsD_)&jyiZf_3Qn?-vwgohcXa3Lo)tq(M{Nbac+-=2VPV0y?ohH)(7KBoxb z;GVRXoUvC=*sCX9WdYaxfNOEExI9=mE9jaLEUT^vl+{m`%z<_3cw^92d^|O5WE?f; z7_}qqbA&L$g$w6Rj5Q<7sEp7U3c%<42QI)9Hi93&8IhJ=2+x#5`?a>9VedQcj(~gD zxO;cNy<0Twh4-9{(j@K!?SU!PDP&m3N8SeVJZlEH%el}t^& zQt?XB0z*Bkc(qmmL8n*+1f9xJ%Fq*^Q(u1ZDu&AjUBw=R@j$6y0;w?8ujo@mWx|Qe zWr8DNx;TKfWb1D-<$atFRMZcXDRZ6ngX#$}VH$OCqj7HogEwT*AgF zI?(U{hE&0%)Zu}Nq!KZygf5p2k84j0+vhbz*$%wH&5mjKD_sVYahNgm{a}1 zjAg;>{D&Jr6?KGV+|0~#3;=Mz%uLWVE33k4MJ})zf?AUawEzq&&nGjs!ZX&232Q|# zGxy;FTy(cg<`e~TY6Ce7g09kF-i%;QG1VvMPg|K6z|w4HY_rZW@u-*rgb^-;t(4C$ zggI7i3~CJ#meWF0uV18Cq-I`KbBhvWE;V^S!91`VJCPs^BRqg`96_?}p(_ueH3BHt zy+Z)|qyNH6m7)~HMFdP5G-F1=%9PAJIWx?xU==x+;u;9r%rP@7$9!V*#xPT+FkT96 zY&r%;Vc{kIXpCf2Z1vZaKFc-%bC5eIJ19S>IH)|z49W+=GD(`s*DJ3F*>D%F^Ku(H>C+w8$MU+Ezn{BB)e%)DU~et+6-3g) zjy*Zz?uc%*rxc-Vsh$l9)9TU2%BA(_WBz`5cl9M@qw$tLmzA9*l|ArXWgDiK%^UmJ zYZxH`+GdP}^NCZma(rSp%O`=7R3535+DqhMCS`e3EJvRV_SB#PNfrRw5&gs`(1Y6gY?XiihK4R+dDer-u~hKq55ItolA$7p2!`u zJY71tIB0bRtTP9f2ldwbmIv%*0ej_yzEadzPUgBsvma=Gxcx-NgIyzC!xba!v8v&! zvE-AQK+TFF706&?%6;vDv>Ac4stIG2Xsr6CEN&@J+>7au!I3kXEILZzJ+Wl0I#5x6 zT0T*+Qmj}xv~FmfxMsuPlF5{eQN@_*bh7B)F7DVHaPI^CX(@LF2cOg?4iqauP^#-SrC`@n0g)sT9rNMQxsrj3i-SCs)tQw!MwxMc*)3|fW1_-m!9T_)Ud!6|~`DFl#AD*4|5*F|>ucQMiL=9lJw z3=|4yFkwD`ixW*gyO07Cyi_W5Mr1I<%X&?(nqi^z0QzFms5E zFE$ZNy^H;nEU3>()0>A)w_Z1R9ZbJKQVD3L#@w@(w2@rVk`J%{#`-)q_jj0a5` zE#%@lgg-~B&7_uiAmY;lEq6h6s8<>?Od|TkO-pSF2zv>_F5eL$w+8P zLn_DI0%~GJa-&KK(uf*B)FfkhEooU=R;q_Y+E}7JESf|P$orHis@KAy@GbgSER=h8 z=^RB|pF+|Wt7WjkjJx=-m}PibR9MPlqK}tHge8%YnIv+TBsnNT(Qhd!gra; z!M(Z+SZ6@Zgkswi#}yZ=uKlV$)rgifoyyHdi3xko=mTcRcly;nM>K%x(39|67NZ6# z&_!CnEBjyuEnh@iU_gC0S>30hZGirJ48(sm1-H{zNXxhC{S|}-U!ZS?>OKZiWTu2G~;_blnbQ1oA+}u#gCX;6y z%9OuHnjRn@Hk23+qBAP+h2zAz!oh;+XN7gUVH)F5wRQy4Vzt6)fRK(d43Fk2+|b^K zVd}2EExsLa4#m?At?(G8{v*O!gx3+?M0f+hYXESP_g7R7B*Jp%?E`=nAZWG?YJx`d zpmH+BaUXxI%fm`?)vDA4z1kFJA}~n~u3EG#7_E{O3+J8G zoQ7K02ej)&?fPih!DNeQD;n(>%RgCvI#)C|g%vU^AK-HYC|^asT;(1}hq7@CoCgE) z`~rD>9`o|N1qtwZRky$fpHq37WlGtp8H)_dSlMY-0Uj!mVMBxiDSUzOO~V|52zrRP z)j1y@$*iGlrB)x|uMm3#B==k|YWF|@Jg5QGDs*Ay+n|p%DqyjQLM}0`UQk@BfFAl6 zVVi$o^?Ww#OCTe@te9LQeI`ByZ3 zpJPvwGixUjjS#W{r*R@U_A_U*bI8xu%}Q4>y42xW0d0nuSrgFC7UwpOYa3xPR2RUM zG8Fsu>swfkV@YpwauF6PU^&pQbCD)on50_`(XF$f!9?*adA>PEV*)oWh~Dp!zciO+ zj?4wyKFpzItP&;>Aj~4bae2Kc&k(cfVK&X#(3-UmTdHg85n4UeR@Sr2N6#}uao2c+ zZ9QOk{xx}dgLyWUF#4_@-qYCvC)I9mOP9M>8utXjf@EMxf)!jLg?k@ty`^JO8%qou zU1n53gcM9hWkxExc9F)7WeO)a`~5d;?3J_Kq|xl(*S1&9{*-*)vzr|toxOJ00o~G@rF#zQbLC4r z;YRX&Z%y?WHYaZCz&s>_#SE^Iz$Fo(8mS&a`c5INMejO&}`##2ChX_AK zh)19nT&zdxk~jgktC9h>k(|5Q5>E@+jD;KqJ~!nHuVMm(8f;_+XcQo!K(y4tdmgl9%bg8J8;>-KS+mDYa{{J0qG`@qTNe8c)za`GuMop)&@*#Mbp|z@*A%u@v_p($ltuf|1-i${o}r?S>p;={-BLhO}%h_ zC0TpTskke_bOR@Jr03HtvS9c7GR2m){+vBGurhgfD|u{hA$(W*-`i{B6teDlK0PU` zN+!C#ei_&OssFaV^(;7+_&?}xV)bnIu|5@_J*7+j+`o8WADhq}gs`HiunGTf_Wy{@ zQXDFvn`+#mOg(&=uO{LK+k*pd3mAqJht^034}Eam#W$Dy{l-Fe2FbfAqZn^E1&bLd`$ zu$FAO`39CJ@84WHi}KM!P_FPW!jAwRRZ4tFqb@OT((QLE={Pvs{7o@#^8CSqfxS=& z7)|kFTu27DdxS@@Xq2%o4cIv0%p79rsl>HN4wY}1ho*z`hh5z*?R<>6C4L*g1p5&V z06coxA$p7)ElLDQq_1SiV}r#Ox_h)>HcnaqFaSLXvd}$-p;75KO$~kkBT!cnjvTpl zSr#3_5%kjjJpdJg27%YPrp~iKi0r&I*8;nDMnJe3Wyt(ENrtqN(OVaqxgrh7gCfm4 zs+8f{aaBH8Fx1K9t6S6cD>=3K9D_KxEX=JSh2KA)HR&ulu?e@*m9W51mn75+LL@oO zzxYr(n+fMGqk9APnz7j@uK}Th)>r@#I<^$lFH_E|$idqluzyfoA9UuNP=V4=k_Hha zaP9&C1(N-{SpLaP6a*hfAd~}tr(IWIs#kT3aDc00g+SRZ=7xh^i@aV zV3TP)o=^UMpF{8R&54C}zy;gtA}4GBjn^5W$lpq?*j5l^T&k}QWLzq_{fExt|1Z^% z^7~V>3K|8_ET`lLEPK8%u4A~D5lA*Oyl4An$l5jmjOn%YwV}gjvp&Kv>6`R}0){*?BzeGYW zNh3*BOxTZb9YRdr-HtixFzf)jZba8j9wREg1G8 zhQ;JkRI~|$7)zxYC886&)L_~&b$&~Qlq9%r!!jr_-I%F|1rmkj7)S-zL3BkW)wC@; z$}v~sG$A#DDPz(r)_)IRge1Mtbq7{+JHimcVT4-|s8~C|!kB~nay;m?e1gdlN+8Vp z8(4r*2fB`|doXh$$SXtAP{l2K5A7YEaclozKk&ZJIAlDuba3feqvdGIk(A+vQRS1G zpJ+~0jn$9mF9_r>7&k5q7#9NXs}n$#dHmKQ^4kZEu5S>9HfO+E7G}1yC<+l#6tjyu)`9P3anI?iiAIFhFyS?@QD#<6v4jm)b_joPLRZakd>&v=GrO{rqM zux3qxVj^1!Arl1(2$?8#g8OZwW?j1M?bMoeDY8I{VqH3EesohdwFM!QOt+(k;@quZ zvw@ZQUKszU{BJ+HmX%LHA~e7{zS96p4n9`OK1ZH=YC$(bh{3cM?S%Kx_~5?I)dsNih=I0OXz z2Tr^xQ$B^s{*A18`f)CmCG0Z`XN+OU^9WQD(9HyHFhZtIH|(F zWWh5K2Q0`Hl6lXjF8n~7aA^5Z&$u=>pv?_4GF`zWSe$?n!G*S`elYv!3Xo9~fl1TH z^{#;46^7s6B!#t%Hu)R_OUJYM&a;M;;k6S6r)Y4#lU+SlKao9O%$|R8B>?2j$|Hei zvnztu+MuIQEUpvn^CxYNV<{t7o|r9W&l|VZ1#ETj3ln?dIR@Z!gdr8oU}G9t`CM*p z(B%AF#$_dbm}ohYb2#tLJRr(g1k@Uf4mS)f8a9K@kyk|Ser{cfT%ns(nSv6yXJg7ZIo|mqv6KoUsYgh<=2ne~dr}D~zt# z0LKYWQNzQ_SO6X1Q|NjPAY|%l_j=*9tOf7t!}Ww~+Ic9nonE6`|Ds{z>yStXQrC+z z8bxoGLIXph)BkUVJ;M z3Y9o!)pWT$o1QJo?O|>ur^uO%H;c*nj~9=}R|n#&!^|G?_Ae{gUZOg^VT;3uo_f=gcxmFyJW~9+g*^GDbyLx$ao=3fIR%qa7fxWzy=>6p49i$E zT+TvJGG!H_sc15#IOtp_RyPEy*NEV7t|O&ySQO{fkcJGs{(OQtOU$jDFjb1C$`4Xz zkf&bHf(ypxDv(Br7BQu8!dNI83(uPJMr%Y96j%|o6rZq(mf7&0EiI4K&?L@U8kn_O zbT+{Yw^7JTuUlL5MqQ%09Nyr!B}~hLv@oqxLGS^L!e{MB^+t78Fy9NBhT<<_41HhM6SN`=%{Fn7vt?cST?x z49jMCgRaVN9~twdi8J#cJt=$LgoCpL2IpTZi%H56wb^GAvjHzm#)Dlv5%0K2Kvw^1 zkwd0l!%k|G4y}SE0-Qld(KFoB8qqNu-Vex=znaJ8B$7{l^}6LPoZBch;Q|UWmLM?$ z`OU8_~HU^4c@v{3Lf^w85dPv{9C)5(hsoFP)Ef3- z)iH6fgRFQrPyZ1nMPQXsOfNa`t~o=dPMI{@1Lm1$%yTEqbH{d$n;Qc7!tws#61e+B zUV3+{0fo=B9}#Lv>6xmmc=&aomj13(7fM>R%j@guT1$V;wL!p3971EOf}HrXiM^Q& zpUKq2jlJ0KoBBBNhuz6axXY3Zmj+LkrqoUURUcl$h2z(0zw^ToVHhZn2r)Wrr?HV# z`c;v_pu?h&e#vwmn@LAlN<|!2E4e!8$TVTIu_<_5C;etCs}g^mgPWiS zx%U=wX-cx>JtukL&uWX*dbn8#S*1s?gO40YN+*UZ45aQR4TD!d4x`%=H5&q}6_L^F_7Tv70MYx+QZD#$8zF8pbe$TT(FH3cBIO{S8b_uh+hW9_IwhHN^VW+4SmBd@T7>4P2GdZ~H?tQqg-hIgUNHw6H*$|N@qfF zZqy4$M^Zglq8kCrkmy!PbjPZF8#BxK_Ew+pGA5-WB|^)>4_3#m#<9}Fk5%8#CNF;Q zHMvi(X6ya0{aGf5IX^X3U(}%suOfSpNZb7 z(<5-Xpmsq-D4}$?+h@)Y=0KW|0#3MmZkJJ->XZ#CX_9Ww9-ly&K!&ZNOb`#QkgBDn z2qRlsgeokN_Fe*}u_EAcw(uZEn9y~aA>IF#!a-Nw`d@a=!IAD@idM>g6}EUGdF#J+ z=byzXnH`oTt2eT689LxvaB|CpYmMky6HIW9)IZ!HCX~QC>G%GhYgwyGdq!oQP+5o5 zM%4k^%yCtDKvgcP%E?C`S(PrXvC*ZYcPamhR3MWGvYdK$tm#uIj5Kc6}ji*t!bGG(0MKgsoqch zmQPAp_P1n7xU?3fZb%8IKCQcXVG@bKxfGI90e%oEq$)a2BbPE1-`dmN*1o-^*9|ui z#>i9QJW$Bq3+F2GplLVy<>xlaXT66>S&54%(^2~roiex}NHZuld2n_JspRnaS!Hw! zkO}^f3U+Zqi@STT#NY~VFI>V9{tzKgK<*R{k}u9@n6SD~oENOxa93FHW5hG0@QdsO zmb*6!4MxzvcNgIaZZ>skWGIr)TnK1;g|=J9QtchZm_UoEbe5Q zSapS1*(4UM5wm0Og>Vw4yKvr0j29LUWMC;WCcRRGs%00^dJakd{5$FY?$-Sau{O9j z@Quj|{2yP?vv_L!_1{wB=m-N7I={?R%z>UB^Dq7K1zGhgILM<2Xz9ZrXms_!K{%aS zR2H}~4yRj4=H_`YQv4DFUM92uIYYYdJr_rjE|Z7BHjzvY|1(ul4`DC)pZceP8&Kf5 zgI5Ar7-#^UbO{o2y=xrRkqq7a$HiWrOAyAVjHGfj0C)zuODRU;cdeKX*nRj4fdiMGu zR;F(Oe(LV=x$#cD7li6-7*B1kR&>$r_}EdmjI|Gh@H#VA!je7!PHrWs_RIB%2bNg9J1IN&9%*d`nw8s>o(jRg|iV zR0;ePqfx>L$h-p1E1~|VLC!if8^H~+jQ2;+Rms^Oa|3eL20M+*V6YQ*BCHK|B9W*B z1)C0+@-G8S38A0Iq6|qa+(W+{hE@XOu|{CLNPLO>=$1IPY@m!`lSRfn{SC`GHVx$T zWiTX%Ga(kwgd$OzO)@O-oADeC|9pDvJ~-jF1QQbB4acRjy71Ar;#gzm)f}4?W~O1c zBA$h9SQP9>wJO$XRkEuQjm@Ki(v;+y7*#?#^GlUuV zL~t|b;h=%cG`vDFp9W3g(Ka=^w8>P2sV;`A*c_;iiNYFoHOIoHZ5p=8mF!~L8JA$< z(XZ8PTDz7l3^P&4?qXS3X1^KF(eTfw$8M8pu^~~&W$MaH+(W z+Q3m0aSMnF6)QSgsY^xFy41ek4ISz@k{VRn&)U|ZLSNpuRo~}1lNGhg_x--#cXQ`| z&w0*%&U2pUIZLiSr96F)GV<|=2rUPX8x(PhUeyUV(jgl821>@ zaZkW6^sKFE;zS~nMuixEJ6XF>Ot_s}yGTp~T&&^nPh21-LrASyEvCYE2~AR%X`w>d zCDw=;Kv=qk6H`P5`%4oE&$*U~nU4`D1b(4M3NWiNN_0Tl8$v~}Vit-Y%?NAx?rytrUv@SBaBv@8Z_37K_2NMw|jzCr$-i zD^3GkCoU4F-%gzC#S-vs5N7~x6c>x7P-MMW2EI*VIiORV3AmZ{%`E7f+1K?=gIEFK z;&4CIit`}7ORNG~qc~p+pDiL;&N-HNXDENmlWJ;HzeJRDNS#6dq}F;>VS7l_GUx&K z9Dr;4xS*EPWY8aK_O4ul)Y=er9~b%ys<@;Kzs2geCG}bEvprx7s>6iDAoni=U9C;A z7D1-_xn9o2ttBG2mLKV0@15FLNYq9w*BvT1sN*!bbhU1-!=IQIT*ad zIT1D^G$3q3SOgG=-rgpMR0wI3$I zEMzq3f^qSbjHZyZ9newZ=mOF`qw|>KMAdM&#M7=v!>KL7Ac$ce4M+E3-y}rRt^^Bx zH%TK%&Eu$ov`WKu(5H`D==(W#`elMTnMtWoozgfduTz*%ipCQ`8(&VLc_~IJzoMk| zi5AM=Wa^sB^^gre>{LN+Dp9+QHB{=FkTiUv=pito;~O+K(w!5V%8v_CdrTwU1x?m@ z%)>sqCdC>t;sIK}jxM`o5)>V3kAV){WVAL%j--Q9^&@UEf*yR)+;}qbF#)6^{6Y`m zS|5OB=!cte?}TO;{!g1hFDKa)QJy5aEG01_X{4-g*46f1b+u7B$D|OW*_Rh%udo9cld=K0t0a%GnMmb>1WCD3e)I#B`eIH6fB$OCWk8}y(g8PIbf$tr)ZUsVn1C! z#YWGh=oDikd|is@(G;q@g-bxNJG6?23Zq&AgO7Ef~Ssrd{ULHv1KX;21>HaK{$i zxY$Fq!(pfQJD%0ef((P0>$Tugx-R>Bn-Xh`+}RZSoO14L89{fswz639nmV7X1fgZp*$+~ft>{99Q(c;3p^*64+|iYp8?THzg~3P|s01t7hnSD@Qi5xp>% z&d#xDAAnxy1q-8tP-jkz356_lPUsmHT?+4=IeSUbuZE|5*zjE0?t!EukEr>Uxk6ug z#OzQ{$Z?#M?+lxeK{%b2SE)yN5I&lzjvmZ2XPk_93`%-pv^2sU*qzD|ZbI_N-Zl*} z(wcv&RAc`ANVD`Dc?E8P%DjcNg%3CY63mwY^j%A?)J7;U`yl~ zsR;o)LfVF~gGLohN@gr2p&uF3p^$DHaWj^@aWkz*pGhApNS}h*-4P)%_Cy5~Qbitg z?L^pxup41M!XAXqLxr|k4d}WVz@Z7Lr_rj03S*V)!btlNMrV~<1Ky-0x~ecc@+vsp z9uUjG+{yy)U4?PPVM5jdK!!C&<@qg%{j<(lieyXCpfz32SUzA~;j^xgtt)6>RvfKx z#M0m-ZN@k@MW)tpwpo`s0ht;gh;Z#%P)FOR?=u8*MvAx=%vEZlXnl4j{jkVXo65af zWS#yPMj?*NFFI2c zdr!}PhnUe~-Ougk_bc`*_Y3<~yY#eT>{iK-XondE?*>%P_I42K0fT#I zLj#y)Oz@+;4Hz21*xD(%Mk(~Mn$1($oEU4lBA|3iTigNFw%zQjZ;+hstuQ6q+awPZ z=EkWH+?)ecV7ecRyj0P*Wd`}q&{l$9^%SNbyN%9(SAGIN3g z7nOEZ#YLt2?TP+`On+7>eR{Epu3TnUCt_Kg-n(nrQ-mC%8Ox^-CoDGR#yyEOW>f7a z;BrIX!(fDF+*9U2!H0T5+1%FD3SBhTY9}?Xs33QF*Q_`~Qpd<% zNEFaCI2*RQ>bJS}22>!6ceZ$>XQ^#fIXO&MuS!oanETQWMfOAv#@Kx^`9V$-l`&*Z z?4K%I9q@V&ugWADKgAk6k2O$9FQ9v35z^0)_ALGV>Jt7Zgr=-1&>Ca=cFD$cc?iI$`0|L3VLcWCsz)*}I}*SgrDqhVEEbMpSfPU4fv39dXmQ{5>*pxCKY>9 z#=@mMFjWnA8{7$2IWvqk$b^x55g$GwkND z-DLNOi5gbR6m&r4_DC-0_7x}ifXV}c1tuuti2c;Mp@10af(?b_G`)F4e)u@0Okdg% z-_;3)2O>leLYv#15}2z}i&P9WhklHw0$Mj@>Z$iMZFhy(4XuJe1mj~7?Fr!FkhP0Z z0-Ei6>pfecLZXBUJP_lNoDD7%lJ#SH<7EhuUQ9=P1mPIQ95YfHceXa5esSZnhSqJI z+Vui~8965;4*e!ux7HtJ>Q)8$Y>m;cvv=3_&6RaI{b0%FgxIR#`jm>(8FG3BzQ&54#X$v`N0yKwTSkw{Z;9`T7=kHf$~dl+P18=P0O0Me z^3%@x1iG}|s2|RoL`-IR(;wHTuexMS_F1zAE$M!UfRcoB$b{lY)Z6mVB85 zxPs7K9Sq|l%s*Q4{Urmsc%Lp_*2NFS*t!>A8`N8aoIwLd`v_d3$(t%^*Ctb}-yDaT zB?JYfE%Gu4a0LM})6$choOQ{N^wqf6wx(0h#+^iIOAvk28cB{WRHUbih&2!Se_6O=WTUxbZ>)5YworL+GK>s_-sk)*zl)%pP80>U6bK-bW=vlTeG zZJVprE!|HKHxw8qMv>m9!8c9x&kcIR$B^Q4YywV_ecTYOKi%a$Ao4u9pZ=)v98`_t zzQpio0BBqom!3i0Tc*UTxQN)kDL!4YoKoV`&5+A$2XwWdp)}b;`dsguTk1*8MMw*F zbvb;b=?EoA;3X_m!i-1Oh)iW@8``$Fx44+WH5yy`6yyGZRrltmC@m;=&Ia?ty`J9J zRG8{Fr~2(VLA7E5fh8Ow@DkvFa;~gQmQ&{j!{}#C4XIloSs((|wGA#d!`pUxm=PaP z?{Z0-+hCpW7~Roqn!)D0KD52$Y-w`3>$f}GLmUw){U=rhbCRxMc7pRpu!KTuf&b#@ znP!IxtqAGA5ild^ZwL+atL8!h);C<5ck;G&C3%G&Ys*Um^NkB=w>$R?k5fPk%kU;o z{bnds`U(?%&C#oE2f?bL-R-%ggFe^3W?dAdl5iv>0YQZjgYXKbC`3@B*N6~~5Qd;f z=)h1Dy4ZlQx<;UPGX%J!ss6BucJ7E3>_D62eQ?JCqVsF@J!R*#iLy3vh)$Bs-i7W9 zByOygAywbHGqg3}hJNu@?tF_tWp%qZi)V41uq>PuoWhVg4pf{j0<9N|KL+NmB3cDu z45HNIVqv0?5i)MeK^iOn8b^P&dm7krmvyHh(5U|cUq92*lKJKnrr5s<^e;p&yWn9Ye)sgHKYRH8dCQyxEjVqo4sG|*{dOx z?!I+5X`!Fr8V82E?zYs39w^F@6tRnxwYYfC6-4K-tvV z&$L~=7EV zSwT#T$rTO}j9pJhEo@GSOw?5qwCZ-MBlCOAA9=wqldPwUZ zftou^^?8fwqa=~EMgy@hMC>6_xMy@%6gFHZ(f=rkR}^`svwg(GAKgf1(@lqL>R55I zSll0vd#==Z$gU0x8F%#Q1hY<@@&Lbu3={f)R8OWK>Pe$_+`T+ibzl}-d5$k(st4!R z@Y9@#D^Ej@>cE!JMq`IsO7m3EvL|9=O&y$PBp!UlX%Fz7W}fR%cffLWmL+e_SRfd; z$(;!kyAyN2E@TeSZcL-Eo{aBG!c9{tneAT%1lRV>E-|2heO9R+)1d}&XhEp$ zZFfQFE}VCu{)eS^Bi0_{!SZTnYkiY=eK{={0na<6R)7;>V>J(L#yW`=!p16x#f9p= zgZ^?#RutP9V=Chb3`r!kwJ)!0Ke{cLAs^5%eBKSb=%31mLB8^x^v!q&ktxmZ@xL8X*V_v2iC)6~l#Gpm?nt74L^3WO_ChD`C^PwodB@qE^d(#Y1uN^rHv8-x2ckL$}Yol$m|J z_{iR4drwU|+S%XPH@83Pf%$#&PiMT6{X(|in$lnOc+Jt8PFW1^fU(wRtd)(m^p>X+Y12ctu2r?7Z53cU?h85GD<3&R6v_>cdp(boqsDobK6~sz_=?^Yk17cH$h+pT{Y1gH!stJ~ zGvcETjhHyBaMn=q_({}4w*xU{nD~M>;e9;seD3E0>%-q@+$g--QNpz0y`Jr8r!)5ofqq=1f5q}_y4qf`t=jm)cjfrtgT_~04CS)@i`Ml!@`QrI9cde zje?7rgG`wWYeb-}!GJo1bqE!do=jD3LgyU%(#f=H5uGl8fCBc0ht-pWa!%TUz$|N4 z9LxAR-M1B^uzli6>Uc719TTG0#bLp7G2&)~280%D2~%TCh9@Eo6`It9t{v!VMi*08 zOjB*6eNX0hr7+DC(gUckq!x_D^inT+Sr4=!jcKGYy)vqsq*e@N>WEbgcd7&7sAq;v zjgWrXfw9Jv%o6l2C0&5*Jm5N|!*-@dwaj%o$PqoZ9Tid~W145IcSki($SU8B1@1!F zgRmDtA~AqYY$qEZsLk*KFFv%-`wj*n`~j8F8CA$a@1kG-B&BRHHlfdRU&#X{{W}hq z_m%f3dzRg;?ol6ae5(1!&8O{WR=v6Q>{@y0YPoL1z`TvVc^e1v>wWq4-NK+fmo7Q% zpg(#lDwE`vqb4gK)nwL0pS3W^Z6~P75K)u0(6^sj_7Bw)(@{)4b^L1@kAD5kR1ozr zFQ0Dvzf(+MpqP@W;+eg4`O{W<_cKm^Jqren zHoDWhvIO?D$15xOUsqO9Pyhc>RD;HhkgoA3(0ltc$RzKJUNf1s%EG-Cvn+Z=Chtq< z!Q;!+uACzHCY7u#6wYU>fO5W20Lu9(N#OpCMY}p(@tfq5)k%u?l7!Xi)bL!L>Im}N zQQG#LO}G^pbi4P5&n+k9iGO|^QSp2Manl{o&*<`F7Vjej032G08FA}JsUM*00>U7| zZxQZC_z+;UhfN$1Qz#aU-Srf1EI%T z`qD2Isvlsq15|hBN&asfeeukkNe7W~2!RQ&9&}NFfU(61dlii?*ynJG^hE@#V57k&BFl&-t6 zAcQ_pChp%tC~V^JaY6*xFB-FKUU+&v0K5ZXi+o{=j_O82@4LJIvB|aRXZpNyjk}Q#(h{brj(dgkuOyi(Q1w0>Vx>-<3j)@fPO( zHUb;*U!rS_Pt?+VlQHcH1jZ*%qU$FB0ps>2w;PUg>+xz(yVSO;NrZK`5}A&E@n%Ym z7a~26%!3UA=22RO4dl|l%_KoCnr?r~u|jq%k=JgOn>*yXP8jKVd^L{QJbnod<1&5& ze(L!q{4C|0aa8BQxU(p1_^oX80f7YAy#Ou=(RbeZNgjb}{j(8-K@7d#Q*ZaE41dR7 z`T_O?6vIh=x>PQ+pyYr(gJPuM>}u-42>abR|{uKfn#pmd$bMe*J853jjfpN(KEd5(XcVy4~B@BNI zvwV@7f1OC!=3SBWDv}Y-3$cs=XMyn*oPzVPWmu8Vg3 zx|r;yPyc%L;-n}uEWu#2P0pI(%c_#W@ogJ#Ll%e6GUas4IZK&rDf3UBCR+;!6DAGC zEtQL_e8sggIO3MlyL<&VTu=0;WcnS&uv$vA!_}2Uyt zqH^S@9KSL1c!_K*hSwiE=~SH@I}=|2SaH>I-Yj3jKAue2xM=4^9F&=AK(ryq|v z7d1tWngR#)a%|r5&ViV6Urc$BGt;Q|Y|%q;H^@^L_@=^m-2kuOF_j+l<*tNLn@Gn| zoBD5#+Qb}K&X%~CbXk`+6rBcmZ7>{91o6602x#!V%6Ou=fuQ1n;RM&2CFfVy`BppS zx%dv!rT&?GvVr#a-?luCoPP!YE^@FXKxnGa_V+E5Q3%3K2d2r{JY~}a#h)7{1(yU* z$SODiJ_%tvec}Ddi3?<9`GB(Ar!0R@S$R%bIiQ^9Q_hn|Uau|?&WDdJHO|tM3v;xG zF%N{D-qs60FL7M&HOd|?TkSGt}lteiWQ};0K_%Kd&1Sv=9LmyfTnNa=@@CVfG z&b=*da2}1q`nrm~h6xaU;Qif)KOBfd5pVm9!}r-Xj(j8^4A4oDtEd)ZQGZ#ll|2a-aC?N8$ z?@`jrdZP-iSeQMULk;#BaGiW~PXlkV;{EK26lot+A8w7X0c2yxWTS-+ejIOChHm10 zh|L_0z`BdtKgqTK3|l>ywR+^9c%fqQooToS;s3kWziZmF7P{eBPshTuK=`wfJAeCSh45=*I& zeuuG?jZLt)M#B~Ap9UZ4Q$nYGdVqgdpznSb>mB-Z8JY7j*8LNNO9*HVN*57WPD+-O z^m`1#lBNF*a6&PX8xTGt)cx6N{yo9_?q{n=0%jKoA3G~^hjtPeXI^~L9W8KCw z3usrNoj4D-ucf~d`u#sut1e={HBk98GyVKeAK5Qrb^aSt!Djm3@WYRkPe1!}3;(I$ zt^2%;uVY2)LKYC5$+p9>EGw4BsYVu7fm@E${;ljHs33Mmmle@KOqTze4yLL4(=7 zfDp+_b2b%Cl1d>YAi%#Mc$^MHh=~}3nOT%p*B+0=*z1p&j`80W3<;~Ruf)LmddY$z z7qNiCY3mHv*d?<0H#0P+WiKM*-KF7Y3j-J7JJ7g4x3R%9jv1v)y~D z7u@Y|qh5M@q<(CIN;`WM3X*KZ0t0F{oQB}V)B7>bgT2!gBrWn&a0f`U6b7|}!-TJz z<9hE^-ee`Q zWvo2ye~mWGo_2DZ1CiDSvMP*-mUP=$OTYVl0u5cVq*}D0FJFv9b^NltIY#f2|~=i zi(3N0%qUuoRXB*iq#RrK)S#D{D$Evn1HIVvK$xd#8@qWF(7`;cceR2xghe83!}Rt< zk}t{6d+*YdtzGl6o7gem!?Jj9QP*jR@Tczd`sI0oIV51nWdj;xMt5 zFY*Z2yrUgO+}g^ef{-8Mt~_2#=23=at5EBeXmyR=9 z#uBp}La?4JAf(U`f!m#I>K$(6!US{+w<3m>{blAjE dNaG6T6MxQRf9{;1kV+DQ+(cN^`?iJT{tui?ih%$C diff --git a/litellm/integrations/litedebugger.py b/litellm/integrations/litedebugger.py index 5187d555f82..314fb7568bd 100644 --- a/litellm/integrations/litedebugger.py +++ b/litellm/integrations/litedebugger.py @@ -1,5 +1,5 @@ import requests, traceback, json, os - +import types class LiteDebugger: user_email = None @@ -7,13 +7,12 @@ class LiteDebugger: def __init__(self, email=None): self.api_url = "https://api.litellm.ai/debugger" - # self.api_url = "http://0.0.0.0:4000/debugger" self.validate_environment(email) pass def validate_environment(self, email): try: - self.user_email = os.getenv("LITELLM_EMAIL") or email + self.user_email = (email or os.getenv("LITELLM_TOKEN") or os.getenv("LITELLM_EMAIL")) self.dashboard_url = "https://admin.litellm.ai/" + self.user_email try: print( @@ -23,11 +22,11 @@ class LiteDebugger: print(f"Here's your LiteLLM Dashboard 👉 {self.dashboard_url}") if self.user_email == None: raise Exception( - "[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_EMAIL. Set it in your environment. Eg.: os.environ['LITELLM_EMAIL']= " + "[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_TOKEN. Set it in your environment. Eg.: os.environ['LITELLM_TOKEN']= " ) except Exception as e: raise ValueError( - "[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_EMAIL. Set it in your environment. Eg.: os.environ['LITELLM_EMAIL']= " + "[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_TOKEN. Set it in your environment. Eg.: os.environ['LITELLM_TOKEN']= " ) def input_log_event( @@ -36,6 +35,7 @@ class LiteDebugger: messages, end_user, litellm_call_id, + call_type, print_verbose, litellm_params, optional_params, @@ -52,39 +52,76 @@ class LiteDebugger: updated_litellm_params = remove_key_value(litellm_params, "logger_fn") - litellm_data_obj = { - "model": model, - "messages": messages, - "end_user": end_user, - "status": "initiated", - "litellm_call_id": litellm_call_id, - "user_email": self.user_email, - "litellm_params": updated_litellm_params, - "optional_params": optional_params, - } - print_verbose( - f"LiteLLMDebugger: Logging - logged data obj {litellm_data_obj}" - ) - response = requests.post( - url=self.api_url, - headers={"content-type": "application/json"}, - data=json.dumps(litellm_data_obj), - ) - print_verbose(f"LiteDebugger: api response - {response.text}") + if call_type == "embedding": + for message in messages: # assuming the input is a list as required by the embedding function + litellm_data_obj = { + "model": model, + "messages": [{"role": "user", "content": message}], + "end_user": end_user, + "status": "initiated", + "litellm_call_id": litellm_call_id, + "user_email": self.user_email, + "litellm_params": updated_litellm_params, + "optional_params": optional_params, + } + print_verbose( + f"LiteLLMDebugger: Logging - logged data obj {litellm_data_obj}" + ) + response = requests.post( + url=self.api_url, + headers={"content-type": "application/json"}, + data=json.dumps(litellm_data_obj), + ) + print_verbose(f"LiteDebugger: embedding api response - {response.text}") + elif call_type == "completion": + litellm_data_obj = { + "model": model, + "messages": messages if isinstance(messages, list) else [{"role": "user", "content": messages}], + "end_user": end_user, + "status": "initiated", + "litellm_call_id": litellm_call_id, + "user_email": self.user_email, + "litellm_params": updated_litellm_params, + "optional_params": optional_params, + } + print_verbose( + f"LiteLLMDebugger: Logging - logged data obj {litellm_data_obj}" + ) + response = requests.post( + url=self.api_url, + headers={"content-type": "application/json"}, + data=json.dumps(litellm_data_obj), + ) + print_verbose(f"LiteDebugger: completion api response - {response.text}") except: print_verbose( f"[Non-Blocking Error] LiteDebugger: Logging Error - {traceback.format_exc()}" ) pass - def post_call_log_event(self, original_response, litellm_call_id, print_verbose): + def post_call_log_event(self, original_response, litellm_call_id, print_verbose, call_type, stream): try: - litellm_data_obj = { - "status": "received", - "additional_details": {"original_response": original_response}, - "litellm_call_id": litellm_call_id, - "user_email": self.user_email, - } + if call_type == "embedding": + litellm_data_obj = { + "status": "received", + "additional_details": {"original_response": str(original_response["data"][0]["embedding"][:5])}, # don't store the entire vector + "litellm_call_id": litellm_call_id, + "user_email": self.user_email, + } + elif call_type == "completion" and not stream: + litellm_data_obj = { + "status": "received", + "additional_details": {"original_response": original_response}, + "litellm_call_id": litellm_call_id, + "user_email": self.user_email, + } + elif call_type == "completion" and stream: + litellm_data_obj = { + "status": "received", + "additional_details": {"original_response": "Streamed response" if isinstance(original_response, types.GeneratorType) else original_response}, + "litellm_call_id": litellm_call_id, + "user_email": self.user_email, + } response = requests.post( url=self.api_url, headers={"content-type": "application/json"}, @@ -98,32 +135,28 @@ class LiteDebugger: def log_event( self, - model, - messages, end_user, response_obj, start_time, end_time, litellm_call_id, print_verbose, + call_type, + stream = False ): try: print_verbose( - f"LiteLLMDebugger: Logging - Enters handler logging function for model {model} with response object {response_obj}" + f"LiteLLMDebugger: Logging - Enters handler logging function for function {call_type} and stream set to {stream} with response object {response_obj}" ) total_cost = 0 # [TODO] implement cost tracking response_time = (end_time - start_time).total_seconds() - if "choices" in response_obj: + if call_type == "completion" and stream == False: litellm_data_obj = { "response_time": response_time, - "model": response_obj["model"], "total_cost": total_cost, - "messages": messages, - "response": response["choices"][0]["message"]["content"], - "end_user": end_user, + "response": response_obj["choices"][0]["message"]["content"], "litellm_call_id": litellm_call_id, "status": "success", - "user_email": self.user_email, } print_verbose( f"LiteDebugger: Logging - final data object: {litellm_data_obj}" @@ -133,45 +166,26 @@ class LiteDebugger: headers={"content-type": "application/json"}, data=json.dumps(litellm_data_obj), ) - elif ( - "data" in response_obj - and isinstance(response_obj["data"], list) - and len(response_obj["data"]) > 0 - and "embedding" in response_obj["data"][0] - ): - print(f"messages: {messages}") + elif call_type == "embedding": litellm_data_obj = { "response_time": response_time, - "model": response_obj["model"], "total_cost": total_cost, - "messages": messages, "response": str(response_obj["data"][0]["embedding"][:5]), - "end_user": end_user, "litellm_call_id": litellm_call_id, "status": "success", - "user_email": self.user_email, } - print_verbose( - f"LiteDebugger: Logging - final data object: {litellm_data_obj}" - ) response = requests.post( url=self.api_url, headers={"content-type": "application/json"}, data=json.dumps(litellm_data_obj), ) - elif ( - isinstance(response_obj, object) - and response_obj.__class__.__name__ == "CustomStreamWrapper" - ): + elif call_type == "completion" and stream == True: litellm_data_obj = { "response_time": response_time, "total_cost": total_cost, - "messages": messages, - "response": "Streamed response", - "end_user": end_user, + "response": "streamed response", "litellm_call_id": litellm_call_id, "status": "success", - "user_email": self.user_email, } print_verbose( f"LiteDebugger: Logging - final data object: {litellm_data_obj}" @@ -188,7 +202,6 @@ class LiteDebugger: "response_time": response_time, "model": response_obj["model"], "total_cost": total_cost, - "messages": messages, "error": response_obj["error"], "end_user": end_user, "litellm_call_id": litellm_call_id, diff --git a/litellm/llms/ai21.py b/litellm/llms/ai21.py index 9b856be4c7d..6a22b99e897 100644 --- a/litellm/llms/ai21.py +++ b/litellm/llms/ai21.py @@ -31,7 +31,7 @@ class AI21LLM: # set the api key if self.api_key == None: raise ValueError( - "Missing Baseten API Key - A call is being made to baseten but no key is set either in the environment variables or via params" + "Missing AI21 API Key - A call is being made to ai21 but no key is set either in the environment variables or via params" ) self.api_key = api_key self.headers = { diff --git a/litellm/main.py b/litellm/main.py index f20f173ff73..9ab017bbbdb 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -92,6 +92,7 @@ def completion( custom_llm_provider=None, custom_api_base=None, litellm_call_id=None, + litellm_logging_obj=None, # model specific optional params # used by text-bison only top_k=40, @@ -100,6 +101,7 @@ def completion( ) -> ModelResponse: args = locals() try: + logging = litellm_logging_obj if fallbacks != []: return completion_with_fallbacks(**args) if litellm.model_alias_map and model in litellm.model_alias_map: @@ -151,12 +153,7 @@ def completion( litellm_call_id=litellm_call_id, model_alias_map=litellm.model_alias_map, ) - logging = Logging( - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - ) + logging.update_environment_variables(optional_params=optional_params, litellm_params=litellm_params) if custom_llm_provider == "azure": # azure configs openai.api_type = "azure" @@ -306,7 +303,7 @@ def completion( response = openai.Completion.create(model=model, prompt=prompt, **optional_params) if "stream" in optional_params and optional_params["stream"] == True: - response = CustomStreamWrapper(response, model) + response = CustomStreamWrapper(response, model, logging_obj=logging) return response ## LOGGING logging.post_call( @@ -363,7 +360,7 @@ def completion( if "stream" in optional_params and optional_params["stream"] == True: # don't try to access stream object, # let the stream handler know this is replicate - response = CustomStreamWrapper(output, "replicate") + response = CustomStreamWrapper(output, "replicate", logging_obj=logging) return response response = "" for item in output: @@ -413,7 +410,7 @@ def completion( ) if "stream" in optional_params and optional_params["stream"] == True: # don't try to access stream object, - response = CustomStreamWrapper(model_response, model) + response = CustomStreamWrapper(model_response, model, logging_obj=logging) return response response = model_response elif model in litellm.openrouter_models or custom_llm_provider == "openrouter": @@ -486,7 +483,7 @@ def completion( response = co.generate(model=model, prompt=prompt, **optional_params) if "stream" in optional_params and optional_params["stream"] == True: # don't try to access stream object, - response = CustomStreamWrapper(response, model) + response = CustomStreamWrapper(response, model, logging_obj=logging) return response ## LOGGING logging.post_call( @@ -532,7 +529,7 @@ def completion( if "stream" in optional_params and optional_params["stream"] == True: # don't try to access stream object, response = CustomStreamWrapper( - model_response, model, custom_llm_provider="huggingface" + model_response, model, custom_llm_provider="huggingface", logging_obj=logging ) return response response = model_response @@ -572,7 +569,7 @@ def completion( headers=headers, ) response = CustomStreamWrapper( - res.iter_lines(), model, custom_llm_provider="together_ai" + res.iter_lines(), model, custom_llm_provider="together_ai", logging_obj=logging ) return response else: @@ -689,7 +686,7 @@ def completion( if "stream" in optional_params and optional_params["stream"] == True: # don't try to access stream object, response = CustomStreamWrapper( - model_response, model, custom_llm_provider="ai21" + model_response, model, custom_llm_provider="ai21", logging_obj=logging ) return response @@ -732,7 +729,7 @@ def completion( if "stream" in optional_params and optional_params["stream"] == True: # don't try to access stream object, response = CustomStreamWrapper( - model_response, model, custom_llm_provider="baseten" + model_response, model, custom_llm_provider="baseten", logging_obj=logging ) return response response = model_response @@ -775,8 +772,6 @@ def completion( ) return response except Exception as e: - ## LOGGING - logging.post_call(input=messages, api_key=api_key, original_response=e) ## Map to OpenAI Exception raise exception_type( model=model, custom_llm_provider=custom_llm_provider, original_exception=e @@ -816,21 +811,12 @@ def batch_completion(*args, **kwargs): 60 ) ## set timeouts, in case calls hang (e.g. Azure) - default is 60s, override with `force_timeout` def embedding( - model, input=[], azure=False, force_timeout=60, litellm_call_id=None, logger_fn=None + model, input=[], azure=False, force_timeout=60, litellm_call_id=None, litellm_logging_obj=None, logger_fn=None ): try: response = None - logging = Logging( - model=model, - messages=input, - optional_params={}, - litellm_params={ - "azure": azure, - "force_timeout": force_timeout, - "logger_fn": logger_fn, - "litellm_call_id": litellm_call_id, - }, - ) + logging = litellm_logging_obj + logging.update_environment_variables(optional_params={}, litellm_params={"force_timeout": force_timeout, "azure": azure, "litellm_call_id": litellm_call_id, "logger_fn": logger_fn}) if azure == True: # azure configs openai.api_type = "azure" @@ -849,7 +835,6 @@ def embedding( ) ## EMBEDDING CALL response = openai.Embedding.create(input=input, engine=model) - print_verbose(f"response_value: {str(response)[:100]}") elif model in litellm.open_ai_embedding_models: openai.api_type = "openai" openai.api_base = "https://api.openai.com/v1" @@ -867,15 +852,13 @@ def embedding( ) ## EMBEDDING CALL response = openai.Embedding.create(input=input, model=model) - print_verbose(f"response_value: {str(response)[:100]}") else: args = locals() raise ValueError(f"No valid embedding model args passed in - {args}") - + ## LOGGING + logging.post_call(input=input, api_key=openai.api_key, original_response=response) return response except Exception as e: - ## LOGGING - logging.post_call(input=input, api_key=openai.api_key, original_response=e) ## Map to OpenAI Exception raise exception_type( model=model, diff --git a/litellm/tests/test_litedebugger_integration.py b/litellm/tests/test_litedebugger_integration.py index e40e694fe6e..3eca24361c7 100644 --- a/litellm/tests/test_litedebugger_integration.py +++ b/litellm/tests/test_litedebugger_integration.py @@ -1,24 +1,30 @@ -# #### What this tests #### -# # This tests if logging to the litedebugger integration actually works -# # pytest mistakes intentional bad calls as failed tests -> [TODO] fix this -# import sys, os -# import traceback -# import pytest +#### What this tests #### +# This tests if logging to the litedebugger integration actually works +# pytest mistakes intentional bad calls as failed tests -> [TODO] fix this +import sys, os +import traceback +import pytest -# sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path -# import litellm -# from litellm import embedding, completion +sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path +import litellm +from litellm import embedding, completion -# litellm.set_verbose = True +litellm.set_verbose = True -# litellm.email = "krrish@berri.ai" +litellm.use_client = True -# user_message = "Hello, how are you?" -# messages = [{ "content": user_message,"role": "user"}] +user_message = "Hello, how are you?" +messages = [{ "content": user_message,"role": "user"}] -# #openai call -# response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) +# Test 1: On completion call +response = completion(model="claude-instant-1", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) # print(f"response: {response}") -# #bad request call -# # response = completion(model="chatgpt-test", messages=[{"role": "user", "content": "Hi 👋 - i'm a bad request"}]) + +# # Test 2: On embedding call +# response = embedding(model="text-embedding-ada-002", input=["sample text"]) +# print(f"response: {response}") + +# # Test 3: On streaming completion call +response = completion(model="replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}], stream=True) +print(f"response: {response}") \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index bc978912515..fedf960c0ab 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -141,27 +141,41 @@ def install_and_import(package: str): ####### LOGGING ################### +from enum import Enum + +class CallTypes(Enum): + embedding = 'embedding' + completion = 'completion' + # Logging function -> log the exact model details + what's being sent | Non-Blocking class Logging: global supabaseClient, liteDebuggerClient - def __init__(self, model, messages, optional_params, litellm_params): + def __init__(self, model, messages, stream, call_type, litellm_call_id): + if call_type not in [item.value for item in CallTypes]: + allowed_values = ", ".join([item.value for item in CallTypes]) + raise ValueError(f"Invalid call_type {call_type}. Allowed values: {allowed_values}") self.model = model self.messages = messages + self.stream = stream + self.call_type = call_type + self.litellm_call_id = litellm_call_id + + def update_environment_variables(self, optional_params, litellm_params): self.optional_params = optional_params self.litellm_params = litellm_params self.logger_fn = litellm_params["logger_fn"] print_verbose(f"self.optional_params: {self.optional_params}") self.model_call_details = { - "model": model, - "messages": messages, + "model": self.model, + "messages": self.messages, "optional_params": self.optional_params, "litellm_params": self.litellm_params, } def pre_call(self, input, api_key, model=None, additional_args={}): try: - print_verbose(f"logging pre call for model: {self.model}") + print_verbose(f"logging pre call for model: {self.model} with call type: {self.call_type}") self.model_call_details["input"] = input self.model_call_details["api_key"] = api_key self.model_call_details["additional_args"] = additional_args @@ -215,6 +229,7 @@ class Logging: litellm_params=self.model_call_details["litellm_params"], optional_params=self.model_call_details["optional_params"], print_verbose=print_verbose, + call_type=self.call_type, ) except Exception as e: print_verbose( @@ -235,7 +250,7 @@ class Logging: if capture_exception: # log this error to sentry for debugging capture_exception(e) - def post_call(self, input, api_key, original_response, additional_args={}): + def post_call(self, original_response, input=None, api_key=None, additional_args={}): # Do something here try: self.model_call_details["input"] = input @@ -262,13 +277,13 @@ class Logging: try: if callback == "lite_debugger": print_verbose("reaches litedebugger for post-call logging!") - model = self.model_call_details["model"] - messages = self.model_call_details["input"] print_verbose(f"liteDebuggerClient: {liteDebuggerClient}") liteDebuggerClient.post_call_log_event( original_response=original_response, litellm_call_id=self.litellm_params["litellm_call_id"], print_verbose=print_verbose, + call_type = self.call_type, + stream = self.stream ) except: print_verbose( @@ -285,7 +300,72 @@ class Logging: ) pass - # Add more methods as needed + + def success_handler(self, result, start_time, end_time): + try: + for callback in litellm.success_callback: + try: + if callback == "lite_debugger": + print_verbose("reaches lite_debugger for logging!") + print_verbose(f"liteDebuggerClient: {liteDebuggerClient}") + print_verbose(f"liteDebuggerClient details function {self.call_type} and stream set to {self.stream}") + liteDebuggerClient.log_event( + end_user=litellm._thread_context.user, + response_obj=result, + start_time=start_time, + end_time=end_time, + litellm_call_id=self.litellm_call_id, + print_verbose=print_verbose, + call_type = self.call_type, + stream = self.stream + ) + except Exception as e: + print_verbose( + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while post-call logging with integrations {traceback.format_exc()}" + ) + print_verbose( + f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" + ) + if capture_exception: # log this error to sentry for debugging + capture_exception(e) + except: + print_verbose( + f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {traceback.format_exc()}" + ) + pass + + def failure_handler(self, exception, traceback_exception, start_time, end_time): + try: + for callback in litellm.failure_callback: + if callback == "lite_debugger": + print_verbose("reaches lite_debugger for logging!") + print_verbose(f"liteDebuggerClient: {liteDebuggerClient}") + result = { + "model": self.model, + "created": time.time(), + "error": traceback_exception, + "usage": { + "prompt_tokens": prompt_token_calculator( + self.model, messages=self.messages + ), + "completion_tokens": 0, + }, + } + liteDebuggerClient.log_event( + model=self.model, + messages=self.messages, + end_user=litellm._thread_context.user, + response_obj=result, + start_time=start_time, + end_time=end_time, + litellm_call_id=self.litellm_call_id, + print_verbose=print_verbose, + call_type = self.call_type, + stream = self.stream + ) + pass + except: + pass def exception_logging( @@ -327,7 +407,7 @@ def client(original_function): *args, **kwargs ): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc. try: - global callback_list, add_breadcrumb, user_logger_fn + global callback_list, add_breadcrumb, user_logger_fn, Logging if ( litellm.email is not None or os.getenv("LITELLM_EMAIL", None) is not None @@ -369,12 +449,22 @@ def client(original_function): ) if "logger_fn" in kwargs: user_logger_fn = kwargs["logger_fn"] - # LOG SUCCESS + # CRASH REPORTING TELEMETRY crash_reporting(*args, **kwargs) + # INIT LOGGER - for user-specified integrations + model = args[0] if len(args) > 1 else kwargs["model"] + call_type = original_function.__name__ + if call_type == CallTypes.completion.value: + messages = args[1] if len(args) > 2 else kwargs["messages"] + elif call_type == CallTypes.embedding.value: + messages = args[1] if len(args) > 2 else kwargs["input"] + stream = True if "stream" in kwargs and kwargs["stream"] == True else False + logging_obj = Logging(model=model, messages=messages, stream=stream, litellm_call_id=kwargs["litellm_call_id"], call_type=call_type) + return logging_obj except: # DO NOT BLOCK running the function because of this print_verbose(f"[Non-Blocking] {traceback.format_exc()}") pass - + def crash_reporting(*args, **kwargs): if litellm.telemetry: try: @@ -397,10 +487,11 @@ def client(original_function): def wrapper(*args, **kwargs): start_time = None result = None + litellm_call_id = str(uuid.uuid4()) + kwargs["litellm_call_id"] = litellm_call_id + logging_obj = function_setup(*args, **kwargs) + kwargs["litellm_logging_obj"] = logging_obj try: - function_setup(*args, **kwargs) - litellm_call_id = str(uuid.uuid4()) - kwargs["litellm_call_id"] = litellm_call_id start_time = datetime.datetime.now() # [OPTIONAL] CHECK CACHE # remove this after deprecating litellm.caching @@ -415,10 +506,13 @@ def client(original_function): # MODEL CALL result = original_function(*args, **kwargs) + end_time = datetime.datetime.now() + # LOG SUCCESS + logging_obj.success_handler(result, start_time, end_time) + if "stream" in kwargs and kwargs["stream"] == True: # TODO: Add to cache for streaming return result - end_time = datetime.datetime.now() # [OPTIONAL] ADD TO CACHE if litellm.caching or litellm.caching_with_models or litellm.cache != None: # user init a cache object litellm.cache.add_cache(result, *args, **kwargs) @@ -433,6 +527,7 @@ def client(original_function): traceback_exception = traceback.format_exc() crash_reporting(*args, **kwargs, exception=traceback_exception) end_time = datetime.datetime.now() + logging_obj.failure_handler(e, traceback_exception, start_time, end_time) my_thread = threading.Thread( target=handle_failure, args=(e, traceback_exception, start_time, end_time, args, kwargs), @@ -917,44 +1012,6 @@ def handle_failure(exception, traceback_exception, start_time, end_time, args, k litellm_call_id=kwargs["litellm_call_id"], print_verbose=print_verbose, ) - elif callback == "lite_debugger": - print_verbose("reaches lite_debugger for logging!") - print_verbose(f"liteDebuggerClient: {liteDebuggerClient}") - model = args[0] if len(args) > 0 else kwargs["model"] - messages = ( - args[1] - if len(args) > 1 - else kwargs.get( - "messages", - [ - { - "role": "user", - "content": " ".join(kwargs.get("input", "")), - } - ], - ) - ) - result = { - "model": model, - "created": time.time(), - "error": traceback_exception, - "usage": { - "prompt_tokens": prompt_token_calculator( - model, messages=messages - ), - "completion_tokens": 0, - }, - } - liteDebuggerClient.log_event( - model=model, - messages=messages, - end_user=litellm._thread_context.user, - response_obj=result, - start_time=start_time, - end_time=end_time, - litellm_call_id=kwargs["litellm_call_id"], - print_verbose=print_verbose, - ) except: print_verbose( f"Error Occurred while logging failure: {traceback.format_exc()}" @@ -1085,32 +1142,6 @@ def handle_success(args, kwargs, result, start_time, end_time): litellm_call_id=kwargs["litellm_call_id"], print_verbose=print_verbose, ) - elif callback == "lite_debugger": - print_verbose("reaches lite_debugger for logging!") - print_verbose(f"liteDebuggerClient: {liteDebuggerClient}") - messages = ( - args[1] - if len(args) > 1 - else kwargs.get( - "messages", - [ - { - "role": "user", - "content": " ".join(kwargs.get("input", "")), - } - ], - ) - ) - liteDebuggerClient.log_event( - model=model, - messages=messages, - end_user=litellm._thread_context.user, - response_obj=result, - start_time=start_time, - end_time=end_time, - litellm_call_id=kwargs["litellm_call_id"], - print_verbose=print_verbose, - ) except Exception as e: # LOGGING exception_logging(logger_fn=user_logger_fn, exception=e) @@ -1486,9 +1517,10 @@ def get_secret(secret_name): # wraps the completion stream to return the correct format for the model # replicate/anthropic/cohere class CustomStreamWrapper: - def __init__(self, completion_stream, model, custom_llm_provider=None): + def __init__(self, completion_stream, model, custom_llm_provider=None, logging_obj=None): self.model = model self.custom_llm_provider = custom_llm_provider + self.logging_obj = logging_obj if model in litellm.cohere_models: # cohere does not return an iterator, so we need to wrap it in one self.completion_stream = iter(completion_stream) @@ -1497,6 +1529,10 @@ class CustomStreamWrapper: def __iter__(self): return self + + def logging(self, text): + if self.logging_obj: + self.logging_obj.post_call(text) def handle_anthropic_chunk(self, chunk): str_line = chunk.decode("utf-8") # Convert bytes to string @@ -1586,6 +1622,8 @@ class CustomStreamWrapper: elif self.model in litellm.open_ai_text_completion_models: chunk = next(self.completion_stream) completion_obj["content"] = self.handle_openai_text_completion_chunk(chunk) + # LOGGING + self.logging_obj(completion_obj["content"]) # return this for all models return {"choices": [{"delta": completion_obj}]} diff --git a/pyproject.toml b/pyproject.toml index 9f625122f55..d332b4e36ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.495" +version = "0.1.496" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"