From df8b3b498e33b369dea893490c901fd3611e1f5c Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Fri, 13 Nov 2026 08:32:13 +0800 Subject: [PATCH] feat(proxy): add customer hierarchy usage analytics (#51495) --- .../before-main-6685525.png | Bin 0 -> 8473 bytes .../customer-usage-detail/final-overview.png | Bin 0 -> 67408 bytes .../customer-usage-detail/final-report.png | Bin 0 -> 62798 bytes litellm/accounting/application.py | 42 ++ litellm/accounting/projection.py | 44 ++ litellm/accounting/snapshot.py | 1 + litellm/integrations/otel/model/spans.py | 1 + .../migration.sql | 47 +++ .../litellm_proxy_extras/schema.prisma | 39 ++ litellm/proxy/_types.py | 4 + litellm/proxy/auth/login_utils.py | 1 + litellm/proxy/common_utils/customer_usage.py | 187 +++++++++ litellm/proxy/litellm_pre_call_utils.py | 40 +- .../internal_user_endpoints.py | 17 +- .../key_management_endpoints.py | 59 ++- .../management_v1/__init__.py | 4 + .../management_v1/customer_usage.py | 220 ++++++++++ litellm/proxy/proxy_server.py | 1 + litellm/proxy/schema.prisma | 39 ++ .../accounting_delivery_repository.py | 12 + .../repositories/daily_activity_repository.py | 5 + litellm/repositories/daily_activity_sql.py | 1 + litellm/schema.prisma | 39 ++ litellm/types/repositories/daily_activity.py | 2 + .../_components/CustomerMemberSpend.tsx | 166 ++++++++ .../_components/CustomerUsageHeader.tsx | 112 ++++++ .../_components/CustomerUsageRankings.tsx | 200 +++++++++ .../_components/CustomerUsageTrend.tsx | 152 +++++++ .../_components/customerUsageApi.test.ts | 336 ++++++++++++++++ .../customers/_components/customerUsageApi.ts | 265 ++++++++++++ .../customerUsageChartData.test.ts | 185 +++++++++ .../_components/customerUsageChartData.ts | 71 ++++ .../_components/customerUsageData.test.ts | 104 +++++ .../_components/customerUsageData.ts | 262 ++++++++++++ .../_components/useCustomerHierarchyUsage.ts | 122 ++++++ .../customers/usage/page.integration.test.tsx | 379 ++++++++++++++++++ .../app/(dashboard)/customers/usage/page.tsx | 136 +++++++ .../components/overview/BreakdownChart.tsx | 2 +- .../src/i18n/locales/en.json | 66 +++ .../src/lib/http/schema.d.ts | 174 ++++++++ tests/unit/accounting/test_application.py | 103 +++++ tests/unit/accounting/test_projection.py | 100 +++++ tests/unit/accounting/test_worker.py | 129 +++++- .../gcs_pubsub/spend_logs_payload.json | 2 +- .../proxy/common_utils/test_customer_usage.py | 225 +++++++++++ .../hooks/test_dynamic_rate_limiter_v3.py | 20 +- .../management_v1/test_customer_usage.py | 278 +++++++++++++ .../test_internal_user_endpoints.py | 25 ++ .../test_key_management_endpoints.py | 88 +++- .../test_spend_management_endpoints.py | 2 +- 50 files changed, 4484 insertions(+), 25 deletions(-) create mode 100644 .github/pr-assets/customer-usage-detail/before-main-6685525.png create mode 100644 .github/pr-assets/customer-usage-detail/final-overview.png create mode 100644 .github/pr-assets/customer-usage-detail/final-report.png create mode 100644 litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql create mode 100644 litellm/proxy/common_utils/customer_usage.py create mode 100644 litellm/proxy/management_endpoints/management_v1/customer_usage.py create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerMemberSpend.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageHeader.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageRankings.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageTrend.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.test.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.test.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.test.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/useCustomerHierarchyUsage.ts create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.integration.test.tsx create mode 100644 litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.tsx create mode 100644 tests/unit/accounting/test_application.py create mode 100644 tests/unit/proxy/common_utils/test_customer_usage.py create mode 100644 tests/unit/proxy/management_endpoints/management_v1/test_customer_usage.py diff --git a/.github/pr-assets/customer-usage-detail/before-main-6685525.png b/.github/pr-assets/customer-usage-detail/before-main-6685525.png new file mode 100644 index 0000000000000000000000000000000000000000..563c231cfbe90d21fac0a1aba2af84e2fa247963 GIT binary patch literal 8473 zcmeI!X;hO}8UWxx3RtSv3>YeF2S!#wLWm*>Fm^y&B}5D%D4Ql_3lIb($d=UUSXqSB zqC$v33rCg&LL*C>M8wv_AO;vfwh*9AG^`-#nS#OQR)=lOE$l48;E#`dY-jJ_Fp^?Ez2n+)i z_@9P)jQ2hfKdG>To7kBrmLwF7%-8b#g7Dq2lNAcL0KVrG7pFW%kazGJ%ImVUpSpjs z_nlv*Xsd@(3RiP&xY)Oswli(koVEj1eXq!`qr2Vz*JZw+-1u$WQl;p6&D^ohK^1ppZU=BqvtLq2FXtzwMAf^v%ut+PGpFkPnvlNaQYdO8 z!cUtrxdD$I{Q;Wk8x;g*-j2PWbc7DJSBAzjoK=kKuW-&Yty2yYb1#}!sKB~A%90uk z2ew6le`tqfs&dBf$=t%*J);ynzx^ke1*}}P!tTk5=@oVHm+jiTN4&db1#w5=XYz$Z zbDfq$#6_VmML`TN!S`Ekg%ZC>3Ebx3h{(HLcngo$VsEv3+8i6nV=oleR9mrCCXrAR zl>lQRK8tz~rgfVevpPFBGTh6}J{q|}x_hG3G@`rCw&cXJWh#bYn|s=7;Aq#JDsFz# zqm#>d{O%|2gG)artd4O~Y;hGTEDVi(YV$4tvRWZ8Ir%)>^?T=Oi$O@bA}OAi8x+(? zZd4YL8rSOiuVJa8Im`a%Qtt`gn>Fgna?ZKI)LGGB(?ZDQUj~pqU0)LIE8?hSh;&hp z7xkOQp!=BCP(9SeXa}Llt_Vw7Rv$FPaJOdt^Utdmbj=m6?eszhC z-ty}6sZ0!KCO-QPLwZ;z&0hUHOGcxjHnX<;UfF#8g<-BupM3kZ-}B0KDOJi`cc;Lq z8+Ud9_L(zjerYYQJ~Dyh6xUPS7MF4(Q4Z$uvPo}p03RmI711hYS@vEtq61&$Sijjn zk#jj|q4V;#XfUZqf)o`UtvDwP~miU^5NLoL&q=6t-wLaDjWiIFR? z-s9(Lj}9_}UN){QuzD+)EE*q#O%B=IRjDrPIx>2Ey9^ti;Ufn_A20UY=hlqIShr>W zFzSjHXQs*+vN)M$W_o<}d++NHl#^zEc-kxKe~P48cjZh^Qqv&V+r98pFN+c_^HjxV+E3dwYVutno z*p-fa#=cRU=%M+J;!&7t+dJjsEK{}Z-c8bJn)ce=7aufipe>Hwf_q7doA&Oroub#x zXI^2yeb|gOt5d!xjeVs#P00({0o>XBXIa0cvAHX(8iGVa!<)Hnmz6!ZbdfQ+*p!Qq zsBmHdHj)2^u>4?wE3;krh@XsOoYQhs#%! zM@bQ9Ao2WaLN8-p#$Z%Awu6wS>X2m=4AoVQ_%Zt;jQm(kId$ca=%xdP`f zU4pMNfp?LTwlafzCqv|k(UTW$6K2L;ic)XlZL6jhZbx8Xqo}8dU{6sOO+zGFt+%r< z{ETKZo3M=?K)M`Le7pEAva1yr&@;E(qup5wc?8O3Xt(D}K%e$_sTML^e$`q}Ky4>5sL;$<26zDw^v) z%P6Qf`3+-^^$IJLEDm5Orq|@WMokitcz0=E=gr3G9{z(@kwe7bQ$#oW7_$_)FeAVd zg8?aQ8!K?WWPJHo95lz5&gBRQ5}#Xq?>sOhX&}XgGo0T}D0HZ@$5BMnGb_}Xn|_Vv zc{ybb-Yk3HIPEHlb!muHNPfT8esWqUYXJ#OY(Dg*X!ayyA5?6Y-5_lvx15Q%VFGly zSuqf~xQ7b?VYK*7TYH>+?sEUV`jC!{&QdOUk=-DlnV^GN!qBAv%n(dc!!4}wc{Gg3 zOOTNhgMOK|IL|mvVty-MfU=h8n2X))GUN{;K~Y%(Kc|C7s9>dIK!3Y8t@Gr$(YDI| zmayTK@;(k(bGRsrlons(#7P@wxRR|xhJ9X&gZrT%-@NPbQ(K371U>wml*k`hLho4L z?}7}n+JzHV$u&n zh@hZ*Ygl&=($w!p%bI_gAJc{;v=$T<70pUn^mL+I_NqWyN2}j#D2$r{L99=rXQu{+ zj-RS+ni^4!-Ewt7am&2cJmlsn7;7iRV~39X>bDF#5o8s|9#rP|8QvG8`0?CQf+EVA z9-8cz_?(T(6cMEBtSTo4UP8!M*anB2U6I9`5bP<2mVR3OQj1Ak`XHrWp1eX2jnQENl+;yvW5oXY$Hrc;0TORKBeG zxK@^w9CtMA&nquH$2LOQH#D1J^!xr2*{e z+Ui0Zy7V@&YL59(x=zcli`zz|3z{v$DayTp&5cb9;P$Z*Pgjpl=;GK&G6uJpn(f4$ z#-xT^<14CsWQZ+X8}ZwOYw)Ky0w+~nZ*y`G545j{uYJs0?^5J1$r3g%<|!Nm@3dc$ z@2yn3TuIC!D%vyk`zjkp=g~JplfkV=LGJTvJ1=@w76hzGAzjykl45d^0Z4yNE>t zr)xoz*3}P0LS=tuW`BvE`G2)V{CX&(5`9)A77f9UNa2S%wvN!Rt<7_gTCZ{?%NFNc z;#@AvTB~Lt4Ew4p%jy&o@zBE*SY9g_8@4*67gY1%j^E_yV>>XTls zqA2UoL99P&v?H#B5`7h*|4ByW`K$4R4(EF-o!zq8Hx`Cx11PK6xf-?h4HW=5^yXi> zl^-{k{Fcz0wCT$3t)8y0Fa1{u{%vq;ffxmmeMX}xa$h|8x4U!;Isu)4PCzH16VM6h O1at!bCjpzC@xK7`fiLC& literal 0 HcmV?d00001 diff --git a/.github/pr-assets/customer-usage-detail/final-overview.png b/.github/pr-assets/customer-usage-detail/final-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..fb9438653a535bb794cea1155baf75a20f95911c GIT binary patch literal 67408 zcmeFZ2UJt-wkR5;suWQ=2_5OE^e#vV9qC=cfRxZfkEnnmQUU}Fz4y>N0cp~E@4eU1 zi-PFo|Nni?*!%4B-Z}T3cgG#?-LtaBH?r3J)|~U3YkqB(^?T~~65yVyl8O=l2L}Ma zx%mNp&jRED1bFxa_;>^a_ymN61Vnf4-MMr7_8kgRGU9vG6f`u{6jW4nAa+JNdR8D6 z6%#KL>q8DMZZ29zzQ=r=kJ&l7IR8L`Lr6$?hv?3OJ9i#%(oxZI{!bsjn*roRw}$Wm zxHv3;TjV&n+!;{a~q z;^C7M+<(AKNby8Y8_A-hYZ)~xAmr-iUq;C)_$-}>iaNTyg-uvQPydyBz?YFRc7u!x z8Zmj8$G7E&9HJ1b$|FV3zz;Du=;?57jvIddmGq6!{y&a&<+$RVp6!Ly^^GXUZnl()!n$pKFR zm)Yd^$jNd3gTniQfH;OBBNVYn$N_33mE-2D-8p8h%o6OpZI9^`Hinti!584P#)=1S5e%9#Vwfh;N!T)ujL8lYL zDJ2WbEf%K#>w=fWscbYxKL}ZTCPyi+Ir(IiOyN5tS-VQllLOT0X~kH=w}*ax$K1>) z@fEAmhH|ANOo?Te%Mrdo0qEP`5c5C%nO^laYvgb$b7Xw1@=<~NhTTJVrpN(?A+fJz z+eu5_p8U-iu;yiXfx1Z7j49c+zkcU}sPY2C0)xya#Omx){jac2FUk*&Vf9xJ(Qu9u zNQB{Z%8Uy-1?BrZfJ4?Yl+0= zP|QN?ipV@VbJiekn_*a3(J$~+cqP6wl-|f_k1^xC*Pj{G=Ct;9sIoDpPH^m9+>$z86ZxACwGV^m$)+d3bPuROe~^ z%xkjEA;y{535k$9zCOt>-9iv@k6lTit)E%(U~4tmoxcGwUf&~>e*=zYoc9_$u3Ot4 zMNf=&GBB!%A*Scw-TnlljqkB67>1g@nG9XWx6(EJm{%#0QQ_dKNyHY{E!Eulw31Uo zaN-E+C`rtL*>Pw}rN!dA=7r1t&rl!Kj) z%kSR@KLthrkK+-^#hco0={fm`l$`1Wy3aBnbW|rWixwR0_7b~B26k9TCLcxR>9HI< z4@JAku3|s}zzqd28XT5XS#MdPH0cvOd0Vw&( zEioiMr8|P=!$x1H*LBTkf#Ozy(Znv1^I%DC?uXfZ9c^$6>1o2yV@;LKerL%~!DjD% zh~MK9PPq|2!{876^{wsISh9koP6x-5FpOLL{JrTq>j~jYR#kae<;du`Ubi9WF~VhK zPE73L9&GF5yJ62Fx=F?73uav_dC~431)%yv(EVtxZIJivj!0cXl^ zncay*uSS(&kJhYEJmx+`BeP3o(68Mbm+;xQ`};4ivKNPzTvoiLAVj|b-GWa!<33+4 zwsfMhR+AQC&O?6uhW3py+=WlHhpT&tuU}-R&#BPUHhaRt7#7;6flCueA3&q$)`dnm z*C<1Dm3X~9>!^}y#P#mlm%;B3B1-X3g?{m0^N{7_g}68^?$RPEtBHuccCTg>Tv@Vn?*k+C_xzO@JG zbi$5jP4LRdMC76J%<}B?{8Z%@ew_yW+gzNVG#kmof}^x^T4W1uZ(s+?{tf?#1Cx`x zOIcha5hE4&Y#^UAELOy`Dbcuh*i}j@RlnG@VFYt_?}aTVtyoDw3)x@mgz(VM&jhqX zQ&znTf)GIj`$#+Usv=P8eIJ|!{o@UD35S4C)wyEwIyF$;4}6VI5`HKPiT8aeuc{J0 zb|yy&Ewk-@Q=!8t{-|*+T=}(+!;M^Yqf^)joLRr`-=?X##_AH)c2s){gfP^OidvQh z?p{354`DfwIM0r4x`V7ui>gcxrW7^vvR!V|_T7vgsQzz2p?WW$Zt*Qz^=#xFqeE+3 zsRiI<5`zz6y9wAM#b_x=B^M3}5s7Ws7bCHyWlLGg%RuKAX9g69*zR!ISRlDxxxikG z2mp!JG;gxpw=Mu0LiSpz@rk6o zieLU)3zIHQgK}ADN9TlM+hXU>5w#*>zX958vnMpR{47$HRuJHqj72q;A3g2TtvXui zd>%FMuhsUURSOAsa`aUws>ZYxR_~^qN17+s&d-3`buEI*=T&m~u?-8kMWIIYG^>-! z=MrJ6-@iLqN@_g}L<*+-YUivO-fDbVVSX^~n{a0)aUMjK@ymtI$8N_N_L=yZ&O#xl zf_3`GxYmlAy6Z_|(Y#I7(o2CWx9_r4*~7y@gXZ8% znG7!9Y>Ke>Js0Nn!PgTJpn4Uty{;ouRdr%%&Fo5DgF2M&gvjgKC!5evn$;=bPe;4W zDb+rH4_I+dRi_tf-X$yV!?@Jg^lk?Jh_d-M!P|>0pRiB(j(PUz^{+3~&$PAG)M>{- z!Y!v8RH$3PshJVB-f}YM*Wx@?e4|lrSZlYa^0{n>4Z0o9jMlYU38qQZr?)v&5o+Qa z<_CwyAt*f+w{hV^6*XJ7|NznC~NA*s|A5m#mPuD+CB<*1&Xq}7whT^A5B zTUw4KbdZw91NN|e$P93Fh3wVlZ@{^{gK*BgcRm`OFe-&?vj;tn9@@AFOh$Bf?w>~? zR4;qgF1b%Etfdvzrp@J9a~&$gmj^7uJ^?gwPJ`F|KV-~i@X2d8rZ{Smuy2&Bs##RfZ$LX| z^$&eRO2H+E_N76N+lfZz+ZHL_z)!4w$GU}7c_`(b!ds#k|BHuJvznr$6iE@mYg84A zUY1pMJA&qk@9SJPLPO&LWYNs6;+@0T3ogYvI(|D1CF}Hxaf$M#a8q{_>J@hi&Zjqx z?`hYhar$7`jjx4YDu?93+AnprnWhce9lu@qpsa5B`9Ig z%EPdgTrNABjA#~vt=Fv!4S0RW2gDU4+#V0d?Ph!Ly4cHOmRx3< zoG($NPzX6%~Gw141#)g=ENWyTffA0vE0|nUZjrA>~ zW~(06)@0-%9wuy?9})HLCH#DNrjd>8j#?oM*ogTiLShIq<<=Ok)K{o8 zX7!w)s*$Yr@4KX7vv}`1W$+G6Ju+tr+7`gw;z*Wr= zLBm~JKmO=aQJ>nkCrd$r+(2VIpCd`>hq^lKyK5L`inf`7-M)I>8RYnFQTfV4ffT*A z%G^oP)7auN_5t~tS+q^~8MJqiQh$)hISYA9Y zyCXr@y*|xL`Y5J8I=-RS%8|{oioerkZ?oI!NyuyKnW^)x=I?XB5dk7`eBY{TeDzD| zM=E$RNFDKyzX1i2$hYy582_;IRa{!rTWyuRl)nK6#)XuRN9TF)bbD}!iDAKZVskFy zq~pmiWnR9nne#YLi+z4JP;?t5c3JOcvH>a_eBb;%vMS6(eY9wJ7b|M)8WNJS0^+SD zekdwj>?%gm4UCvwss4C87d1-|7vN1|^i?G7v*uGh+Q2YIhQZbSmug06tHQEbbS@3b z*LKZYhToRs#83)~VR3vD{9)1!jnN!Y}v+*H0>sq}|4e`KGO$f|viOxKE!0AaX zPpIPA*@fNK>l(db2{BVi-xjCM%G#Q;tkpQ_UBcXhDBXocr)=Wf4~0}W00P89o=*h@oQL@DX2V$Tf*4h-n^V9;l+hT63&c^c)`07zFkK7pM(V- zDVSQ9qm;huq?09dorLGJZp-|foDfmx6@{k#JHnVS$ROIji5syOxEJIXyMq|}pZBN~RZ)77WLvt2KO)Cuw=aA?RUu&orMrbc zpv)2|~PT{v4Z9VSu{;Y@e3+~$z(ZFK)^o8Gwnx#)r}Xw)?eVi6OD zjY|^5ZDL!x`fL^;N$|B|2yFQ}wZ?F$ohJ*DilE~Ad*E_X8 z8cT+LDn+0*!>nmGy+kG95^dinKX_|FnSUl7e9b{=W8WU^rWzyJF91@EeGJzGcDN0n{SanYw5nBe*$3EYl zH6OuWAn2WF-k*lDxIZF$QQRlrUGrosTIV#`b=$r?RCjc|dQ7arUfk|nUgN#@8&M0J zlp*ot=IBA%m-5_9(V@g{1ekOz>C91pfF#k8e!id;QE*0R;%To|@kY?tuLxs21SFf% zwQK4&6Ltzrk0JmiF&xd#kC4M$F6Y9P$7UbActCv{C7F>bqDZYZR9nYwM6>b)RlV@E zLC(GM6#Fjg8sr)DKv8hEZNb(vOzhrj4gZ>~6y|uXF+p(~ob!;w-FAENSJQfX_iupv z&zXt3Vb5e|yWE;Zh5g(BOG$452gyJ+_?yQom99TiPQ{*9VBY-(7~br6CbiuwYxh~f zpah>ja7>Ma_qyogW~0jN>Q27~^DTvWr~Px8T=?-^jo6H2W0c8tMTR+_{kTKZtOYu! zz$(X*0+vGLN+=WYNwBxni|?9>HDsb=mkS$`Dz`Z~Q@iuTZHHA>z@T>10;K_{&YQZ1>?c4ym~-8qd_ z$<2!Tn&orN7t+Au$sx_~YO`&gZDW^RiV6Ym3%xZbXkd_%)U%AHG;?Zmb8SOOektQM z_mXx^f=ytO+eLA4=)1Ps`uV=Zi(DR3gpI~f4l<`)ZzjX~onBsENcNAs)0z$-eIQ~` zqxaRgFF`BX*|VT%Aw`87J3ty-b!D;y1mboc5F-YCOm+CBY8O~}YX)F`jnZ&-PYAhQ zbAL8p`L43CeA{BSW%iCE^RG{2Y}bs8$|uH?LgLxEaXe$@Zz_5v=0(lJ6%<$rl-&3h zB!gMG+Puo%wKoZ~@SX#WkM`Z*RH@XvPywn5#6vj?t>a|IB2djG+; zl(U_`Y*3usOo<6dPQ|n0hn8C9rL$+s(#{ zc4XEzGb?SZ4t>Y=q+G=8F@XvPYVp1ycuBOqyV*JPOJxpSi)~iv^tK~vLp-LkZ>o8d zTI{5oEoDT7E!DTb<;I%iO?<@t{PE*|$EK&PxH>agS%oEq1`)9`ecWNHNl(kHX@jI? z=Vpf`x0iO%+&*f)6UVPD@BrTiLq3FdnT*7o?-Ft%N{FLH)>;2pQu zv1hH^W8%Gv5|MiViNTBrJq8@_kAQuocdX=`<>gi6ZU+UR?$~~wFQ;jxPX*$;1iuds zCi;I|X1i$pxU8BoA`Ep_i#~^@)?*m0?OcL7!TE$9sCKglaC8kgz~XK@ zPr<(o5`WJn{+?EufK0Fxi`H#ri<`yt;jOmH+%f3rzzjg?fO}3%u&4HkL2q<}-aI$y zRMB9h{-kwRtf_LQS@gP*TayAFhYg==2}<%KUy_o%^|y+qdb)Bt%{_6D__Vd#bU>V= zje_M0_{*~T1d%|5h4(vr(Z2|{PtMYszm(Gz8VS<=+S6qkUi|j1d<5??-o-^hae-F5 zz#+@Z{o8L_kYLN6y5&@go0DpVD9~j10RUPf%U(`#J@xFP=Sg zow7T1$`2n)x|=BxI><1aXWaDhyW&NaBXjw?bA5WvK)Q`38<)z|rrxK4f`%%Dw3odShCg^NO0dU#Y@OHkQ+0O5NagW&f?`xJVUbShj>ONSxl#~YF- z41^2fA61yU`_!K|z8SmTY`ouM-oeH1ri-~V5M zOewZu_i@K$Ye@n6A|<264t z?Jzs>AitDtmAeSyBHn1{uVH}^6=j;+-w$l9ikw{OH;?D{J;lDy?9Awc2A-2f`9RiAy>TMPGjJ_kLSJ z{00BVel1weGYvIw$oUX7cpof6pbYKw1|G`(flZ;K-LiC8c)-sUokek#`02(}_hLm- ziH0$?%CS3a&^_Thgq(Jj((Cjn^zizX$wu15jF;vdt-Md2qME_hhi@Fh1MD9nL0t^S zc8mc7&&OL{Dry<3Fd-AXXL+3g;r%XMQrGRS4RCMA)SS<3Z}T%o1Jv+HR`qpne`s2M z;LAi!%y6cZ9)1~cmbdA6x$>P$s9oBFt&-ABx8p}}+~-!Ky|-Ip;=mu9IBr-EQAxM* zbj*mM-o)rg*GoQ%4B8o^4XmrJ3oR>Q^rt70wI3I-?}AOYxORC!GMK92J3g)RAtvu> zW@1Gd<T7kJZDhzR3 zff#Pmho!yG;N8VU{*fvjU2j}uc%|RCY{XmXi=^R5$fb=ITDesh$#r&))5!!LRC77N%Z^orvT3Q(^VS#xa{8-DP{1-LcKcJAksL&bHQ^3b^Qz`B-WQ8QLwwdAZqrnd9YJjG|v@-L!?_0$L~cS<1UimJ6rE46lI4doUuYSX2j4E%OtKk@Osh7 zVlyH2W$$5MN_RMm^`oFy;no}5l;8THH*UC{DxH=G96Rj?@5qL+e0wf?9Fbg=3s^56 zZ27F#OJQ>>xk_S(FB%a7GF0_ju}?5wGT!ThciD)T$ftZtlByhH&U=0#3yh8R!EkiN zi2ew+;7j&$zm(x)rxu(k#+^-ETJ9H7RYF5ho)4A!;}l+Z?uG-&+l5CgE=ud9J0uJ$ z!QWhb@CP|Kq>}<1MeEn`M?#inD!|XX71;)JQ=R;7(}(xV97Uq5%V>%EoY+lA@+xOGQ89FpfcW2p)on-0_;wkC__bV(i&SREDZ&w{e?f`Gy4s z`}*5NirFAb&iLG=qVe5i=C2YHY|`Gc-0E-;g4Wf^3*u}mRgvnp?1-7OaZ2@47QMUP zfG8C)(N*4l%<5=t&+^s2-TXo+E|xCAC4=AxOyB zotJIXrFoG5jLS>1oFA6XKR!Nxe|*Atxls5>(O~9@zK(rZgq;_0xgo$REQuhNcr{jL zyl0QvG^+H4P1YETiGNPP>sr~~kLK$ANs6ehrE`MOuE{k=Qrdk&T8*5|QPlygTV|!{ zdND?`$5*+m(QuQb8{i4=x*JNDkj$a?gSn(}qSY|##*p4V$;fsSJ`(22`e=^TlTA7h zfYXZ{+1(#i%c-g&;p@^j?ax8v7vGG|q*$g*#uWX)5{lFm%8m?rz~WKD*7`A`2=0gG3mR< zD+5lpdtz&n`0eMLzMf6>0vTtZx+BU3PV7*xsotbSBJT{X8qmqA(0d-!$p+k{(a5=Y zQ*X9~rFc|?(Gy7df`EWnPu{;+>Oj2UcVP_q`cfD=vt#d{4{0#p*BI+-YNeoBl?vd` z;%0fmKs_{N`pLP&Yc*AhFnd5O^2m#CoFaFMH)PuUGQVJlD=f93ODfYH8pGj1wRRr2YjwXtcj zdn~nq2;$j^ie!FHFL1sba2wb>$UzEWU8tSkhhNKN{<4)QFYQLZ1RI0Ywcx%Lbc=^W z-&~BI+++S)*1mU_Hs5K|^KO_{F_MSlq!^u2ZCAjai{NY{kIQr~3;4eJNczUb#Ad@) zM}<@IksAZSE5M*~@58%@aEidR&S5$&C7 ztT$8qr7F5To69Qg4YKRtFyA_U00D-b2K$F$2k!PHD)(H0K9W<|{#P8@|H)_NP}=po z#8<^XzeOn>6bkOwpWJmqD|RlAPV7fYXiT{5Y;>DySo;a{k>DinS@FG*+mD4R_a>>d zekG8MwJ-_F+q&$}bNaT;8%ysu_^}y@o;7CYjRx;Xm0rqYHwsC7J7KXJ3#YGRt6$Q> ztMvz6f=uFHu%n*b6)k5u)XMtinRL>Oi|jD-IM4S$Y->Aahp!~+jGJVmgMw5mRMcq2 zBENMKN^JrnUH47Hot5g`;FLMz8u%KG^m;)iTzU2{iH9)FpU9acKYMDrX}YC_W?-eY zY#bXuPjV}Dnp}O*zh}_Zy%N6)4UQIPLMQ;&CCSrz^Qsuq8{J*C9mlD@_0;urJ@lXy z`bMknD!o`3egO5K0Zbia94J5N;@PfK7$>P5+kGf9`ExvAcZ{UA?N{ZN_X_T#2cEbW zNqe(lUKd4!mx80%YSX;!-vDLZZaIU?tpb)p?|i`~DR7_ZycRB6sxb3#Btekc=*t3s>2VAE=x!Lt%aQcb^ zWdh4J^f|7dql@eAw|k(j+5M!Db*D_bPYYQI?kPA~ z0;iUf*1WQtH$0@-?Po}M^4XD5730rU6vvz>sS-kH&qGiLqec~7_-Z9e!|HpjR zy3tepJX61d95m*xwP`9J??am3fY-;kCT3d9{z)et`sAljE0M&HBxV;T`%P^3_4Eu3 z8tqbByx%}z^#DHL`^|;w`YJqXQ+?~kLa|Z(Z{OKT#Mn#h>{|BMR8cZMTN@!_=t_-I zRlBR}k^*hlgQ{`zZc1S@^Ote9@G{nQgWLd5IQ;m_zyspOVIPbI`1jF(0f~oPH`iV@QaK5voblyP#8ZEllBbI1?5alX?)|7d~qd?0=ec z42PqR`$-^9A^9W40V=#_KwUPc$o)PfEl|E5oV5b}#2!+zRo(eQ9u0MNiaHGlLCd*D z6&D6m-IgsXg_+O(K3gEdc;W$7{1bC3#2P#{)D}o9*O{oq)YpqYLGQ$775sEUd)Pe3 zs0iy8nCx7Ns9Ppak-qVvadHv&bOXk+tMMUoWv9@5GY=66Xp$E^D&Ns6N{8xV5O#{`K30gz9i6c*0ac_(8vhOU~|-kl7G`I=s8mZ!T_VfR1eFq! zq%=`k8#y#_e)8153{v%uwE(N@TvLymvZ*bTSR$Nt44FAEjXM7nc0JUXXD0tc(B08A zi|6^TtC1fHWV4b-VSslY7^%T`%qzu?KvxCUg8UG)?cCNpFND6-MU-?CQt6#{AQJ2p z7F?&2zw=5wB!kc~Djo~?PTmy}X()0j0rz8buZ#*_4zny`Y>QsdvXkEVe)r?eX4bjk zH=u_iM65As5q=ZF%Sp*-7`va^4KYyt+Ya}sN$&ZP&$j_hN6z><^)hNTe0D#h0Ene?>%qbry|`e@2|@e<4-=1g-2p)B7+K9x8#_tJSlhn5lMf=u)yI74LAS4t)2#WOaOR>gVpx=_PIRG%kj-QRP>J1c!Zl2ibv^VOOBME>>H7cR;6 zsk7~yw_i_h#o~$|s1tAEO2_{@luN0>haC{Lv7Zf0kt}?ST0A5_{-Im{ zUFp9?@c&&cNYd`!YKUlIxRn!V-EvwnUYpuzs{fr;UU}&Miq>69zq0WTZ%ta;^Q7AZWfG#` zU;IayqoNhMU%9wD+rrEwTlToK`%km?FNn{ttcxQ~=MmDS)vGOj{9T`zu!C%B{tH8b z2(>IuB222<1(k-bQ`w#FZP89XE2Qisrk@XzNOX!LCII{ms4Gq%_kyqlnAyr`+E17=?Fu3fF>N?ZTFlUj{M?VdOd&036v zD!*f{NAf-vmXRDohMY9i`%dhT9z(;5-+!O>q-<`s&jKQ|lJHJIFrL5O#4bdZQnBx;^k^oke=q^YzDqGX2F-h>Q*`JhgS=Bz| z6nY+it@)HQ^De(Ox&KoAGw08w&CqlnX3yj3KBn`$U2p|K7<)UH*cF-tikvth1O!g^ zeYADhDnm4k2qSAhKLJ5Flec~Yus{>S8d&N1TGJ@?5!zxXcD-f3&-6ls&t=iLD?K0 zK^S?TA6UR0&EW?BYh=ZQ-9d{OYo*%}ZN02wzPSg9qD(rvrh4^t8iHm8 zPU0e&^=LlH*Kx1;dtuY|r5x-Us4x!?ZC!TQ46mQ^yv$K{!BuNmiwGW|ph!s038Eg;?HATV>b;^NsU377^~M!iufxj^L!v%lotxjLG0j3ph76#4EnB$oc>@k?^FI=DmZS9-a6@aJ|i(5)=IGs|IyTYtAo4p3p%wak$k8_tF@NYAm2L;^jW>3S3=xot3 zg(&*pc&@BSU`vG#9celV5VBoUW;+)=n^OPTq<56a$~ntA9gh%KR#|u^oXu(S304fI ze1yk-w@u;sf1FJFzhc;j`u^%Pep`CT2kX>w4KnE0;{6a*d1BWMNX^*9FK4+6cLaJ~ zdGJx-lItpT89g1h1-w;VCdW?_=@E0PqUKy~cT{i)QBnKcOP>U|-3%OjEx1s`c{-ni zD3?InYeaYQj+V`VBPV+i%zz|V^LiNR9&a5}^Y}JBpMiQzx$s8%nVB-0Eqn$1okX{>7qi4F6XPrZU z`h3)_+0oFyT=z6hj!>$!@_b8tZLHK#byn}Oy!oVe%y1WbWPq~M$V$fvD?f06`k6vm zK<&PC+ig-V%biEvh?FBJ@UDSbMlTiBv@>pIh>^XXf5kzK$HW4$JeiNk99bA zWJ(^*KSf5}}d`8M( zI_DYTI}>FZlY_C>|& z{dBm?5iX({V=Xqi*2CL!<1fEojsUPX@W`|LwgBW zBkzAIAah@TIrljK2IP?V>?YQ}$vl;4d}o8fbB7S&(ZXkAKm35FZwz!CqkhT~-UmCg@6%k6mA!VN^C}kEl9(81 z&>(rbcTy4b0yaA+c_r@2d--r%)$VlS&Lz4an%n+^$aJx^^$?BQ4_q~()|2k6N}DLD zdY^2A5Ua9Ls$Lb|!}t#St+ruQH&^u5J`GHzS^Q|d--#$OD~BbgtGn{2LDvSk1UW6@ zwZa4z4g^j*(9p~ccR^Z=J99aZWnZes`%o2yos#oFdKy!Dvx!oYwul@}6&{E}Rn8~x z>NzWnmZCyW$B5QT65YTI%i!fb*nHc3r1X8yRdELV7cH6Qpbd#l{pNjOg)-y#JF_F! z7xN{N^>I$WWC7xDS)CQ;xinr>sGKG0voo5{j$=z|oR@7pmuVKmj;~W~nre_3ME{F^ zMd`uQaLH*-^58{PwannRtDq)aj(AC*)KcC{_MTC-^r4@GU;B-51M?qdJwn7$Cy zwt;~`wMmQ7Og^y7^Knqe)=WouoBt;pwp5hPsPi?$Oj-Mi>yB04rPyrkCg|(YN=X!- z!vC&u^*{J7!oLjVc>nHl$e?^K?^U5STUBLW!Mov)BbCyN5EwFG`vxh6-z$c4uJ2`M z0RtRlF7q{F^N2Yc)N|$F{5N41><3U;{J8;f(vg5HJvkwdd!0h}MU=B_VGOO2OK>9T zWrP1xr1Jk=lyiR66<|$8>OE*D>CYe-Wi~|y>oeNTr)+-(!2~F1Z~R^S&Hs5|P1Xc3 z`Fq@6rcENYzxmRr4B^?JjOfC-hsz3PG;>0j0oOgK?1ok?kE)-VeO?a2dk0i?#+Nm- zH8&|@2qeH{S-s_6G6qIpp;mimD$-+b zp9fto2#F0|U@HnkXnCJ{F?jFe-qpSr9)P}b%JP8Q+vE8uhWW|y$ppXkynu0l0=dT% zawaat(MFfCBGkd_VLIJyVNp?-L@yk~PfLsJ%A0g&d+wfIb$CI$Bx^b|4IrXy5FD|SV$u3p#R{@#zp)9wOXHyo2lOSv{dDNvly$aRc{6HULrT)y% zZ1)9yyeuqO`3LJc=M8N{(DnZLFuU!$rebfjpsQ@}1-^;Gy7rAA5DYl;bj@|N?BcFj zAAki*!Ug?Pn!gZ2 zp?PBlbzU>G?8^OvAVU(4&qm}*8pEWjYmPq{N%l%K) zunDh_BP}8v8n;8DR!m3<<~!aHOl`PLVn^(<6g zJB}Kt2->4LvB#z#g^(?+b0=-Cyr=%GQ3J9+butDfG&l3HBBOx3e17-gqd4a9krc*a z&%mIo$uD&oqe?SimFGX`^=T7W@gBv(2m}rkfTWhMUOJVOzDU-6UHvvP3mL*~(7^!( z-Hu@XIzKZ)w6_^qrfo;x_SiTr2kpmuv(@4AQy_JsXl)n~*m~$a884=L(vQO{>JLFRBR|qbP5*}{PvS~t-+z1~lnf4*( zJPTP1#}UIQZW>$l2cvqrGDgHjlDG_a2lkdOQWnmj1@7as&&~G*!EAbCha=T@%=gFy z>RFyG$p^J!3TavEE83rX);ydXt=nX8D3P`oOB9T+Jho)RO&J){9~ou27T3t5clSfu z2TI?p)*N8p7J%sk{J=Q@@$-+E{3og`EdM+rvi@yns`xLfus_mjHpT@_SxOiACTXU2 ze~yk(wHe3fL=AZ1_pKwJRN%SMT<{~~Xa{}Egu?HqUnt4v+2cD}=4H7*t`-~E=pcR^ z&p?l`A{1|3pbRY``8bKvF)u*pnQ%^13?@BRda*6~+E8dY7eH+hH9g;(KA~G)j51;u zAW)vq>NxHsD@9nE%wUDiF9ul|rAD=CKJy+^V_GoHn{xs_@SbX2)pCOgb8|W6tC$sB zb6s*|O4`>(kq+Q+8y|iv$2LSv=ec>(42*A~F1_+_c5_9&)6r-8xe*cTMKAz*LraTukHe|cKn7hG_Xb*kGpt{Do7aVsQ2~;{I zUu^UYb*%ZQb1~jhEPc&ZjMtv#YCS2r5o6eWUIvF_t5hVsvfE z->L@xH2VdKPH8Bg+b5_)i;m1~TR4in+X;NZL}o{`nZnvQFjP~H!^&Frz*U_ztv1wp zW|&U{Y?ubmwkm;@_Rk6otxnImRQ43CD_5KDtkg-6P$)6#Gb^ku2dof@=n!1x>fSj5>Y#m6``!`(LG%j5O1J2qt=s!67F&Fux4+&*N3 zm%9|TXc)?yL3j3D?u2~baE}@AGZ)WyFR_YuhKf0o-*Xe$+#3+6Ilb|QcxFV4l>U3# z_Mc1~!gQ*8NzT_AeQbm$Q-&C?QQKdcbLQpeTG;HIS5{5gdAco)C=o_H-rB-Y7G%*> z{`&0rW+nh+#$9Z#G|e=v!4n#!;6chH{4q@|_MSKdr5dn2Vy(D=80+;G>HQ*;TQlo) zuU*Q6*IlPxCM#1OtJ})@ie(-^BY5J-1U7MG?Wv*iN)y_9*+ghKR zPzR*m#wKqgccBNA8Wr32S06)=UuSJL5W+RD}2 z0fEiY8BM0?xj~B_2}tTrN%IP*k+%Xs@crJw!8~I9!=%U+_K~vhtAJ4BMZ3I%as5|D znM<#d3qOtsmu(34vNE82*h#;JM%$BExC<{_+UL}p_ouY0JwMmdrZ;}#J`nMCn5OMh zWqoNFF}}W+YHXcEVgp+pRiVKwmtu!^_%|keX2Q|=!O>GQ5#s9y{hS`|W;M^bR`^fV%%VgmY9@}HP_0`MLuB9P@OSWlZD?7 z!(<^KtS7hSy%fK>la`^$gK)7DpSaS8Exsxv=e)%RLmlQnJtPPES@pPR8^qMOZiKw0NeBQ2_H zU$eOMrc&5VUqE(KK!!D81K~(7$CR>Q7UX04{Io{b@bXT@k9=dwkRPsyML8EgU4iR; zKC(9!POm+Ygi@=OZarBZR%RLe@mp*S?e}UThRWyODS=;3zFLX*a#$Sy6&K;fw*Wk> ztBYa%TlhCi_@a@`yw?UrFy8yy8~|=e6jIdP89KW%8d;T+%aGYZ?x!h3nWDTgA#?tzC3Jn&d1#D&lH{A`LFFQCUA<%ld4MiJ}Zhkx5Sj&y-g14F0t%h@jF_r5_rrYTWzNNnb3M%|7Q_Y=%W$A63Wv%U; zHc|G{9Oh7LwJIH4Y6Vrd?V4NNsbBi^J=?o%Cc8CdTTj7WF%$DEotHv_Rr*dlu*9U{hxz;EQOb9YU@Vv5CprLr9co5NoQmW>tR zHS;^kriQQU)SP#npeEZ>blo(Y1WQh{OoRX!?YHN1+7^xpb-l0#4ZDo|__LwdPi14} zWtn!Z4WdFqCx=p3sHzcZrL>wIkf5QWmm^<_X*N4B#`W3VO#$iI-UV0?766!ou|-d1 zlLm#N?Cm^2vCn61BoOwY$vyj1cD$uvMnezMqc(p0#MVTHn|abg5s9&e}qt0i-MCXI`b>}}e`=pRR#J2owFm2P z{eqW=2}(v?Ic(nw>gCVOZ&JT{^W}e#_nuKrc3ZzUNKja7n7K zK&anR(oCFT=eTs-Oej+G}F?StXth^9dcQ*ZiK`8NG5+@~pnLZTd zf{y5^>2T!SR@_N2YW`}Bu842R+T=skBpEF%us1(@#{$1$PwfL~ko9Kay4L*XVJT*M z*OZTmDow&z*Tk_yLL+{&X-iePkSaT1)p75PgAsmdl_?2vDpv$MMD0Ef{t--c#9!Ac z;F9`Ruq*#6DSYM60)X~k(a?X&{`=d4nfrxaqF)4RRhoL#2oDy%tLPNzj>PQCja=tX zcgNrEZD92?c{pHfrC8{5quzUgd#zTB=HVXvB+gYa;73|?%^7u+)ozZumD_&6XU|V1 z;(bq1C6EQS>!d53BaUlWrd|a(4mOi%o2{K>MX#?2HrWH%Y6%rl(#xyaQ{8YF0%DW@ z%b?b9rvi8n)qorSOlhiRida}2cB+D_>)qg|7KQI~wu^^wrH$WF=iX?b4zBRzV6&>6 zvI1U+>cZgG8eU;fn1pF%GMXC0!zATla+_gh*HVgdJW-gUm|>htHg??jNg*A9{~~x# z_BQ$DldM-AFxXt*hHH(W?dENg7Qfc=jopU^JB#;GJ7Q^m+2|WS32wwiqiNkM%<;4B z^FxS`-aEyY;F+0Y%VN(UndA^p4UsT3A|(n$<-tX-o=bDk%d#AfS7PueYKzn8s1F4N z1h<7vW6~8teD{hb4AX%$cRB(GBPn$ou2BN6FRl(dcwR}I<~$!?H=LF7#9BMEb3$Ab zTh1a1(^)dQXzp05K{2R-5tewhity;^(_^^JWW>QsjA`yWn7hcaaB!bJGzESNO^w1VUC3 zdrr)*J&sLioN3*}4A(2xpZ&<7&P#NbCzqK2EfLj484x-U8}9~9J-fb3uFQl}Mj)r2 zp<`o#$~^rs&`0Z8Ie51HTZiLu&VHOp(jhdFIO&OE98H4M(Oe(*g95ImKP+0 zQ;R&J+o0*FC&>rrH$83r$Kk7mt}q)aUpYM4rERTV)2UoAbXD%slt%$1lOP5bFC{x352igR4s-z*VhWNzBWK{^b3NAi|<;S>8^(Pw-) zmHVT-^L%YcjRDNB#_7rC$e8Z;e&L5q41e59UbpJSCKbjZqnz`S2368q$SnLIILy`& zx~f^TDVrk3_WR9A?%!c5L)KR(FULx zu-b<46A_I8<&`_0e=llw-@R`E#+O!z7F0~7Ng2sa>F-`wmkFDeh*aHvxXTb0=(#y; zw-96%YZ1!dviS?!4J|3lFNr~-!qAn;IUD_)oZ{*)J8R_$(zr5==yLgmkm7J9a?>LR z!)R)VwQz>225a{zeNU(^!}Nq^E8nGNDku$7#KCn+70PUk^%)B3By+k8sNzv(mvTQ| zOhxQe6}9@5UOt9rG|%oIPZ%CQ)E?Hf`9(gRD1OY734AeqG8+1-t0;shHp({Wk6oQj zcnkI2_uP>)P#=J9ryV1P$Tf@2W@Jo~XmN2g6W6S{D|UqAS7pMSw}7M={tvbFke`9 z*{p(8i9a=ia1~=EI_#@1+@}!J3u5MyzlDjHC;EDty-g5e73MeaAPr9Se)u1sSRd9K zTE2S%8J0r6bw-o6!15VdODwf#9&>I} zX!ks}-KiM?^x#x^T{Cw|kkSNcw7cKIZz6*!EttpoAjZ0^n+N^no2$2*&l>bJQ=OcA zJb}rlL~3+5#1wU3*(>~#-g0y0%Y{#44a;#05A;E9XYPouqXvcIj4c60s8A+N^_!@u zxoS7qhIz5o`pqYTab@|mXMdYz6R~h1lC!MwBC%~xIDMDq&Tg<={g-d@tB!9jhYn9a zf=|5_6$2}}nQHn!EAS*G8%$aro*J0e&XCoj3C1;3tRbaLt(o))Kz3bugVT}%aG}_P zyiJqfep&~Sdz$}hQW^4c=}{f{MbClPM-q~KYazaP^0I=rivgGg0WrPzne5gFObY@p zT8LfPV2vIy+g*yFCzLi5t<<<8h9S>PqU^XuW!!GJn`F zT%B(dPZL$pGhwu&eaDBsPPgaV2>5uqGf(_-DGC}I+_}``T&8Dt#~%m3$#SMzKj)P+ zRg2t&I*H>t*mHr>qP&m2y~iegEd023sZ8H1ia9m-VY*wH*F4_qdx#(4KaT#a}T?jO-d2ED-i_{h!DTTF)HrPT)lYi(ojkt!%OU>8SW*4 zl>CzU=I7$eUlr9$m!q_&HvSS{o&~NQhm!}mB4wTdwwqiHHSEj$E-a^!!)nv;Ga-c- zy{gWUU&00OgSkz_9TytdVnmVQsiOKLX(y=$;A+7Kl}szS43=Tc7wzxX)ye8vU4eHS zSpZGvm4z|0xC%D0Qr-BT^|H#OClFx)P@RWVs}-{`z3zM+?2Q;dZ^{A@v7?BU-og9& zB0g-t)sLK!%5CEEM$mPd73XpvQQj*SM+k+=((~HpKltL9>nF-5r=Jv07v`77JS#=J z7r4Rla%Cwbe&};cimM%vMpV)umSIJcVp9QLICJ)4DL-fls(5c06(hMV{%CQ8E2{A$ zZ+3P7*%+NJn?{zVP3Ysc-BPcu1FM1C^Se3vpea&`58*uDm3yNPDo$Uq^m5%!SEjRM z@~r%*OQ3o+jvM@n_roNzjT@4yLBBk~J(;13a{aJz%j`?c>~_V@CPC&CeoD4F*-)ll z41<({yAIF~awgt5(m>U7qNwETUt&cWiXUc~HAlOYhxq$Do1|gVp!Nyz4~~=BB-w}v z+)FuXN_!0vW=b>gZ=!7yPtniHb|p6UjW0$+86tsmc!})uuf4AWwGI4nix{nAF>oxVjTvuVYX|~|+y&#X+3V&#kZ&BlR)W=YA)vKHW z)}9BWoMf&^)@&Su1wle>$$}Ugf4ztgH@M5+uWqL4`MF5k^Ob3oS~-y?ZhdQZu96_U zvt46sb9SSude(`4zT`3wp}JLHD<<;3_eD?RtH=g^(sQxsH;(TNccQ?-BjuCWrj3fS z(&Du9OU_%@G6?D;>0*_1dV?7_l|4;JnP-ER-;1||B0Ea%NUs3moc)N>THaOGqmPsV z))O*k`N#maNcR4y*%sGpjs}Y{PDui#9p^p#q)jD1+yPDazW3aR_LZLeIK8Qn^%rDLKY4f&sw!W#+>?2WLiE>AlcD-rocsCOM58IiuQQbiBf+)h)N7HaLihcd`@xG; z@zEy6{f+Wzx01Fi48ZerP1ySz=D1ysr1SOudQ$FfAC*|q&E zkqyLBcawlfWdp=NpLx))5>*ht4Y5^v|Ejm$6=P#Gscxu8YXWooX3x-f zPRB6wwncm~3lr()F1(688Sj20riPk-AxaYgS3sqbg#d~^gKzY z5TvnF3W<1z^YfEIzb^;@Js!7a*O3$HfNzt>y-M_MFzS)t9 zB!>6Kh&UhKX|0#eYJsK0PR-|PW-I6@q7O`{XrVKn#r6wIv|^5l<_3lnYPwVQ#gPgg zbe|lFTxi|eH%Iv_GPQ=4rTyMn2ajKJZx&bjW8;aSzX`cS)|^ z{3e<&Ha!(4AdR(j?u0CVFKg>o2Qt~D84@^BX!1J`9ElGJzr);)FWLfNt`cPaO$(k} zJG^#ZJM0%uK24;!4+~x1q`g29P~i+B#Tu=W;4A*IgdN3tQ;Efu$ULKtsRc_j-rzwS zj`&6zA8`NGMBKU9Zz9~i^s5K&F?`rTT_APO_Tz7z=h5F1ln=c*dB3U%vyQ7odPWrl zQ)d~ES?^WK9E$-RU8uw|j@WIq!owO!HnWB!v4cO;E9w@wpWoX=TbomIia)cp(v+UP z*ynu~$AtC!=v+{L+TqgXKC=j}Z^cEj4Er~AS4A8|KY*$#XQ^?)?)I|w+@4OWH^Gd{ z2;tp?xWNmoQVuJeLn{|wh4H02?#-^orW2jYijh=C)$)ZQQOQujuM}F0%AB^%wwOBL zM{_$N0o4-E5%8-pdJ+UDXHTPS%aY!;-H!vIdNX?}F+%GYb>1^Zf4XTFM^@PELX>^L zid+ll=0pW{fYL@UW==y_ivgi4xaML14nV|&PsdlKo|!+sgrAKmDL`MP(Yq^T%84Gn zzokgm)S~7Mmq~zZ;AmPe&dF6F{Ly@h4h=ZOhAyP$qKJXyusaHYTDHSWO;AWu$pV}V|-v%YFJ z`B&pdz#P6wd9-nDUoKd#I+lvT-L&!L3X$Vz zjq|diVPo`1ke0#DN)3*crC&aTXbi^|;M@`daLkb3gpCjLo`v=Y6k34So+o3mf|P8M z>J|<#gmOpWFUtMfmKBJ`*zk&a%@1&sx9r__jvR%86xehGmIf3o1ZTfC4abPLudk_s zx>YnMo0~^Nf9Xmgv;5WHdDpceg!*#Wv#QgTsOZI4;JHITrp~LLc6Z8eBKd$p&&p*;{P?^8*ll(_T_%^g=f*|k%fhh8NUl0B4|bkeZ|`Q!k9{(612UuNjPmr&C3fKo7fh8 z`)|S#_dNgqD4i$}Zr+_Q?;3V^#r{0}jwTp0n7@4-Im-05Ui(d?*yjF7?G9Pn{jV)s z+5Z|Rcs2fYY4N{?3m?&a`PZf3|2hAn)dK{n3)6qY`^&xludX%B>F(E|$mZso$;V_S zdXb+p!+$cov8z(UW{C#nS^ybn(k$ILKx`y!!CDS!)%`<{Xm7&V7iR zD+ZN$v*eqLsGhNqpsZ?CMglW<$BS$YrlzF*T0o!>67T{ zI>a&8MdH}xWv4g~)sxe1Sr4iMMSLW!EI9it8Z|C4?XRRyxs?N6N9rEnrf*l$rb0Asoimhwu2z>*RFOons|}u&m#yaoa5m}nyEJ>DYxI6xdnTt z?Mn12*Vz((W?TwBOfj{FIC}B{U&T9F=kSY@sFGe_i$y@VmiP+8gI zei+<%Q19DEtKNs~A-D9@NI4&sW|fv`FnOFwm`_fN)p+Ckn<$TnbUOfk(*!x1 zKfR9+v(81K^Y96-Jb^G{DF*5g6CX_PJjvvP9OxM#N-jhLo z-JbtR_}dZgsWZ+JL=3WI?C^Q&NYV3zxuzDrL;?(4Y(8afiD}Xb^?Jg5(3r`Ty(Ii` zjfqCo>Tg2xEQXxk#NHs2qPjlJjQbu-$!FKH;)2IooC>Fgfv)3ys^B6?rgi zs8~0pm*4Ao;x#E7ZohGQa<=Juxx?+jB%??W^UZlQ8iL>DDXK!y+$ceS1qr-UMO~ki zC3yhDN@`GkOn@H#t@{p@$yUi(BNiGB|6rdxVw0c}5;KV$$)9nl&YCo8K)=Ii6alI;{7*+lxEnfnd&n+9hM@s^NJDor~53leen8R5`?@5 z!QSwOnd19|s^_su*Fc6d(%cxEl=R|b^m5^+r8T1d{vO4dC{u$ofo&&guZ*vmhT5|q zZTLv|iV9ckVIw}T(W37(Af6S@pW@T#dZpC5!U01f!P;_M;YXBq(*ZG;LM_hJSOsKT zr4fgcLG5EH=`PIsF6xE4(I#&4$yhtK%GWBZZ9-Tz)$Q7#bdAgEtwfpdT&E3t;-wGJ z@_fd#T9XNLBOs8%(vOLbyl>~qC2l3|r{BrAloX4n z3Dr8-KD3O&yZD zdDFrnT4OE`mgjfL_>wQ!VR>&-X-wDV+k z14~^qDaXW3497(tRT3N(qLWRN|(tLyZo6v zV#PnLBlO_GwGUqU=A*;XJ`TzL3sjpgKC574av(BuW%4NE1TXkmK>&0X+DE=PO`L=*VWrGPL~bvqb$LK)Nm52J3|TQ|K!pOK&K;;n*Q zdbXJx*#2bzQDrV=I=fL2mYCRk9-A+g;hSSHKR8nfe+p& zio=c42DfUWt|*El07mMM zg-YE^XSXr0M(&nFtTcgmJ>b3>Q&8zw_uPK6t=Vy9le1eO4)(_=`yQy{8fzbo+{R%? z_BzIjeLxhd?4Y4sio_z4H%+H`pZ{?k{3Z$EaCddRrjh5vRfeIgjWwT44^G|AJ@lv+014_M91auB&l}#g2AyrslQ?F3l+e{@*IytyM@hqq3Z~E3A-SbRIOcI9EoF~!t+xe9 z!xWJ;zILP4uB@zr>K+<#HA}2;#8bS^o>te&jF$P0xUa0F@GzCm^YYBUYd(1WFXzx- zZ5IDQD97;Qt0w6eR_of9E_HU_$%Uxr&a`h|iK*UN&2@Zh?Jmrq{-h3kfj=~n)DpG> z>3T9+%G1KaUo9ere2*;#s1@P$k~4;?d#q)GyBkHB@=vyEhWxTRAmKpLTZ&B;c{&nYH)y1PX|~OK1?xy0Aw34 zYO*vbYi)uh^;EjC+|cAq{~5gn_o}Vhw+0goQeI5e{k$-8Sysol%C|SJN$NZ7>kK~( z@Vl+!4hFwwV)|6`HsiQiGg-FlWE>fe7}K9^USlKJ?PICw^NoLzMO4*HKXZ87jwT^3-(m`0QRClR=AoYC`}m$3pDlyK2!4 z`T?-a6^|vTm^VK&{|3a)oD^lGM`frWKE2I4()UwQf2|Tjlb(5G2oGhxCs}$E-kF`P z!r$~O@lLZZ1yAnOKPN!|=kGga!#d@e3LEJwaJ^~iv@u7L({#v@-9)t9{bVmhR@(ni zF-L?kO;75|_6uRIUv|c3u56Kf281!=33u?c;ykwsH4k|fDQWMgw;~mJZ3R|OuP3Pw zw$OD>{)QiXP*&I=L+M1XIzbnlbI*bephgrxV|l#QK&orymda*!L{i#FKaOv}_0tGA zC)NUOPn~8}>DJ$kZ;xWEI}7&MF25hr5xfUL4`kAnnWZ})>P4^wvn_mRq^?w{g5Mkh=iE1jHW~trbv+<%vs*FI zg=r@`@BThg@UCJMx0N277EfPg*qe|DDXWS*=E_Q@#%#btWnZ0s zdsuFtZR`7pSDiQ(YAB7Omc+1*U$;SRjE`5H@?ohoy>Yq@dwe6i^VZ7oAg=w(MuUm5 zU+T1oN0a4n>5P#E@a?EbtygdtT^w_@{KBZLl=EVHc^ZCE)OmW?n7x2*eB?VX4L>iz zutlKWaZ(Dsai7|pfQ>qS80%N^NUp6pGiYt{_Cn3ndAA?A^_it}Jbd<{c2&A-OwE{& zWG4@CyLD)=Pb@gwik1*Il0n_l1s^j`$3!i;3w5ro_Iong)5AW(5W$M_zll!Gnei@G z^%zmn{$uZ5fjPk8h-G9o1!u@Y%s{?X({#o}vdwAt5YXW0OPsO6&^aLHneAHytrA1Z z5sPI|EE(=ApG`yY_(ba@{rDevbNO{iN7#Z5#mmkYOAx3=fR6EF8R1QjE#26Ip?QzY zi1>&j`QT3PHC?UyT6HbUxz?tYIg`i%oj^+6)rh3VhU&yI8m0lk2M~~0f}X8oaRc5Z zIpI`;TG!QSSZrN;S`o%#9Y>Pwv|W843Xd8CTUFK>^c6+6&Rj2Yf~Hj{dG0R|eEp7c z`&+@ttv6sjPtAE`QPUGtVMWoJN|*qvvG$X@kk<&GgitWAD=!>+Asp$3zmVB^WOk|7 zKqXNtu$BCnS+_A=ez>1E;1}oFn4#_BO?Uh|y{SdlW|drRHld3A_^bs@PR?zo6;6wX zS4^_(o|m@zmzr`1X^fNmb3nk?zD`!2i9q7lw>gKl6MvXZ$5?{Me-jl}1_*{WzP`q6lzo0n_h3)jN> z4uZjQ*Y_B_KY%wB(|PL8QEQ!>jm~?u?{uO$*85N~;M(;O`Ss*(kmpBrJ9Aqv&xUyK zh`17jeEPC2a+03H)MUsL%p5@_DhnugB#_IFE^nAxO7gb%7cH?-)!#&92d1TgUSDUVkE074G7WAhHXQLQ=VVWV>8H>Qx}$td zG01$2*M9KtZl#nr`tQfb4a&nf-yHP%)>ZE+lnTYb40S!+P;ZX0`SN^z$@Giw=k@RT z@;}sFXm}06rA}8M*-DL3&yxQL)ii3mflb+!tEpfQgt3ATKJ|7NdohaDjXt`(aqqpq z!*k!()!G|{3uD*4xGBKBUJ34th9amph6QE!!q}RIIW)RVhs{N3g6Ak3ux4To|0m)8!V0TL;Lb?exCXfkrr7Ur*I%edmB zPZFlT|J@ND8r)Eq{LYN-F}X4y6*OSp^*CAFRo>Bxwvomr9rVZma=V3T z@O%5B(_@L#0EQu6kSz zGO|Y!j%fn4K|CJSj6(cG{o>-Z%(bMRHdmrgF(QGHY7ZKu8l~cz+J0&A*F4#!W>=03 zR@^L&G$gdRd%V)ad?@*tmy~9OsE2tkW_JD5M3&C4dQOnq3Hif$rd&p_qUz1KCgq5o z1RsJE$=;ALsxl;U7FDngZZVesn7^zJGT4jW^p}v<>*uQw8yOFKk;uwL(><0X0{g)n zOUU*+<~1&qY+~@m(mv4+Fz4sF$T6TCp0aF&F3NU2O^=}^xYYf;nbH`6-@CkE}@#dlAfvsiX-@Uv^df=y&v zX^mYm)AOQfexDZ2b=r4??ygIu5fN<@>B}h5cq;$=v`i=Bj%No=f>)oo-+QTLbgrPF z<}7nG6y7i?vkTI?>wMeF)$G2!Pp}1Bu%6`M z`20#msWjPPKDi{zdT|=2e*Ab@SIQ%;R*~#|Em0ud=z^wrmJ{|omKF__M&bo{3ybNS zP9k;rsZV5>SCUVrgC02%4|YFgFM+1PY84@($;7Jo-ae;52x7Y)Lns#=fdNh1f*ROU zD`mZUbTIyhKgrgcM0Lgd6%fzHn$$yxO%Jj@GU>EMKgzxBm$!|z&4og4`f0G3XhSY^ zEoYYa7a%t3em~(p&^JixSHYnuu`~=)C{CTMxwLiP(J{C&qQWVU;^G#3o1=?ns>JVb za59||Sd_Ucn#@1ZoS!^lGFv(?SQag5TTHgj3Q<+cRa1VWv+vfpm7iOCFn-YX)v0Zd zy>ktluEEsoVmQq6CG?XED`_OsQ-mmPn_Ad**SOFUHI^zfXjXzMXjT~Hz*nv^uH^zp z+8q%@-UqmcI4-KluZY%+7Nm^?dVF0^ z9fDb1hXnXKwsHnqR{<|0+hPeylosBTL1W)sL|U3umc_|j?wBp-AxMlgGG*3=*4JPs zy2&Lva@RX=^3pf|AQ-zKD0G+h!DMZ%`?K7hmNQdQ=Z+OcvQ81+^(GGvsu@$8OVJ6_ z^qX(oOScLjo0+cZ%G>WG^(BoWw=!6JgF3;xcT5f8<*6>pBrm>pZe5evq9joOtd8&H*5Qg<(AT%b{q*n5MX41HtcUeWu^(Fjw z=RgV8H)Iu#!ECA{>VZV6N>dfTtk9*>eCon`;2dzV<8B6HMEPQ;|U%|)AY2y?u(Y21DSybD3vO`eoDuy6yCeT`|!0B`5)DVmFTiLSPQnfD8+Cg1H>Vh z7~SZw4+gtBh|%yXf9Mgf#LeSGgD!c?Cz!T9|^fjkYwdF$2bFr!$nk{SiQT1aG8gQ=;UO5k0x zBNFxmm!n?RpJtXC|2=e%KUFaQ!I%FPCdMfI*$SDkwU9+6fUGji+#k_r$n_LJN_qXB z{&CRV5izgUKl*fto9c0p?+ix=rX1y6;!&AV9CK9>{EFso6aGO;uFa$b7-IV%hfh>x z?zLBpitsY852iCqowAz!S4q2Nm)QoY$O$KcDnTc&3~2R=%9@IIaawI{^Cn-=E0Pm! zFq<|*X9Zn~B_5v-Y$=b1JVK<1$UD%AEl4be_q))E=~&u$`ZcgTMWAbQDH}AuZ9ew` zcd*icDXNq3)i}2w&l~7?-foR!lj_T38-=%Tk>$>4Wk}TQEB&> z*0iLFMi8JfBRuXZF1W4{lR>!SY#IoAg&#+(MO&tIw^EKv&d{6^g9@vRjOK?PUF|`j zm&5*uCe2L(VuLMhu%JTJyEs1#Vs}$P;RX77QLdFQ*SVfVQ_;{7ejnl5Rh~1bYjJ2M zJGXa`I)ZH`*6o16pK^(_;TD)*%2UxrWNL`GUmTF3jxeIR>t6TtuA@T9^DRhgsFDC- zvQo+g{3*5aWu?P+pO%Mhl9{$9t9i!=xO3ls@oDy>3){kKR&Xpq@Nq!wnnYfs``&{Q z8C2#hEp1|QX7r{wDQyB9_(URXH~d>vBFQe?*E~~S*UFF3f7Zf2GdDq=4rnT}nq3ol z3^*uQ5NGzhWo6}N>4BXX@7h+bktHt|FnZ;`H2C8X;}0-6P+AX$|=jED3hx35Ix6uAKB zBniH#Vm+%;K`?sxu_e<3Dj#pB=g#FkWKCdFLL;;D@@Bri{9KmZ6$A^*>{|T$I7dwq zRTd(lKQ(Iq_axkZ>!*L~nEB6_hW}vo_;0!7Z%;yoO^0Hl39KB2=VaW7cjuzYhwKS! zuR24YSN$aV&QHBG_nV0PC!rUM3VUnaFY5mZBP{p}=up+hnj%HNX=ea7wBsXh)Wxr3 zBd&zVBM$;l{sr(~qkIII;s4_!I*BtGHcP@^*l6vKMq3t8h7F%-e^mS&p!oB>Ncr>r zbqIoo|CvLSnP1LjV}1fIXp_tSoD6M(T2Jd6`j*z8FVAYr{KJ*ye{_HPe`+Aquj;4j zcb+FJD{~Slvsw-W_SVCv>r3N}Fa8&=&coTOdCoA0Y`ms&^9%+lJoBC3Wa3^m{zDke4!{|+2)RJlTi+5F&9Zc~ zkLa9KA}UU6{ci@9l;8UxrzWjbr6^~?|C;iNW!HAYaSt=LsM#n#(2`VAR~XyNuDkG@ zh*kJey(Mo&30B?L*=Z6~T*mVTkQljTaV{0MrX@PBj_{@s)4iiCZ8H5f6f|xtc0_i(G`|+EIa#?R7sVN^ZM*I zGgPXTGwpw$=LDW zM;?rY=AzQEaU^S@hr)!U`LwO4k&^-s^l&g z403cu2slG9@k+;rlYgu}+3M3!Ivi58A;}R&3>PO+ktBWj)S?STw2Wa#FAYY25my7d z@kjZ=$8o=jNU^p%P-t2z>^6{@a{W0p{(zzy*+_!W%AB^SbwuDBB#g%&h{6{eeKz~PQt}vsESWBs zc@w8+Od@eL3l_ZgEIk*sW^y&Hg-w+Q`~74x$Yu0)%=?Hh+5{2P7X3sb0H=266{+Cu z2g%a|5Qk!C4kxk5RfFw1w62I`ebZ0nj@NSWg$~1g6d14O=K~&o98YsjabGN(Z+~bV zlH2F~%73s`*i9#V`ftfHGWHh)AAa-l5Sr2Rvh&yHP|2`IkJKWd#hB(s!#C~-LHQSK0`N$Vw!F+lEm-W{d^_YwLV zlS8+!p8_M>vcdHOg}jk#^LCI1{diR^ebc6Jsxr5 zR)8!A8x8TYRMn$Fl0D})gqO~tVD_wyU*)bJ-#x*<6E2c^ zUDb!iBx(M+^)MCr4|8qx+oD^V#~w)q?>nj&=M@{?~+<0DAsU6}RmC)lj+JZtYWF*BKG zaksA&_)!&z4R;|jr<2>>TLCxq?6+wy2%xBq%E<<8jpDs$JR!$N&4vp?CBs~v0)>u? zwVor3r!Qio=b`{KTA9_&6DA6qR<^o6{&N$5curudU_8;%8D#AT(CaDQ{JFs=jQR6l z7p5gf0*NbFc`wKw)AuS#3{&F26}kljCK6XP)hJaN=?Brm`BMI-qG*5BWZ)|D0lTi= z_{RVmO@^1__y@UbwdVt`+;;Zl1$Q^Yo(YnW8foq<_V!3b(GWmMoeLGrvpd3ftBO1| z9>nJRc$3Dt(s2D^y}1erDDy~dW;(~q41DJVfdI!w zH{mf1WFk{E2+)X_sA#bIon#ujzHcjx(Y1>lx1}e26j&ke?8so1F4*i2lb#QcxE5J@ zR(D>Vz5;Kjd@GOTuo&ob1MvmWrK5#7_{n-yC}S69yn3#mzU&FShnpWj1g;fBs7_?W zFR(J{0Ozh1+WNE+5k}d2Ch}8am56%d$(wc5fOtOy*ZNsiisNE7^T8dWyvSK+OjC-r z&^;8Yv@mr*us9w^v=6*Yrw<@LjfG|ZNC86&V|(mtTGXg}xioK=3H@&{t-8-87xHJFo4oc*a*ML2w+_%nR~WqUgE}*1SmofMET**|1G~ zszRwKN#PUZDe5IUCi%i#1cReqG)oEtvMg=OIguw&W#_Im1p5;@J6*c>$zCtz_9$Nv zhZ87~#|Ae4+k@x+2d|5p3vp-pN4IvtzhWKk<6ekvf<5a-` ze)_p%IQ@&6G9yKynA}`8AFeQ&YVEH)HB0Mw zvxKT zbADgi`87S&J8l*);)B%c{GvR{Dyt~MhCwp5hWvlLEC2TXTwng-_M3=_JN`EjHh~Z;bBj_G>7|N!Y9vkciwy0y(~o+zg2S?uZsa4k+}RP8g~)sV(keS~CuMBZI%xAI1ck z)|`fe+6$9TFG{$aFeWff3r*F9EttHEJ`^_Oig6JBn93Pl*gUM;ysgYD`Ng@hL$@sr zF`|O>)%qZ{-6AxJXpRxmc?%J|i!Mw{F&^sa2gJ!k@S18AA8wh;OozyTLBcOe>P=)e zy~~TBd+)u9>9KD^D|KIPCS|*F>>2D1_Dab)wA0OpF+b2? zQ>3}h^_BFN8ku{NUkz$&SHvwSz;rlZVP~kOXM*I(^zEjI`MN|lwaxi{0k55=%DUZj zy_k@8-rUAd>;uT$+D)nvDr}p zSMA^2r3rPfmtiY)b6*vo?3v@{mBTD%kkb8B6*A3p*F*YEg+H11i!dz9JhJAMps8ZF z1&F0}l+)9%sL4|#Dn$#28vJ{@>Hkb$6V8BtTO0ka>Z*TBTxRt9O=KQ>n9#3XP<;8p z&hLdbuhl3Ju44;)@5LOnw`ILj9u+nHMAZY@C(6(31SLTRo{MHU_#?COrtnYOy5W#v z4o4ZK>eXScDCzcZQAAp^;hT{rcFoG=l~_oKEObf%o;#&MroOWzBbf>k4AXPWSg?_Q zHM@|M{Vs1K4>LI{iqBG?tMCt)J*dWxxrbcC`;t$)%DYj)nqE+*cK>{LMh)7_pMXu9 z-;k;hOw3dV0}BAZTdvOOHYw*kYRAVr9){pWOFH zV6vWkZex=*2AMF$w)EY}dLMQab=*berP~If*jdPvb9u~|q4c2} z1+P8xIl>n0<(fQl#OZ; zboKPU2Sa@$wkh*Vu^~dekc@)*S-T{|Lc(Vz} zwiveUlP)xpCSiX(l2x#%rR!dtqSvkK8+Tv-IR42@Yh${1Anb#aUgRFz&1!p>iIrp3 z0)a5oGf9K5*GCfpO^NXJbna$GWqP(>36+_47Vf$cKc`GH8zHWwx{vp%TLR=4-^#Og zENEWqnh#=ZoLuraQ<%4!aaRm^zow%gDA)yBkO=$QG~Ytu^FL2Sic?eO7q*Gr2ajy5lzya%v>VV5ivNb92jU{h1U) z3wMHJ3UURQV`X*oIyz?4FT@>nd*-l#A=9bYVXfy=^n2t%)TsM~xBTH!j&7Yf*(cuw zS7aIgEfyn?%K;BRUc@A6(H>o`6{^}8f7f~f5fiCB4X_Wrsw&xX%TBmT6IV$|XiNnD zCNfQ+#8gn|p9V}S=(R$uD<)3U{GFcO)NcMrU2hnacL-m9Ii4+h(G;Z9J}$a5F-S6zu(l zdv6^USF){*H)$jxXprC%EWv_H@WDNJa7g2gyAvQdG!h`egS*qX26wl{JxJqDLVi6n zXYQFZXXc*sJu=l-61`JwQEU-3iSWl#DMm-uEXU%?$GUM_i6SW zc|ciKVtQ4URYXLV1@)K&5xk6+%$D>^#nck6mTYm^s|X1zSPQ$h6Xn`?uM$1P+_T-5 zNGQK8i5A58h5BA`Hc`*!RX8MX*bcf3pG6b1=VCvAc<@JEllk}A^KA{%)Bzu<`~WPM zpT08zUNjm?-YOrNOuxVMI^O>FOt|VR33m3U8sVsV9qW%T6la>etCp;2)&)nlS7obd zs9b}0dKr?mgzzjF4WhZ~z7{Y>xhPZ%XXRCnA2=U5zF1Sw7eN~H83S@z z+1YY3hn@kX7UBDt7MOWa^|lXvSKUA%^D7&>NFVMM9pTfBw*q&jc{5Ogw_ZUE7(9fW zPN8ctN^2ieKddjB*T6~IvcemOZFow@8`JX@m-pV{Y7kD=X1!R-4utc7Xyl6Q76M#&& z$#sPO03>)1;T$E_9X`sI^Mspc+c|5LW+KSEh~8$}2(Qtw2fu@N+5*VCb^XsyrU`C4 zv-C;FC-p837;Sr|-(XLe;BV!axB)0BFO>8dZs^6xx%0Kffa1@>FKCFy7sti`x=I;e zeV5F3J-5n0CCgUtqr*e|{gJBxl=R;h&vs8?!WnjSiC>Gee*m^Ip7#9!$X+0#uw9Ny zM}a||i^96_+qXv=h_#^wv}}2R?LY;D_`Em44XSeX5VQ6> zl`IC&;|A-RdY|$6k7zb3@ZEKQ9j<;Fah4yutAXoj!L^(UnpSA*s-goqrLp3K{PU;- z;B!D*M^w(ibWM8OcVJLnNtF(CcpyT0dblTUiVB!Hrs|l(!)^5GFFZd0n;Xn$ z&M$J8)MaDDX;6KnhK^UhQL#whuu2kWur?Q$fD9c9TNMUMH|U0YM1>C;mDlSUmylG* zN2Sw53Ye@f!2k@OTCzlq3J}tc{irc=%;DsmM=9v_Iky2^-o4496r`P(SeW9N{>M2}~_ zQ)c;w*8%Zd$C9s|>k6T)@y%~6K=mE?%yIU+xu7>an7JwTi-{)bRx3*9v$|rRxm`3X z^AOo#b_c3Ox7AJ|d2jV*e9}KqraE>y3KDh{-O`m8%(ygWNbU`kay!(saX=&U@p^dx zr0-aX1Vf^QLB!fZ1mlG1lecg4CY>V|1c?LHt+Hh{I z&f^Set-Ih%=pr{=?`vtt7Nvpum1*s+s9{haA4rE6Zw{&N0LiCNUIkO1O9LO_TjTMG z79z7qHd_-J@<2e0?6g6u>??!(a#^Eq(Y&;6jhJmQ-n3e`1R3mWXR?CQc`yPDjg8az50wj3;gAULH>)QVnltCo zKO@azH7==7&|28Mz{pNaUXa8;4%_ykAQ2C9rpq69pdr0E!+gwA%tAQ0WK$dYx@$kk zciuehfb31-e89T010oP>S@<#O-EQx2It;RX8j%S8>B{h>OC*G$*LJUIk2b#6)1a3E z=!GRlp4-4r@$v04lm!8u*&M4X44BVoJj4`-kH+n~lL~J)bq6w7ahYX!~wO zvEDE_;VvjDN09l1{|jofQYRBQfeV;a`9e zo!wWWe7if2=ULwlLH&r?WrA^LP>NL@CRV^r(0}1q78QTozM(KrHbUL0v)l=DfFUt3 z@EMHhgBkZNb`LSI6Vu`CSjl|X{&@*UI~9o`h};90j_`pdc{_`Spw=7LfI%^38!WIT z%WHL$T9&G(!1E;|h?V6hsJb?D(Y zZbO||dW1`;{`=wZ7m&ipZVC>U7S7R~qx|IVmg|v1b@=wD(c3IGECK!pwYp3CpmKFr zm|V|m9vM1>(_N%sld%O1Y3puYc(oJ}qgBJSol~A`P7+?4z>*j6h@R`X`rY~&%wT5+ zi`@TR?2$G?dA3m;$fRBK*{uz9>-hAppx;Q(i{D3<|BRUHdpWDmz266RIO#CHxMwDF2WliI@n! zog#1ZwWbNWzALxzT~7nknahQJqVr)`E9A2K|nHq#@C(yR$J1NkHu2M}(5HjW&_1;p)ybL3 zR`kAS>1f}dZ&^4g!_bom7^!VgP{=TXDpWg2jZi94r2vr>V=EX}Yx0W7TooK77TLjs z7(9G1l9})CUf|*j_Hu*8Y_saZP*BK)*kTU-Q0c23PWZ^CC#dSugX3IB=665GR==AU z-#`mRGjf9n?Y+{~XpJqgVL@SL0bNJoewkTYa6KdX5}=Oe24D1|6vf7(0`;jeRmIn+H*NrXEk*qxKQX;%hDP2GWkH=*+pZMUlI5|0&HwC#M%j!-L^qu|LLR}~Sp~~38^XIgQawLpa<^xVN zSz#rtclP`RaIFZRnMEe}g@dA$-nN5S`=Fi-10J=VP-yj-o zF$e>(T0CL<7+GUp;~w-ikypgXOTOV0ErWH6P*vN;aiS_w-UhGm6C57eKGDS%UJTC2PlP>!htgm3zDbwk0yp}X-=Emo!br>6E4^AK3 zfJmH)sWNKQP&R>J@O(5hK4J7PJD!=fRYB7nx7-N|+3zcCQKff)AI*)o1D}OMnB9d_ z_yz3j{6b(Q%vz14TglWDnty#Ze`_U59swWX~eyodyd0bsH>T@ZM?7m@7 z-Revfx&P_W?-lGn7}yB2OQ85O#Zk=a)!n32(WTSS>%sNq8Mv+2Ny}o3vgbk~%M7VE zGy%%}x;9U)C3DdXxb^sM6DK_D)n^71mb>iWKy%kO3wid-EKM z9426*=KJW|Gz2WrJU@IQRC7VEnbuL_RHX~!ARSyGfY6|gJKkyCHr-^n-c?q zW)BE%m;E=p&7-l{hx|C?&CcH)E{RnNx5#wj5qahz4a_(Yh!`0W@^Vgnj z?2O<&;q&HTud|(4e$(s6?5NZUjn0`>*WeB&+K8-NS9@eCPj*EF({p2UGNR5$zULcA zC)XhbZk5*!D@;l;86&|xV&D5i^dW;OgUe;R=TmG?t6sql3v_9;19=|u~+^|L8a$Xw&~8n2`UibsfJ=)iBpJ9S_n)vGNIN}uE8DhT=tVXd+t*l zvUgTaZ~BQvcf|%}Ls2&!&v`CQ$}`7#@mJk!7@X_D#9)-7HJ}wkR`LQD*XzT~peQp$ zF2mK13g`_;iacF=VDR)Yl`KZrC-8T%M_liM^2ogMID zSCp01XJ2VAeqb-Jzt8I=L_OMmrk{)FpqyTF|Dk!|K{An~ZuVL{#*C4N|)^+;Tu6;v?fcQqZ8?AG*z`*Lm4lNeZX&Jzdbh&x5z9Xqww9J)by+;@uTyj^FvQO+dzd0b-JMjz2asR@= z`CIy?-)5?RSO9;wbN(w&=s))_p8Rg`vQdN%;avQS&P(+h)9CM4*FRK_Y=4h{gBc}% z#m4_2{QaT(4rB4sr*+5CX&u8gGx%al@CRT$g8OhQ zBb(+027AU;E{opvF>GG4htu%+O?4wpcYW66vlc5DEVN5g*QOD()1$bUQ?O3(cy8gH z!+q7_j0;!3Qk{Y~{rw=`+rOc)>ft=!jeh`K+LH9-{>2X>q~Aqwe3nEcEC(cYUYVq2 zu3&7IFg1!-%il8v@4ATGhX+gZ4y}@Y)8Zi$E^~0-;65~S3_Y=Zn3ezRn#`+>W*^=}Jz=m=?N2Tp@{GL$~$zdcBDj6XU9^}@n7>}P$OrnAf ziyGmzrmZ$zNH#s{6@>UT%aDa1fP1<4bISmMFm%zIjBFkagIlp(Z1r;o+qe}_K~2B1 zsNqv|{(aY)%3zJ$IAR1eZjNv7scSEHCb`4hHv5y-?W4Ud2015 z-+#+Zo9m6Rlgw+G@GMv$G^g+G6|U!}P&fRs0A_=P*C&8Y0pw7x>X+Us;H*II)1 z$;*7RZPiB#N5Sh6a-92~LuZvbbISH{MN-k3bIT<*rPk;Od|2$rI;ZMku&THH8XVD$sgqf=X2BGMp+2)d&@!rAd75F4x2P9=9V)gKXcYHTf&ZS!{(34mSZ zO_ys=MVo<_R4%hk+$(49BpYGuwx4ryiF)^_!6H=U_BpT=EsQOG#i7#YM&}EcB}Khl zp5kzAg6`z@KWcND1ke76l2q3(*F_7ldN z&rGM4Bc7HIBH)|Gmo7>rjo-DHYKhx)+3>;=*|`)!>9n-c4W7;tDp?nHx(vTfX%u%& zwpfkq=ZcI9gpS$+eCfbY*pd8re`n&AG;Qlh+M=5`aa^UtSD{yQPo~CPc0;TaCR#sd zcr43k@4Z6TA-Z-}uTZP5wpIzoey3#?EX*nL{QJb|Tm3SLZ>2h(cRI@B9(njD8NR#n z9=_^iStPBQq*U^bzMR{^d3+5J3i^2oy+J5mZDcp z*w(JM^wQB+=>~puxZ^yxk-iDVP5KWHOUh!0Piuk^ar)3THt0ToZ#DN}XmpizG$F}C z-oea}%VP~pfsi9fU4O(Uo_L4#B%wY8(9~7+c4CuRSG39>=w(RCs6l`f~ z)0wHk5qVJJ5%C5pp9zs=J;%y2&USHr6E9;(>QGN1j!l{}m|Z7QLL;U)87h%TcHD3Y8aK$zjg%Cl!)MKRN^2NhNWv2B9ka_GZA)&> z%WP?Exniue+pnBNw*Z+$vp2oO@>+Ba;6$MC4v=Xt-2Drtg*T_qWoxB1pJ!-v6F+%E zn)d0Ix{H?|eyAR${n(b5(PMRPfX$;yanu0|6O+03+4@>t($eAUexv$E?SjBvKZ#)j zrZj4&`s~RkM3)<6%fg9X*l*ifboN;}2QHP#Sc1`91Wnd6hpA)^4V%)%Gc1K4&2gNU zC(2o5sNe*fb3N@LZ!$h2K6>-jg)Yl=0AXr#dJEGNsH#c*1abg%}y5x$4SN|cB=yypg8+;eD%2Z+skS)pj`WkHMvu)oI(Pl-} z+~L`@x*pvgX7N6odMD7tD_f8Uu|~U&d4xQ{5^xxdYfKI}-b-eR310+%=H!|TX{k(< zt-HF6)8oPNl5&)$vqJqTUq$CdR(_(@wS-f37-_KHV<8ZvJxh~NN7<8yxT47p3#^?LMCMFnd z0&()k!f@}>zrmz^qJE+2dCL9x>)x|q&#RuV6pseQK_D52US_ypk3(wVYsRAK+JYBZ zX;rP7!kIOj9K5!mv}N5$h_q1ZT~&F|AcvlR98RYM>(iv9`|`CPt4>;a7L-eA^EpnC>}a7iR(aRN>DTfGc~Q&*L*X zlJq`V>{)pc4c%qZzZF>EV3FA0H)&hnd%V={KB;Tk+uf0UU8GdhJUX(HeJI&7Qd|&> zz1Gls&8nn-^Kf&TwX0aJ_cULU&C_y}I5P6C{12o{VX%HsT{MDSKZ^ zUPI=~A+!?LT52%kKuLNhftaD4l}4s>$(gi8UVYanpm%`d3=Oz`qp_e_FSs|dyukv$ zC<3P~?+rBMjjry1u9*fVqQN5ZzGsh|#;azn9&@(|dBW}TXlmqN_fW2Dk|@P;73=8| z%qP4gtDWek-Mk@DukqCEkc1SP)=5Yo@gyZ(Z!od;Es%=88JmH$qXOjCHoll|;NHr9 zSM=Vm&%I5|n9>lvRb>__}qgLNu7*{*1pf=CEbT;DUg+P+m~qgZq_0Xd-A;Jj5J z5Yp+$&{M}SR~iQ{lX4wWcKfT^yh`^gwb~_`gu&z7l$C_HHxv8XoSy|pwQ(fD+&?M+>z8DfDJkuFEy)HXz^NnZ&USw zH6&7>JaJwel53ll-)oMFT1^Qkzbtgko9^TvQFCIFsm+^aor2{Su@vbThb>&+%R*%V z-@)oh2Tx@}>l6OLM@Zt0S9(DOtcsO4!bc#4B|0s0S#JwW#xVjWs!0kxGQ+iEFw|wB zyi_Du-(R(SZdg5XJcIDel-^y^&>jd;5SN*pXX$CkrUcVqiF7-8=9T{dSd@Er`AmBA z;_HA4xTEY|m5PbkS#DJYgm|0{Yn?gXHA^kYeo5e!#q-U0kIK61LO4uBSyzV3 zRp;T#{wJNC8;J6WcKa9rV1xn|r+kzV?<3&5`(4reX4J3v2Y}F$F_G<37C|6@;ZtgR z<%c2`t*aSF-=J-ShzUBc87*-CW`H2W`BGa#Pp|``=)t8lZ0Q{YL2QB}6fG(AzIh%Wi5_4hJ6n) zhILe6WeE}RSYu80{g<`c*fqACFUIn?zkD%EzyI>#(fdjYJah&6CtdCeFW5fGiLJ86o_OiCYH)j!PLteY}nZ_xPPqeEIsi&5X! zX|vb}wtZowEp)$H^UuC6Gt?LnDvI{*U1E6TgB1+XlT|D7 zWJEsz9t%s~G(?7&n0H(z%k>TSwPxgB-amo2b@0DZft#O(WwbmMB2Gy!pRBEt^x0d% z>pdif8E&D8bbzp0>xSkP^Qt6jK(ot#%WFp=;9-4e}WqX$biS-HKH{v{&q%)!%s z$Dk!V5t3W2y^N_D%sY+gnCH%2_)$V}dQ33(%PO{wZyNL;-pn`^wduk)TH^5qMw7KD z4@&c`Y_u|P57uyx+gegQ2i0D2C-8rN>D{_0-WqRL!Y<{v6k(yIPQt15y3I!@IU;PK zD~_G!+erUC(_2`Is(YGt_!9SNbUZ02i(YDmjXMg{ViQ1wq!DB=1*^BoB~oC#%$ur| z!b5KVI~d{s&v*9|JIC5N1a7zePXhsQikf&^}?dxqqA>NlI*9 z9vQhk*^2!)2$5Du}#)RlWkdXD+WmVFf9-0P1#LR`*Z;%(U6hh^pJh`F2&Sr_v zU@7WNYF2L|tZY=_iiL0gmAI&WAtXq1dS5pnkx4=__V=@%n2l3be#2`;XXBNW|IgeOyjWUgJ8bkd%J^!T3?m|@vpj1e#M zo9IVD^mvcn{&JEo^!O7(^G>AscSY~`gE;chb7FprfsU10DG@oWJ7Yw^WT@GSpPh%E zgy~xb=OH{9Yf7Ao$oz;O=@PKe;>U*W8if<}9`>}4AEWPB+2}RwTUmK|DBNw)5J5H( zQwUGqbg$LVO`C%K(wd)p5YaXpM2{g_nu6%S{{vxo%DMDhi`ZQFKRM+8soVbV#@`=D z-@jv${=MnoKM((rBwkA=ZNRu?=yIn@_$vVM@6g`AjR4~O0weww$(Zxo!})ih%RlU_ z{#B#?10BkWIl;zk?(v~n6~vMI_Z8n5W1~?hUBCoaqr#E|ELrRW=dOiN9C5$i=8$1=672#T~>%MpMoV3>KFn0Z<@${%eKd zALougoj?EJvhx4O!#}PoKWjCQ)yI;{6cx1!GvgBC*GF9wVH2|-9mIi3XM?c;V%1OsgVd*1`G9)~$2o70PL%UJ~_)w-6YM{RN& zg{Y|x9u|>=(=OH~LL=vH8%^W`6T<#k)3>fo%ELZ6Ew5$GLj-jyIo%y%Axc8MpEfTjm>nBNqUByjWU(XDEU1Brg2oNU`b)acNhOFSU+6RogHo&;`^_;cjhw)#tL0$d`T*Ch^Ykt&Z{OP%>m}ggI2tl#uGyOH@Rk|wWHeEdqxo@s>xc@f->X%`DjGc zDbe$DPa@okvFavC0=BqY`+e)QG`k)-vni1sU`+^0H=8gmNf(sdn!VRFJXEi}eG8BB zK1Drrdqn2B9Rvw*8=?iCYYToHk7r<_l9vrV7@+T!k*e&ehRF{5tbt5O~M`uR}b z%E`;dWEsnsm4?hUqwl+J-YY&Ze|QozkGc`*)V}$-ZFg{ZA$y~QQuYljr7RC}q7TT$ zQzXFf+6c?T8&HhutWB4o@Cg%d9^W5k(S2LoqH0(`51qhaWF&sh^8SN=Q?m(zLAD4} z+qhn*jQ163Df8u>Q;w;Igx%mXW(%iNFa8JEbZ4$Pb@!hpTBw#;T(W{~#9*dKVL)&XEd)b#1zt&xkJL;4#7$SCF|bovN!-qF zN!`9v{fQ>SmY?-$p3dbi%Om4G?pS?m*a(A`#I06!@NB>8c9R2U zSZ;R}?D5{#w3#%T+SZ|+4(8H)s&(IJ02J6Z=K@Kc!kCwM4lUa@DA13Z`SWCa}?tY~^`BWe{Z??wMEI>+DBX9Sy zt}@{`XOimUIW{&ukR=gZD11NL!tzLnX7?_LZs1Ipt}5JOlXCb{cQqfBR3paJvlwHq zLqf(eIme+IGWIaED(jDihrie$0YfclQvuo7v7IcDNcYtheso*P2g<9tVicwqG-nxc zW`?RF{5Q-ly(&5^(6`C08;Z*ONDH*?mK_I73}lgsJpFjE>t$f_c4`2&F$xO*$EoNM zU)dxdnQuFrh*V75zg$lLYh9;_c_exK=clz&(8I{Y)U{zgnq#UU#)oGw3n^2-ec-r_ zy&3KIKvCevPO2D~ZG3vXi76sEhlaKiAk3)#k5wTUe}273(l0lZ@82`_Kd!RW=~cKV zGs?(0!1KR;;D_>us%4T_Fx?}OWIQ;v}TL%cB`pB zf0UML>H4gHEP;V{nF_0PyYrH&s)dOyX&6sX$o`{HvHBnlb=axgWfjkZ9sXXV&#s7g zW^BHnZ^bus(H3x;Zc${pk)AFkk6emv5^3*bFv0PVCKlPPz}y0SKB-X~(6m=V>Fsn*wiU`5tUUuDYKh7e0y5|BtHC$8rAR;4y)&eO4wj0W|)+`)0l; zp=1fNXmSQeU+1lr;y!682L+2KJ3c&lBw6nM^UwL^eiosu*MC1~b_&#y4+Xf!#lQTu zN!t5N5@O!`5Sh<;44)3w3bGif9h{BQP3dW+9AF+)uQY)Hn;X)IcB|=-us!Hj-$QIH z#}*jvH2Z9+YDlyhH>|Vm2YntOqEa2V0~vavXYL>U{DBTN#@wD4QY$pBA8*Z3*aP92 zS^5!_E^mYQ7fHlJ;lSj_BV4|9$4WncXcQT=U&g%l=rJ$-h8H=id(Ff?^ju#{VMO>g zRCl_^Y_NST(tP50Yc#ngvH#~E_3INLV>Egh7T_MSZRu@;P3$GbAsxCc%xID|>uu)w zw+C9`JkDXU?j31e_2A8yRkBq9dx4YK6EQBOq_Hp}Gq8AS`&vCP^@$)qWq=C!o>Rzf zwFr!${&6h%SV~uh>_*ihA&LKculYa3<^R*+pN1ym5;h}vh3P&U5hc&~2NX3O8i>#FjeksDMs^^RU{ULDC#=8!!3!s9`j^hJ(IC6>)7v!751<2@d(E z43*Ky)o@@X^4ChJ+k!p|aZzGp>SP!xu9pcG(g+{$TwDrkqi9>_Qo4wuxHouMuaF^ZI>dTIL>*?eNB zALW!uOV8p>kI;1dFnICJgsBCVJsg<9d8p2_qhM7`<#jC;Um zXAvdPikh>BSD5XjA-fAQ7C#*iRdTW{GEN!>)Jq*70OwPw(g=l5+_P%bz>+M~2Ur;R6=!xXfxLD;)s*NMY6*&3V)_u+PiPvkB6 z4Jdoh4Yia864hGWQ(m7Wb{POug*s~rWBxN|C=HP+k&x|N74`eBdvtXYx{eF@j8@OW zUTMwNl=IqlLr&nG0tngbEcJ>>=+*3OwStP;jFzg(eRR^O38zutwoK1=Dgn;8hrKTy zbh~X5XAMOt8W$whcmV5)K#BvbgP1a1d8K}u*cgfVXU0(w3_xXb3yxO^_2435>5Jj7 ztXkp8eZ|8Pv2wQbXuT{>q=X&Rb`O8@2(clHwiKL-)8A`T9M@ExnkDf#%=&O~p@8A! zmdGFV&J_tV88|4k@57VgQ15akCO_*p!l$_I?!#=Vc1t)rORjax6H&_Fv26=<`v{D2 zirf)AP5GP`k;lZS?~BjvC@*chKW&%qY};NOudx1R2df=TsI4V%EvRyDe$!=S(2o1V z>j{m1s$>hpP6rkOY zhp6L_7^EA05r28Zl>HgHl;&I(r(M>QFxINq{e`ntsL2l_aN(NerY3$k=>tzXE|>Kh z`&yur_Fmv3KbBmqxD2()Pl-r)wCj}t?9lQNc)J@}oYTfIFaC_D7$%41c5C5OluLIh zCwoWk=bn?k^e;}4cQY?8pfw`7l=Ak4C0W&);xD{5mv_3h?;Cjqu4)6DGT7<{1{ig| zb~0IhE&$0=SM^XJb&%h$ORrbXKBl2GjhnZ0=INswH2mV9V{K2}jwOnhzHIqKm7GpR z=nLb1YE{da#*{!&Mbo%7^Erv=;Znd+7xPoaj7_|@>!>D#m*1B<9Y%|Sg4Nu7+c{$p z2;@zI`rxz{lGvKzudj%yK$?6{HXc&vq(_DrOjPg04yg!ULiHkYywVPXBj$^tY2Qhhx}4`Itp{=h^uVeeVJ{A!)uKuMw6 za4MLC6p#w+02p~jL@8>r){=ThpaQcQzfdAap?Q@mm7ZeFLiVr^P?_377&8-zWe|ES zPu7;jlm_B{2wg5dggzyP2UNeT@70%99lNX<#e0VpP1lJ*g+cMn)TKeYmABd{r5?8- zAKB|mk2cpsv5_$AAmXwVr}&VE0OPcKLB`+a76NW$M4aLmo3ZW z`_^G8G$3C71yRyeN9E?CSb7P&+b%zsmBkztk$v3nvwn^VVq}ze%B`AdlLXwY#bVRl z?ga%o*&Cu)>YaeppLCh@X8VaZzjgrah30hw$%tuShuZuU7mU~`d)xw9(8avZrJr*m z&X;P_i&ZSI_3o&q=QjP*GdxO&Mkg$Xj2ODyAjaXCxTiVF2oa@LdX2^J*E$T8Nk3%g4 zFR$n&EU`^tf$>5S{KD$_p<{Tlsd%*jO)`C#0wgSD&<(ex6lrdhZ6lSypscKM*9&|h zl~V?-%9aaH=2Q-j2r}m25j~PPA@`@;>n&5BKhMI!;t&vc($g@Gu&jpYX#D^LsUkv^ zp=L#`DZbr~r3~)g>CtSc9O1!l=9wPTfebofvlsyl-vt^WNpV(A(?oYAmFiPnW0d&x z<|cs<@fhMmeN2L|JiXSc#w`YQpDbxT1@1GR0Lu}D(!CvobE_9}J#Z!RC976)RD3|< zKC^RPL>Pgjq-@w|h66Sh5@c~Z1`YdTgA0I-O#tVe*H2C1PJ;>hLp&Ec`%wx`uhFHe zo-c*g@khQKl<7xfh}Ghnm7OmmSEO^EM!I+zRnf`Hr--pR+qHd(wX}Ms3p)eyAIZPi zd&XR>G(A2oTUfFlj4l_}%`J;N4Xj+LK0eZE*0{h=j8Mm!b&(TSm9(?LqobID~y z*P-Mk?cC5`jG;PH%OB*m?!vU=Fa+f5`bpRTLPptycJifrP*=s!x@AJqP)+wmRx=D+5vH=9#5M8hvg)ac<5gU7jO&DX*GEXh&y2jV zh2I?NQ+xOietdf&n}VnMoxvx3O-uosb1`4rl3e6KCxNp1+fSE-i9JZ0s`d)znOL7VdicD!N3(2twJ)$*$xLgn1%!z0!*4#9O%58NVi!{-OJj*q# zFnLXCCzM@EGuw1Z&au6@3D)Tcsc*fgaT})5fZ)MYl0{0WH9RqIcS4bcTT*>^zKGR% zT&CA}torM5fc8A7cS}R%xk$vvcxN8 zlQ2vhJgs%|&^ctL%b~LK2f(}WOP6Joka!;aRCVF~$djSHu)K6FdFF|xU7;g44kf@H z;4u?z08gB<@Y;G@G5sY<=H9Jal+3_$m^<&>dcaSDZ)2WP?k!5M)^*H1E$W(NRY9I5 zvMAmP@iMLcY0wa_rF4V*2n~(07|g%AQP-f~3U@S3mXne-*%Np4V4+#{ z_tR_@)KAYdZYPZ`6k0utY5u4{azmzbMD|XMf+>>;97MxZu$31l3o;4Ba#m(`732`W zb5PKXD^A`~)^d(LCGl(s%Zply3~LSLJERTp!>L9rgzkk1N2`1z2+=2^9!NW&9o;No z2Ouz&V376fulZPiZzlUKbo5UxLE~F>OM5I@0?%uO`SzI@97^IWE_oORc5E@RSXp5n z574-qpH*^k$=W>Cc9Gyl4I2sy|^ ztk1SSimivZ%G1ESyuyt2AE_~0Cj`Y+mK(#t!D3pDYAD$>NtYpdR&&JKjvhs^`B!29 z_O?Kvbhni=%=JPb2V9sYO#{E+8ga*w#xwDq03!P)E(Q+IigKKukZnih+X9du$c%VBxXhY#v?iE7SNF2TU6eGXed&dINd$@-rX|DD z?X|O@*lK*Ht%y}8uTizc!i-bv$P&PE=p_Bbm;TE2@`)_}!FEyOL~@Iwq@+>@1}{-| zV(X`drYvh@{-!`#pzG?0G|^*E_0YL_aw?Dn1i;_!#CO7Y_#)CqrTNW1r7~VhvoeNp zm#3F}Qbt8=zh&F-UF0RC95by3Ps}1Q>N-y3IEifL9THjv;fIf)l-B9-Ue+#E`f$su zVu7_Xr`?ju*$G*(X_}y5o>0n$t~fGLOnVwylUt$i9&&z{oF=tdndRmI6@^J)`l5m^ z4k*`F<{XBApO1Pu6WS25KBJ{D$2EfCtSplJSHyet1W4!bsI|50ZPIQBfp;tp?I)Xr zx=M4owNr0Q@MAkVpp6LEp1RA1(02|CLxMXQe&MijM7#^O#yq$v!v{ht4$d%m<{~6@ z-HQemWf2IaL4Jy1LcVx1jEDq=&Jt%5uI|z{a6s#X4$f(?z_pi0OTmdCq7FD*w+ivf z@Rm%>CL(k|8?eHYSIH3vgd^x0&lzZbBSalHT|=>eFTjx)CS1c|q=_+RJ} z5De57Wu-IfuYOv~s0YQ)i5ocdIq6fYf)FmN5RgI?GIy5~F}wrO%|=knY~KXJeIEhS zReN`pC#Vv3mhB9Q_7nW2o_QUHLWdrPirQZa3xo4`d-lvib;d>|N2r(AEh=+qt>NWyrO}s~`B(V$oQe zC2U~a_MpA2<3Dc_+V_@plT{>`0Y|)4cM`XAaNz3kpmXNiDMKmor=t8vZ*I`ai;%*c4;iiRoXmFq5(|on$B~ z`qiF2^fTO`jQ=)yanF9EJgg8({oOCpSM5xB_PTdVgqwpZ2?+|JD0f7{d&%j{iw8+K;y-`6SonzMN#I-{zS49kU%!Y&7Seh7 znz5cedg-1~LQEPZz5GP!v5?_`)o_TCQw{%ZB~J!8%PEy)_^v_=+Slm8c+Sk$r9 z*+({0wLfoP3;qdKj~@XeRd&Fr)^+~REz`(C=VQL`3;(5M1Ui2rDJ%i|hJLyD)xqFD z*`NR2Y2lau{xk^ccktCu_U|HwqWwBJgb>8m6j^69E5{0mL$reCzN>F|h$FGRm%qpm zhd7@9Tl9Z>;Ez3E@cIAH4E(35Z4pLTm0c3JJ#@jC3ybTxPmjJBr%?%uy~c<_Tcp_o z!)s=-!n15x%{>SFyzI&N(3r0O2tQJfILynzKM=-$9Z$p*jNpJKRb=LluTr1xWgaQY z%JWe(=;1>>h|1^Z@9%Git$pa%6LV|Q&k&clDX#10Eb!cfVunpfQ?s&wbsnL7j#)=Q z)=c#^pL4@w&z}TfelPS zY4vdrd*Jg_kfC_|!Ga=&37&;0T9sa1duN=Uc@AyTO?yj*X8)nPQ)^~ zR{AOu_o8bFUXw_tEB>TgQK^&2)l+R*=OHC%)e?c;iK;jmODkP7J?KL98ST&$A$GrjP?|GEE3 zjtc+%pT=+diN9>=pKdh%QB(O#73RNQ5dGx#;FL1?t)YIp_?g(V=O-`AMyYS|=q~0r zMYx_k`)6@EWz4t9C$}dz*u%)7*01_zY2aW_3zaF zFK+(-N-X-DD>3rA0dkTTd(Hjs@tykDU!L(})m_G_9cg1Hc%H=?%ZCzWWUtLgRPr3G zRV9wKA?)x4O;Yu_piBR;sxnX6PKp^>yawap4n(>IC*|Ju_e9y=h4*%9Rl$wuce$Vy zOfGn|1pqIJVr0GNGMWqnqxg~SY{S8=Z63p-yVICUctb(q+t`tqS`L0bBiH6t-^YT( zG-fU|#w9tXPMzsz+^?}?M?418QbDxW0Dea53Fn&HJra2v8yBGETegtNmdpvR4F`J` z7TVafc3|qp!xxSEIkri4qu`<^gL=2_)Wu$C_v7sjk3BF}KSUy>Z$%U|=@)|Osz}yY z)QXJ)7rLj^!m56r$P_-`g|SY zoKjIJ&oq;!syVD0`>nRp66m2L76>Ns;Mm~ccWBHuJ&#VrSd)s2<8*T|y_eovp~vE6 z7+J8*b1rW*nulB=j#{(3z<`?jv0Hpm?Eeb>w@dyX42DFKi^79TokX?8ePleLRH>St zE7JxVLH{EZJ4`_+gDtMpW+k3b%-|6P)@l5*Ao&)P_PyV3#(B11;fH(mzyGT-yyq=_ z>*1YSX5d3tZTBxqt6vDyd>3q0bv)Z;N%?tv=bfw`z0esw`vFk+0mvo&u8?tByw6im zyew9#-Dy}7GsWNd*82)kIiC- zTH#-AT(c`-R%Kc*B-G;%makkh0DvRfRitH^$2&F}#dfa-JYEe|spvQbzwY_Q75xLi z77Y{V^1)ri;&8v<{{OnW(x|4AZ5`W2Z4e?&1A+nyVy-f&Ad^5;q(KpaOaURrHfY_cfIxg)T*=2 zUVGP%T24B-LF<-c6>{?s26SRj+v{Vl-{%M=5tzYw zIEH%`9~5E82c6p?3LZEfeNHo3!}knbZqLKKqFnT&GcYB@w4;fmY^}a*4ORrwW<}#6 zjd3Q@T09nw588eP-GDG3{l$&@)zb?8sQEZ;pQrDwI@8CxC`twmypNFeEe;;G&Q51v z9BkVnz;N$!_@Gp1zA(Vp5P;d@T$9^L>Cy9Vd6{)5<7=DMz#Hb>;)4=_h*rNxWc*z~ zr&z7p#9pg^wI88lI5k`slMhlWZu{@f6WN!(#^r;)27XZeb!eyl)h^@d(3G}$HZFw( z@XCq+!bg?A#Rx2LVxg3e?+$Ct4oZDxR71gvrDx5dPKUustR2ET2Z#7KEm|_qJH(|4bbuLDxvC2+T#gY;<0;dq5 z4MuseCY!F!P5|t<&zQIsHy-(84i(mC+~_#2HLEi#uU3tfmp`V$2jw8wtMa(O3?Sbb zkl=#`Dt7E|@H+S)dL^bsaR&iI$2xpUal2yl+2bEyPcKd#_zA%v)&pCT)*AV74aOXDDZqhJL$dy(HF(S%{8*a2`O&yV9C?~j0?euP!-BcT^ zrVHg%XceoyWd3qCRw%mjDR+6Lf2mIeUU(w*=?U*cddU&FD`1&2gZ)QGROc?<%gK%F z9@XuY|8)=3%jC_r%H)>+i&*=~GJ=-pueBBT{GI#1+dUHDz>wh`cV)So zWtgqCG^>X#%1uPvRLj;|uM+8Q^!4z0`9&@UMOU`}trHWP)DxXq+wF#FjX^ zm`uN+qi*#yZjb<-5qSMrHbC2tZCJA|vm=ajF!jq)s@0n}v(PmAaRn&yPJRgbX=P@P z8Z+4I`*VrrRFq@yPLnGe%fl}=wYOA!vxl_dw0dqJaFYN2|F3 z<6e$ZkKe-lXBtusbdc&5*|1<1I=+`<+O2J}6R4wevgm#-D(-^r*k~ z!W=HISb%CRV998`vGkYZs;!x+y{- zJ405_2CRmuLh1dOvF^#(8!yv-Ks`MC|45#_kf~t{T=d?2qG}8#%I_OIT2=I1#)9=#0OX^+61950-oE3c+qkHxpLuBvbB3o=2au8B>8b!(oVnaw zy2q{#k4Pz9ZIRr$4rN4<1~}THqQ2v~^2a81W%(dw2VOs9OO;xWG@M$9!i~=J7A7P+ z*}fal1KeA_;`FE}44G#IEcBIxT@OTor+YfC8w+ItE3DhHYzT)GkU!HV+Ak^^m_7&` z6MJw4fmdQlaw{G@vcy9I?+LEEJC+47(YQqw(A2t@0y40Z@0tK{>0tlMUnYv+wVdv% zA}`^+FNDu(W)H2A$aCi$=^royuY9`Sp4skfet3WnvM+NqcwD7hly;M0y3*lE?y297 z8GsMN9ETh+NTUa3$#c%dnoZBbFa_gcN#jbm;4J(2phJLv7@S0Hd%q23uwg5MH$0Go zzZ7mNPpy52qYbf@NxZTY-f=aqF1EGMaJy|kAZdA(n|4}-!B+aOUrdb6A49-w2AYSh z1%Y?Jj8wycO3kyDo2UY?kRl&cXn(}Xa4RC;gC@P}eIk$|yr~NPh4KpgPOj2!rdIdl zt>-iHs^{85cpn0t>V_xWF(gfDNNth`Sqyx8Eui`EBoQY<{Dxg&YCk>15P<*Yki*MD082*N;_OO)-B#0MUbCLYH3qmfwU)O zr@~}yqr*u|)UMOZqEs-nm`a;|q;xm)oupxu?Xw9Xv@2!(&wZu?SX^B=)Q;lX=KHf=pXZyT_1^T=pgo}F*xRh zFFMG;Yn4j1$?%P~qF9 z4(Y&QoV~XJ(0J1b*I+5_RqdPc&9lC>a2}>7p8i}h}Ca$<%&Q((~RdZ z$nAo@vdtjKMzr-{p$!n0;L_SuGHDAo`V@HwK^K*Lt(PK0Yv-Wd_#h|fnBy>}JOkRP zImJSA3;N&_She-%xGBlvobCJa@{9&RgtHl=&$AnO{NAG;D z6exdBdGFnBE-i$JnuuHs&CrmTiOhbleW&Q`w`ML$Pi^*{k|ymFP`|vEV>|FTyX2OD z#h+yNfAs!V^B1VWl_((UkIcB%oF$2zF+RDbV)umj`TjT|POr+}UGK4Qtwf;}rv9nE z8Io=LQAXWx;yeZ@pZY8!Rr_$gpMlm^*sGr_ZQ4g!R~fMAGB&nSY^WX5@13n_=ybI^ zvwAUZ*8J+q0j(1^n;Ro@0~OrriAG1g3HvV{bW2o>Z#CYR5b)d4{x-o6t_x#nxegbA z=OMGM#|tAXLJWOOI-%%PeEzo{hd+Q%9LZJI%qTA@3ha)VrOuaR7fmg+PqnoEmGpr4Iw zgx{p9^b>6Qt}k{^&mF0Ze@LLy*Btu`_wRMDof~Ug7pkT9qEwX*bgM$&5pt}GMFKf= zk{k^?wR{bC!FxaZ7J*HCWzsq{%*0h0STO5QgPA;^B`~waW3vF7*%-`KbKpALoq)jy zdB5#ks+NsrY!EDP1q0xoA$ky35z;Jj4sZP|XJ9xZqLShVbLo=Q~S&{Jn;2v8{J z>39?dE|AvgMac}!$Vl}c^yx=Y|Ck-yTF(|*bb~(huz*eTw+7uMu1@{omHM@GJ}BPn z32QQplO-qD++ra2dHRYGts&&frCQ8SJj6}jD0F?i^)b=_bKA4r?dR<-K+I}^DZ<*W zNC1uwC};VgkDDomQLMS~VlVRy9^u|@LXaE6Dc}KIk@hYaey_6_LKz-mLMdz8+=3Xn zwc|a5qJh;!0Oz>ti{X-)I4{_Wc%e#ruGc<79)y&*E8y+N=y3@sT1<76EB8j`gWVpbevZaoAgm4^js{0SZ7)d0|6m X*@`JazA)f{jp_8H#g|<)m_PbY?x+6Z literal 0 HcmV?d00001 diff --git a/.github/pr-assets/customer-usage-detail/final-report.png b/.github/pr-assets/customer-usage-detail/final-report.png new file mode 100644 index 0000000000000000000000000000000000000000..7118992dfdc8300bf9fb80fe09c365ec76d88743 GIT binary patch literal 62798 zcmeFZ2UJtr)-W26Vs8i$rFjlb2u+ZHfavi+2pvNbY5*nFgn)Di#bZMW5)2TKP?S(Y z5;_D3MUNsa2nZzfBGS7cML^+k&bjyf_uTt`WBlKH_xca z>ASq!4>-PmuO0t>e!z9WR@VQ3`>%KVmlySA5T`yEM+`Tf*bWutyP>$PEyz%U%QYS0 zMue?2e zKac-c=GJZ(}ZYRE={#~`cJUp*LR)`IF_X-pCITY7G!6H+tF;Yl0Q%xmxHneLhHow zQ%Cxm*7tL+Rxh|%zE^MRw>vd(1o^s~lI68zwsgU-!s5%k&Mx41R}zq_65~)S!@!$I z5JO*A>v)VJuUUrW-Vn&P83aJun1uBS2skR6?MQatS5Sc&lzqP$IXW#EIo*F9WbON z82KrRc%5>3#jSi9SKe0V8!oT-g|m_AA*N-phP4uAgYUd1O^RGB?mm&fQaT~6P8j42 zN5hZS(U-hp48DEmnc?9deRl!+11A`Y_u@S~WF=GFZwHu|lEo){M^&F%PKgEwxJ50E zeEf55z{MYQwYiF@#4!zT9aeVx4mhmLd}-F-3t#jrh2-%-j;7CB)}G7 z!+y3=k5G)|Pu^PbtWy_u0Z?hZtbwbHyQ}lNfVUTv-Mq~TE(D<(ts7qT1@JB>=(?`7 zl!LeVOP&obsFkfDRgsM*q&A}EXw$?j0=1bN^eqX8!4=fs97Z*L%8f?^@IzBo^$UOk zSAClAO6wZ#0*sKmfUN-Nqg}vu*45X>t?El*@y!8s9LRk^Kgq%lzzDD2cc=s8P*1J2N{9;*v|9bd$BEI6H8T+;VNKu59e0d?~5i$~_rQQc~n_g3d zdP{j?9|rn8RlWu-(QN8w6dur>^1fKDo+399>%3s*MR~IB;heh*SaOsqxckQHLflWQ zYP)~{_5}U%^3@UGK;OVnR}-O2<$|}(+&YV;m2>f@Gtgx#ce5TgEm`Ib(F!G6}t@PE_f|-cvz%3fu7ymo@6Cyc2BX>6StMd zzx{kNt>bdL$t-ec+e&`mh5Hkc;f0|uJ14eI3{+2j6Is35<*^09=C@@-8H&fWr0BxT z6m7@)ywORvRSKgS}E8qT$MqlNS?*kv{GgIrkvhh8#wu%XsO~Z;2g=o_C zaX&O^fpZt(kZH2wBlvXt#=KL#q2z5pCw+N4QHnjCo?yZj2%_VM;_t+MAFrH9`1&zk zkO4Rh02~0luKsTPjIpz{w%@KlCMTTYZCi7)qJ|mrCGEGYSMjb^85evBRQkLU)=Xs4i_S~J$V1!)W4$7FfoV&YI;KkUe z6*0r>ubXPmFAsUN?eXmH`B}Btn??3B%H4RoH1=Y+qA|D_#lyxEfk2@+DwRqrYJIaa zLwm6)ybGA^Z{M3;?#^9ZJ$!vgElzhO^Yh9MdFQA=?zgM2v+v%2cyoGqk$YAiwINcE zQuDv0frvL+P{jCtWqj6~gV%&4C7GUvdFFFQMY4(q3ZEec@9_L=&VZ-mbhP?=m`J@7 zH0SjxH}!Il(IR_EM*6|b@kbr6ee4MbL)aCiA)6(9iWHt%6ha9g; zo6KsKlzTp@^n6Itej{~W-ZmLUE!=Nkyg*j`v^HKs;oY4PP;EiX-PZ+i@D+J@BLgb^481J8w7u-0J z_2RZ-Kt*UxIVPGlOgWl^G!$fUVR{pf+C!1RzOEP)@7hI?1K`A2)70wn4SagJVv(J& z_gq$}n10+Voq^}f>|n4ED7oEH0_L6hX5~9&DT3MWE|2l{_qT)W6e(UUThm~7LZYNC zSDiBqc)9him*&5GVTk~NqAQ->3D|Ebrgtg^C96>G9InW<3xfYD z|4Q?Q;J8oxqAb6PhRC=-D;jEQjC8c_&Kv<=h6TCTklTDS;W~z?$!ME0OL+g172f!1 zaTgbX$Tu-3!{}<5ZmI`zGXE`=W~q~Xk6bqVBm0_ris8zOK0(~8S zwsoVjl;3hTvoVhT{OatKy(yNsq*yg-2s_Ht!FaB9NK_dDhkY+^5Bn~gWImh2oj{^0 zqB-qIpkP^3+mo4sQ#5PaJP1@>on>KCEnBmRzUO0%zjL+Pv%o^#E+Iedb^(>3aLxDu zRlnAjxO$0ob$*IaJ<#h4vVlprV6Y$Ka*Q~#&u0zdpzi4IG$Gk{D9zv%1S|^I%?PG@ zsB%==4kW>7*DlYq*v_Do` zMBYNRU_Pbh9+ht}b)&*aRn2kr?Tu}yoM7_c3l|YfV42CBq z7frEc1z?$3vt#|Kr@P$?=It!oVflexNBoCc&u*wnNRD$fFQmH)OVR>ch zc$%fv64NWL^F1B03VDor=lq=e!diqa3jVNnTh6fmW66{=^=Y7Q zQ|AL!nQyL3e6s60Qg`*TmSvs1_;eao4>P4(6#Zx(1qw6uM!UF~*1}VnuXm^h=R>)6v>fJ_Ux`~J!JSIm29=VRhLYp*s` z#}!T*KC19P$k5E4WPm^}K&>~2X8q2z9M{t5go;;4isUy@$oP2HFMhf=SJFCm+GyB5 z)$Z$da)XUvIvsW;WRMqpTOjtNn9CEkba_5R>zbuPhvX+Ox=6n2y9I}Ln%+MRgDHLa z&GdY3GGak*{%U8kq!SJm;7Pa3bZ3z56-d>_!2Gs+*;K3dZyC2?LZS)n^S-YV#}qVT zl&ZMDH2;)#ZTy+U#pmx=J}~RGR)}+>Zbo*VzPHVM0#Y3}5+;Of$TtB8m@?Dc_%_lj zWE3%CQLQEIVH>|Tf`_a27Q?+AG1v~!JIVMsLGr2g&3iI@^a&%nDo9)mYR*A|cvgLA z1FU!td)_b8>DrZk#vr>Xckg9dXmub+z$_|{S#eW~7+DhHZ@izp!mQM!3`FYl5Oo-P zV%9|Cy~-}Z_bvUs>rL;%yr5vXY%Uve-SqA)`U_~u>N7(_Y02k>tG*z{_cePIOeH<| zEn4O`C)?w|T>w_0`{;9zMRJtj3KgVsZVCjp5>7huMl-od0yP_5ggr6#Z7 zvl=*IJE@fDR($CNP}kO}(>1*US}~wu<3k{%eMe{q`!0kYm&$R>kmtSK0-Y>fiWK z5`}KJBPnBn6ssPq8+?hQa51&HgX?K%Kh(HBvy|T20RE}B%+b*?Fg+3*U@DOLsI($W zK0t>t;Vr`$BPiQE=-VDgY*Y-;VPq96M5{LJ0u>YF<2NAX^(i$(nRPs}RV!!hj@Jjo zFC2G!ld{#%l55wGeq*yx$=A);*GjJ2NBS z$94gm2}S&QbOJHamXhQiDZ?C|2aO6)Mz5sZEwk&qHnn|9e@5rBh1YVr&9};_q6vTh z`A2bea8px3uO4k$3@TbUe7r5IagwcOM*KcyFwF|Tp6$KU$*mQkVtMUXI;y_-qhGu$ zKD;Wd^_(f^M9&GVbiwEl-{!zVl!T(tG_3#9qm@>;DO`})v;Sp`ZItkLZd-Qy9~-t3<(~0s{$2NWjIKvTr%I6#J?%1R^to9LV>FyV zdj`EV(4uo7@T}+es#lS6_L8DsCCIV9jLq2shcF-V+p&9JzNQJYlaY;^5rrSW>9=#t z-eY347AL2~(>{gI_3O}>KkX2MI5f*ag}B=GY)>7K6tZmr*3=^rDrBLg^CEfiae9bE zuco?w9F;LXd2I^uqz4?G`0m{pO$WnlXWoy%PP=7Bb}`Kc^=ZvtyVu)x0p|-H8fhcf zR%V74FZJg-B5%g}N@UtLbt#9YR$^__FpOL@U70LrXKor)Zs=`G?%Ln&+SH?cvCjpH(6smiu~v#Y-bj&86;NbIMkp(ZGqd`-wTPlB@Y2ev&>;ka*( zYWG0bj`UVa->AWbcsEu9IPJ-);}eX=&y>aoOSETqodEoJE2)A_W9 zDWrAxplpieLVy}Kz*z9IvO~@;p!rfz;EQI<796mk!;6ufm`>pKlq7;|)FfeB0dDD5 ztq{&~Td3*0$mXKVYMWnIS7G1AMP+k0a{sr7^SF{JLr0^Q#|GKS@r?|c9K8T40~gD^ zQ@C=hIDp3k`jw2;rbCUsAx6QPzLld3Iw+Cv()Gir=GTj<`C{pArsHsDdLCIlH`%cR z1d^afV6HCsWRfaTzF!7C?W~H_#pt#D1r_)ZDQ1PH*csNA?Ko7DB>rO9WKWGxxv4oj z793++n&uZCzP(DWGl9VOt8?sJ1`O0QW;FVXI#<;h*WV9KSS_w^w!CX&=T0)M2y$De zuI1xd-SB{p+$Rq^oZa}m%4QLWreSUN-TK!SzjA0bL3>*p(Gi)!ixazmv)_#X7yl(k zrhqgk++!|b3){f%o! zDRSL%HuNe~@;qYl`7C(OJ6_8UC+MFbV$8Y;hoZ}z76Ge0sO9mTN6O$?Dh`#KWMi7J9ibNu<`dvTn0{pv(qiGDJz>$aJ2WGFIQl4@A zn4uM0*Ro~xRTgrpkyVIcEkMD|k02)0j4i<2hYzrUPuA0N9y4epH$PI%3A8?XdX%p` zG93|@PN2x~x8XHOU?zP~l&v>e8&31w*PT_-0c%w&OYr|}tU3v;7qBgK=mA~4Fl)FT zik=ZO+71w*JA5C@uoljG}Uk&-kzN$1V=SpCLH6PO^m= zw)m!vfIE<@A#KUlMDzP;`6FdsJ!QxJ%Nge{80-)mv`GCgt-rON|Csx{k#Ek-D9C89 zc<(&Pq$R2aDCY^stPBoG%q zE|JK8wmNrF!#~eIq2sq{ zN!p`l)u}iqNvkc<7YCGk3S@_B^sp;jdxJFN{y+_E|K54vjp?7nd83)S_T!&VI$;3* z+np%Tm#tafy;si<>;Lk?OTcw0v@`noY72@$NXx|v+MRIAN+ehh+?+DS&<@EINx(iz*ZthCEd+^AD3Bvvs>#Bvh#-O zZwoN*bHop|xq-g{0Q+g+a*7>UCp4MxoWH3@<4BqzQK#l)Am>~@SH+@hb_kk^K@|e` zw(yes_N34!DoO;&@0TS13dGz6Tz&?)Ly?_|S~eAtikf+>GQEc%Rl{mVB`1ek0bTNrSBdA z1pc@__@lZY;Qxosj!Z_~BAw9(F|8uA<~5f-E!lj!0s9(q*Ju}TP%U1qJ55v7?rU{- z@H6k7BS#93+nqx8n>=vud^80|5N4Jf9UcbVffD= zS!oeXeBcBY0K>8hSxWe!}WuErf=@M`oAgJt-j%vr+o~E zbJ?MzZGaZ%B)#aah{+i5Z#{?8GP#~1NnE#>hzF`$PnQcnZ0?w^zAVlhB+Y+se&5dL zp9gWAY23WB3+Rt*-4Uz2J9VjBxLUeyTHzm&58K`aU_>;&29E9mQtMc_U4UV0N;;j? z)3=!LjD4qsWh3(flLPbFbiS<<{kX=}5ta_#I{pU)T_(EvgJV8oLmq-a`h!pKM2idg zb^#QSFTz2V7R;gjNv+Gux$VN=j{+|Kw~`Epg%=&WfSpElP4Pk68+>f1Px$6tt8dTv zpraGt<-eWllKwwu{Wh9?;hi@`P7H`MBe67=S zcv^{jW`HpPu@2CYB6`Xog;(eF#vbXxuVA$G8tQKsb%5K#eR&qr*K$U;o=-9d>8+RE zMclpYCF~k5HPVqI0iGdLZ-v%x->pn!mx5! zEcn)Wj&P;?G<|kP%219LQ_`>sF*Wsd5sED~7a16+^W3qOB5lJVrBTxN4=yq2an4EK zsor3I)N_Z@$PW$QU)JJ87ha>?PPT9{?8pneUKRa_<1?1}Q;$&m z#R@TyNSbtjDNaX$u<4#{;D#w~?VP}93V@zFoHUb5xcbXd;OlpZTSxJnM)L~wDk)l; z_`C&ILWQN|b#00o@#@?so<#-C(ljWar8?txKRNIf@sZdGc>ns_GJ^sqZB0FgyD!AU z^kvM04z-6mT9z}uXJ;oahh+6@htfy;D6+HJ2#OS%pU~}M5fHkT$=gbLd}iB>Jacel zByrCF{+JuvV%6YN9;xUZQ%tK?!@UYc zoD{cQA@_^9EF?~}2rB)mD!ccdT>8{6@wNH>Pr1>LB?_-fmZWO&{JBq=rHWgx*xduy zx_JYAa2?OwqBm?S{T>~$7d5$0CQxIMcNh-@n^&#XP87ZH@=(WmxMgq4EfR@VH?sw# zY#&CzjYc{7Nc~xqMELxzFWan!X7M)*BI|obB3@iJP(CMgb|RT%Fw;+2dktzP1AX5i zOz!%V&XzLLcZ~jKXuv~{o7jl2jdxRBtUd-`NO&^qsck5hWBhLQtU*4JLZDVHA&)j& zlxRBR4c_^-fI=$KN#`7nyd!c^sMUhUxTOkVe9ruW_JYn?LwcW$c4oxpjJQbt?84zZ z{2i!|-fU%#_e)vO$HEL+UM8ue3!&dfCDo)i%D~`XAiqq#7t)U5Xo;BW;D*X62IZK0 z>&a4ux!DlX?JpxDx6u`yl4UuM1JH-r%|#1>R-mTxoiUq`D#)Eu+i3=UOA%3|*Sw`# zyEL>}!>dV6B?n79H5u;leKnw1w)O?ttrznW?ac5DA$?|z1fUuw27|{Qi=LN4ARuPO z?>1Yo&a`r?O{07l7o^xCqutQTIH$mp+0kl0_ncCcOMDy3ev1ZKan!p^5B8y?sRg;2 zl{ZoDABl1W@sh*!{`f>KOP1u2rg&}vzQF2g<&W=LmlA}Gtn5c?FH6b z=$L+MPMcI&i3QittWRDenyKUpY{#m}+tyqu3OzK+NuV}XosQ%mT`kdJ%Jqn#x4wZ*-{_j*WKHAB@u`CDpM zIcAGJWyl0;Udg%Hv96zJER?X_8khcR6z&~fK&gh{;IqI|g_7bn2%eFXh1v7TXr|#z z!{K!G{8GoX8wc!Qq9M8I!5Lxp!r(zI_mXLA2Alib~e+XVf1_=bBRTDWm(9fB~1) z{i790P3w{#iezhS<%qj|{&_EC?U#gL)29vOMYjMS|N50-pUFPishk|y1FU_32)p>1{Hb+syW zO{{7eI4ryHp~Vi-qm`o2`81oTp=zlE#`sRa{8Dc@xo)(u+K3&$Yz>fg(!;Y=RI}PE-~bc(nB)8f^pS01y~ta z`R#X*Aps-PUq9y^=+NvRHxyXYWnrlwV^xBps)*Gym``QG8J@*W&1g@OW?{@U5ldl1 zAmh2TN4V*)d{ML(C-b(n4`nYL@tNQ2nXcT&@<|8L3DTMcN z?~tFK_S>7T=9 zbozGzu6^3}v&03(PPtk6yw$nmU7UKul%4lgSy)TcGo?m`k2so_7&r$b%jIi*i2edH zE55!Yq2%T@L+7AvM9KRpsAT0P=b1|Cmq+;cGhjz((XWR;{Go#X`~LPT{B7O&=t%YQ z@$X7IHwQQwW1r{OsyT-ehDsk}Q>1ZV@+!AX?U;6NlkHd1Vv;dZbqZ7<*86-`=U1+J zX7gH#yH~S+qH)Ztsj5LkBrY#5(J*sOJkL?{`Zei$h1QF{^FvE$b%(w%m&HCSoww&P zE_h=Fk7M8xX*_82J+hNM_E7I5T1h*crBplwQYpn41xzLBHk#Cq z8U@09<~4*zWU4iF2wj+*`#B>g7?4B)MmVWXP01IeX`yFtRKS@6v6~OqBzdmsq?K=7 zMA1dWu#5bN8+MWx32r$_Aj4D!=neuk?yvtqsRJkeVdQEm!Y+{7o5#${yYB{rn>^zb zzuUkY46n0=j3nHHhJsQFHK&(MCj>Ae9`pej^`0HZn}F$KUe+%<(@uW>(I$ve6x+@2 zpq-nW)3`q0?l$0A0qgpnVJ^*QH%HAQ6wg?czSeH6GMV7aGn7J7omU)7;~edM{p5qZ zm@|tN+3+n9Mkw#G7P4=wFP%DgLi|fbHP$TUhTqF@a`%Achwo4OZC*yKIO7|-*$%|f zY!`2-vBqA@r3pz7b*Fd?uE)7z(EIEbN7&Ygg10#@QUyoa7Dg5LCFzU3VJ6+t`$T)$ zUKD7UtK{!Z46kpP9H^O&E3nKEZIqy{$)LSYy`Ko^H3oLut&&$)jOIFTbcRdhoE1dG zyhnU$l<`ZL6WafNHY^*5JIGm(DzQ+akGZwb z44hPu?OkcRUj`QJ!N_JsvR7DHQJ%`4Edx&lGbS94JZuXdyLk8;rSELloeXCBoPF2$ zeIo*a)6;r+LmApYNo(e}P36C`mSW$3|5!ewWWY)xkjs0^bl_n|@s*&s#`bGw z?wJIplUIsd>BhFBK6PXS%aDu%VJfSx3u1J7No^5826ugBqP#co3nuMkGmdXaQMqUV z>?ivvgBCNRBfchDtm$k8GJNyO*Tcl^-rU*fJ)>v`1eXPsR+KtUixOwsAIE@IYs2_Y zn6-@8xhZT=FkAYT?1Z>$7%e$W`j)N6Yppavqbl1PtCjZSXB(3i)H`vlb4@uPaFT_B z>eG^O1{p3u?LMt?Epm)~P0u7_Oq4(PVnt;8#3|49Sl29<6N-~Oorbu)c5|!Tjf7|-qX6shwM@HQS$3%BQ{kK zj*4>_5wVl}=F5d7>;iI|llFbh+_!|7y?V3Q>cM#da;14I%zl&c!ef5D(bg0A0XdL- z*)?*2kIvWx%X|`@Hg6|Pm{YEIS~Wl7jFhob;BMJ*IXu+Df{`B2x63$I6BYHnM_J6? zh!IUZ-yIcgYGM`FqLZF^J9oA~%?>QZ66)z_1y6cN*mEF9XZF;h+kJjXu^yb1rd6S1 zk6Kp{-TgC;0l62!psf(!Ee#|W6b*g9co*K={ zAmpVjS&Yfqi8nFYh*uVc%T;t<+7A;_JhOZ(bGlEpPP>R07Gk1t#Z0vF;{I%7#W>n6 zPvqH!sLV{tl-Te%OEE7F1j&VZWswRkS%t`tlah81^(;5rESGhySIEWm;BsGLy zAUGtWmK#&tB9uDG6p!&sNw8c~b}af<7NYmcb~=BxDEs)+4F48^oE&T#)j4qeqB6?M ziop!^>3yv0p&mEI>ax3)o}R2F6)wDINlTN&+7=-7Gws?5J1)`Uyq(8Z{W&AvfroV3 zsNouBSY)uuLZ~(41x4rb*3GS#%coXdS8#SEXE`mrEg-E5W!()m8~^h1k+ftdBSWJ{`2qD#c1d1tXYh(Z|9p+7;B6#$T{Vrd4cGq)~(myWpAcBMtJjf0W-h+s|I}`8@{m| z{GpA8%T2DberK;BNM7@>jLpr;*PosdEXnis3l-yzf9o)bHKHwjT7Ef$`2Nj7O+`gu zcGW!%FSELe&JR3lbhHA=oCVDxtF#~preyzc!y5Ryl&%u=PvHKHh#h6&!%t#$QFDVS zJl?IphMx^zvd5;fRPpM$nBsf74K)Z|>x>$u^8XJjb#fmsM&@S1j+f z5Z1CzIS!*TW-v1Rsx?+$EQbCOnC)$y-=6}5!DX}lf-!54h!GMJZil1NLB{_`_G2pP zEf=0ViF&A~w|?fI`?BXWLOT6>XF?7CWYYg&07}M>s6(3(|C1ws5>=rg^b(5|ir}#A zbX`kKO^67hVZlnnh6c+vp-s=xHTzpzRX2{naD7wCBsy*kj;u!3|06q&-Fc3|4rzG4 z{(7?gZo}lBYy2^raw9_Yt-cA!tljU`>;ADi-lsNu zG)T_do!SMQVC@3Np{vS{x~MIf_hOz-E2cNWVZ>e zG=3xA`u7HB{ohvZ_C@tGzst1RxUhT5t=@=GhEB6YH{{6z_dztTkhSwiXJjG?A#%57 zz{g|Sh+^)oW27q@7N1QDkFSXbL2c~HDk#FHMYL@>Qq$(`iu}u~ZL2ZO4FUEi>rfM& z&mGRsDx5Ni)HfB7j%u17=K7)1+-nlXLtb$lmhQBBm)lP73=}u!oE;2O&h9vK2a3Tn zj33TnWPE;Gy$`tB&iP&+j~K`^w9%2NoXCqa5Z=&abZZvTo)aozEP zdzQf7fcuYhQ;nXGX9PZB18(Qi8!fXM$8E(Qo>Ft!p%S#8s&8~!@XSN$QT%1`h-MHt zAL-}hv>c?sV1Q5emy2&JesPgo3_1us&_|BH4a|q22YDzEgErI#3_dDy_^P*)dU7eQ z2E2kan_gNm7s(*SVt?#>J^vMH3Su;{uLLGng)?8bilxsm`UhOvW)l>1hO|Qkz-+b^ z<}KS_34Vn5!{e+6t>rrgEf}T5F z3u4Z8Ixrh)mfVuD+Huc2+K5wF@q9evbsMCkzUI1$Zs(2MO?k=~yf`61$d;E+fg0qK zS`ofrar$ErV-`hS9<4jVi}(8RFY@7E7$;?EJ99Rho7*d1yCH(*AzWkt4Q$cmpOrsx4hL-jE!~B=txQ;A_?EeKhP?l zpr<$1I6ypD_Rhq#W$5j^rx$LHS6z^!bA73r>#cDvUwuaYBGKktJ=S0ENr5lBiL1w< z`XT1DRyF!4F*U zW9RCE1J1rIU+jM(R^6qBtwHMFZmJ?{QUg~U?{($Z4!BRxVqEhd1eoru*{=E)hXMr( zdokQ@=G&@1y)v)+TAH9zn_4FpY@T#(lB!g=B;y-Y|paOiz+dVN7!(o0))EfM}k{@~uj|sBn^o#NaP_wCF@< zBd;AraI&r7C#2QLK#LMtw5fS$&*3Nj3Mn@*ue{InST_${7{*Aj-~*5JP@!f8gEl#@ zJ%>zBC_HIS%?|}D@pGb6R&QJs_+jTF9XR-!jnm7W;d9d3Cdc zJq4`BEUlTY9DkGdYu-KBJWC$UsZIw1(Ipq(*bj5~CSygRN=O7SGK>Q_N2eu2VV`Sf z#|PgJosnSO8Sk2RC>yOm9>MLbVrV&T7q2ZB&rW%{4w;`BdypBJvtvAU@iMk5@%+0Z$;Js|hD4A7UktH_S}vS#0msC<8N*?VkJzLgOd!WmU* z?0vmIfyI6HzSo6uFOCh%M`!8zGFJ3-+oq>MbnFbBF&|TjDCwvUll|HgIAyd`0g@5Q zhsPFuN~Dy9clVZeqp)IL*;zB7bqQUW1-J#1EOdS>aUyJ_sq&4tLAd>8kd4{a6T6$M zElyTc&F227RO3QA@6V zQx+M;Ow_odw-=q_<6zxx!_DlgHWhOYKyWF0^8pQA5sSc1G%Uk_K8;7}&Y9(%zjN?l zp|dB;#zxK=+nupy*Tj&DnUU<^pCw{RFd#oaOh~+r1ub3YXbUD{Y;nAD{ieoW`ox-N z8RJklxisVE;DGZ#A5^C{dgWGh+-xz4HrD$(PD49PE)HE7L?BR~-_R;e9%BJD;SSfLJ&BX+jcJD` zILms?c>6R~=Nnw)QG1mEhZ#9|@Y)a?*2Jt2{_Mw|N=^iz=gTo73aWLUk3Q5XM1vY@ z{3t{RJPheiE4~!@^Iv#yXpBO{u$l-w6OgjB?&?07uAl2;Y>wlsdpV%8{GeZ-M=`$?vO0guH60+DA@Rd#pK&nqIGAY={Sa-z-`qcsq{QOCx_FJe`p7>vGIsS;^8G)=P~@E zll7?ud?#y04G5^3|8fv47#J8(7d*&Zxf6WtXiIy4zx-7Q!%$C&^AZ1)64=$IG=7 zLNAn(8qFF96@7KJXY3_pSXd-5chTlcNp#CSF1t@@1CL#EE?><_XURDcNG|Bqe4?IT z>I9^#<%-G5jhg|u01*h57M@>%$ngrEuEZ34xzZ>#x)J}v<88EuBCp~QtV_{&!ZS7s zmC)(c(b@)gNYgRy_VBO%DIjtNDn(k7Ti|s-4XPU^$0DVeJJ5rxEfgl0b#TSu#Hv{uCD4Km%jg z{9>`U+m_18TOe-w1&Nl%RU-vYE=Yir5nKirB?Akg6f}AAx_j(yC4P)IKQ7o=>=c$) zFUahEzTK2R+MVm-ZFk+yzrR@qhlGf|GWLZVDu~pY+mDfLrX(15P!j@MglMgjARSuS zfaQT)x|Npu#(M*_KeKwR$4E(J;ktYvwM)MrYIt)~1S#Bl=FPDQ?z;y2N$Tj8jHvLc zS8wnErj)+*M^Xtei*mKJJb!s{SdD!vgOFM|aU5YM860^`O^|1vO`#C73v=(a)=7tU zw7LorDhN3+E%wV?B|B5EZd>=!9ExF17*G-WDG?MRg=Q)7t9CP7teD9j>%=~@#Kri_ zqjnEA{1<-r>OxU{ASM+A+Eu?x#8OLbv;yMNhdhBU1pJ!yS<8Zl=njXf(noU+;ciyg zK+Hm!Q0UW+W+3>gTK@ecJN`NOk~n!1tT`h997Q%OuxC7A`r^X13w-VR*Jfbv`chlv z-w+s2*b}XS3&PHhUU1xguh^P?c!0CT@-R|A~ zeZBDa?*jb#*W&-LrT%B=UKQb-tcP`m81zc}zvz_oG38f^7*(XnYXWAd zL!zA-@4=YU-@>>Z%2Owjm-rlS4D5p9*k#cNEdO-o+z_QLJLQD-W zXLltw#=81#5=iqec{MLayn5yW^eQMv4~q#_Nzg00%NZ1UC8WFYV(Rnp+;LrxAO9bf z8viA5w0D`6CF{us{Ebu-4<$*?M>aN{{0L^~P}7AIBG-UwrU8;n5otAI{BaBmC^{sY z`^p!6zFnZa+T0U&Ift~yKa*UfrO{;&CpnmmaqnUhd`I80q)UJPP1|F-+vsA$Do&vw zxztcq0eiGzkfAC1B)3M3V_vr2BqER<7sy)9YIBH{SbWZW=%0G^9ZGrlH@afV!RoMn zhOuw6pzP)>I#D+5$w)OeeKLoPmq`drayzeWH87wEO#C_#vqYq~duO84zh(M9>1F|~ zB(lnHovSkbUe5g93O!NBP@R;&)4;Z!pSS`7-G)-2SVo!{0 zj2?$EaRn`vF>>$>j#geTrpm^U*F@eC&vrJIfJ%kqf6ypM@wkiT|niOroiFoeun-M zODzF;#PkP4@GpoulZm10fyB^$!ccG*U3h*>=a;PVw}3ymEu0b%Gq1iYCp%6Wo>fn~1j zZ4XIM*J)Ri{Y=YIp3C+xtGuyUapScI&r3&}_%nLYnG6Y;$MU5crDU_Pn5GBW0o8W- zr_1fK1Hhu*_KN;1nfxui$&-O{n4g$5BHkvGazp?w71K1y)`r3FQ^5V90jCmnNPFK< zASADMmqcQ+^`Z)))xl2+J`&$n3_Qp-4=x*+E03VI$S-Vb!|vpHwv*;W&(xw(^1hY& z@*VywCe+bmx}9y>4c-7g@+ z5Tj3}&Y)cQP6QeDAJh3H-11=7rO0f{6=@U`p!Yr$CW-Lk?=MchcvXTR{`S<5t?@IBAEvw`XW-IR>88{gkNl!=nq-_a#4sg8 zz-a72*p6)K=j5i0xnGx!7P4wB>v1z0(o{^}PfVg0rC%@3O=1o|!K`^p4T&O#Ri=Q` zv1#odAgTOn#4ibnRYqh&+VH*3+CuJxmumiKOC)~t$BgNhUzBC62vJMM5If)8N`xV# z^Q`sUdAQj(9RXXJ1b^ekMUVtq#~ttSuk*zJOabJU7rMCC;)qQSPIRJdUOd8B$_vBb z#6dFng_xqPW&0*7((d=;jdwaMJ?+;=bfIOk#5_WIa_6gbd#$t9$O1D^>Yc^&n;7}x z4P5$4eraF~p&&Bd&ay=_hEjFN@C0<*Otepp0dlb~y#EV$U7dZdlovXR$ObUr`NdlEk>fWnXiq}7sR@@bmN2N=ZXOYG25#7gi$Rj zkIblY6bIVHdWAnOy@zgcS#iYIcrJEyI25RBoiFf@$$pe*hrUG=)fC@z1M1YaVDZ~A z&&|%ySZqqyw)oGm)~Qh2~44)@R{u)9XJlPR$R1 ziwd`x$EYbM9L<3p%*S71f(dC!V|PV@AcoFz5CYjJ&MMHX;(Oa*^25_vh5v19-ECE(|OWzxSd zM=OW{MJcRtJ$+Sprwxy4hDhvb1Dk2T9ei2+V_DSAM8n%JjVStW)Mp4I20V@quoA(M zC;1TFYd(A{pU;Nq=;p-Px?-4-w4gIOurumxl_U^@F{KH=!^g3p#5AI9e3QQ`dH*uY zX28xT_#x-@S+GzTT<0OS+^oDj$Q%wnz6Lz4di1eDv)TmP6o*AUVM9LgM$=3E_shxA ztvq?516GDmq)|(NsY!qtv|)n8;5jj2eovJiqQ02vTYUG*W9FCh1}5)o<_>RoBs20O za%LJ&%C#pa|G;q%!lkDz+sVQ@n{8llHFA8tK%C-xMnQ5FTSA%gB$+d+lf#Rss2atD zODX1{n8@?m!1o9Uq>&F|h5~Ay|4|`_H~{_>Ip$*Ik$knNJfAZY2dx)D70<+4Fsdt# zGSiRxrl4It{R;^gk695xQtN9di!jwxB2S6&1>J57;+uRvuW?H?j#5^wvm>Y-~)63~V7ZId$y37wm2ZYoG5; zfW)tR9M?+19QN9?o*X6PZy*IS;(rttQ!lI?n}N?oFL>h;T)0B7hq_Zd1v2o+OtzZ`wZ`0H^|dvUf0BCRFDn^n;`>=F#N9*skyB#gp?3w{dH(MCR)J#`{eIzUr>x1J-%UjM2{ZZqBkUjje40o| zyl65_&p_`5!3XEZ@H=i*-=#N~eFod^g+yDH2J-Z22e;_+A5)M1sOPEC@w{pV)+peX}Y8nnv~E%T@ZQ;NeCr?bV=yFh=sL4 z2m%raN+?P~0!bi92@sl!0ulrS5_%CSp`$bv+sRu0zxH1Hzw4Z{?|Js!cR%;;JkQLW zb7agh$Na`OM#lWc_r6~}+U;hEue)Jc4{u7f9mNx(d`%nQnN3pLe7Yd{I-s$@%O(_P6U8-^Xgn8zT zYl^nAr^A)hXwU-FTsJ^fv_zrr#Wc)|QxWKzya4SmX!|F1GOVi0-=}guH*3d)WMcG0DCA-JMe~nOEUV5P?Lx~PF!nlJq%t|k za-nh>h8)$_@HiUA1?R6N`fCP-f!r;VnM;N`)Fp%jyCQaBD%q|e3-R4r&^fP6GoWyE zdLi}Wmi3$hR#5Ayw^B@s8zJA-!p;0+VMb(BXa%Z69bGs8+46YRee0ie>;D1!??2t) z-<+x*I+#B1ZS;A3rZ+U~cMS2J1wGm2s&Csv4rF_|-+{D;e@EHM^A!IHZTsklgnvcj z{(S4`-vPV7XXiq~{!=b9&XM;1HaEW69Ud-YpVk!dou}mm3(b4kR{r-VJg003Qhx6J zQ!Q&MB$^>LJSHMk?VAw>Xm+7=QpoJw@G!$t!MMD>o)9-uHN>TDDBl}H!;|8f@E26+ zyVe{f6gZI!BRGdUaJV|58u4WKu=`$oP2RCC2vO-B`vs>%k)OF(g)1+ghk5i%m+$$t zHt@*=wJ*`>nh=(qejyvF>zm<{k)ht?GFf(tTz|m39~FJ3$<#EPJowLGNqm%~r1#+!KJTuj@?eF? z$72d-YOE4#RtLodheuv45kT<^+W@L~akN+&nvB#?I8!rYY-7KYx)7ya z38|q}BAfLf$DYIq08$IHe*U5B;b^_erWJ2y(p)$pZvxp>O9@2OmBgcmlTqS)F!J%c7F=>)sK*t>})uUE?AX0drMn^;zF}?xgRO zy^9Lb+F^i8EfgQ&wCkPxkiOmtaQVhnTcT|pOuoa>Xw@eTOV>U;GDu$!G6}w2O?9E9 zkp|YZ7c$(-kAFjrvE&{{Y4^zO=*ASfcF7u9z z4W!}&N;)w?JqesSo`!n!c-X?hlS?S=?~ZY9O(RnmZPPgO^uQ#ZFks zI`cR7z|RhDC_IZzuozlLQ@I0si4SCGz`|v8j^e{W>C#6()7}>9e90QO$;nq(`KhAn z_CUAgzGEd&;KI$q2g4j+m|Nzn3&-5j^V8LPJuFi{U#~miYk1mZnDCxMk-D$=JnKu- zOifhuL>;>domNq+Pa5&FjE^cu#!Djsts9+UAF|atY+KtuMOqc5)$J{h0gaKa%7PeR zInA#S@B|%T3qTic6&}Y%q5Z$+8@AEwc3J!t%GvKxK6jxe5s1k$O8T?ds|z+hMf7@R zQ&0LzS*gXqM&z}rx7}rV!o4O#&vd?dm#`-uyPaMSS?1#-FZg7oI3daxyhj&MIST#g z2w+ChIBs!5-jDH`xvz_Kv!?ebMv#3R>9As2=p<)f%n&oNhkyl6Z^rCGpd8yG{VrsFEC8EQ-k{n#ssSx#CF~^k$t8-C5C`T#TnPW>as)Uwd=F{DxhG((C|S%S%TU!sUOMPL-Iz2RFn!h zF3h83w7>^jZFk~C{1Aj_tf(MmPM1-7Nug#MI59l5sfx*}##Hj*A=uz#NR1oWH<>%9 zF<(BrlLO6)7|_&JE3zW$%$?nnXv(O-rRb5Sl!Hm+vW%RH06jxma=$5KY2G%dU+`H+ zv}OE5x(~zc3=+5aoMisUeDKJNg$N-smgx&g+EYDepnoxNeyH2kaDz^#3u#hK3mRjr zeXRPAVwsfpuzm-D685pynxtdtlgh+|nUJZ{*c*Drh2>Mqv(HaLLp(GxqKEq}fZa1b zZI=3hm6@dIK32P;3wMYn!9^R(P8ACztGoamb3S7^Vj*)V3*%oA>q3ke_!e{SY`o;| zmOi^#doXBZ-~E2sy0g%JL)=tcMzGpL{S6NF!BSMrNZGV}qCL*b;`3__PV~RJKB+}lgzNX;i+oKPtWS>X1@a;%Q z8jZvXFTszFUM#w0RcbcA`NP*E=2`S<-hH9==;Lx%PcQTDH9viFWl(B8=Jt)`(4t?C zjvo3)Ns+0a-QIYVAmpz8S1oW(I!#e1n{Xe_3j+Mi3h3fz_$MzU94TD}O^5 zTnX0=P0fcL+!0q;%*6k=czYcqBAVqr&iI`wGL%M1Ny$V~$~N94qd%Qbc2l-R{dOtW zL4s?1EW^a7i3zIF5g!<#875qAPCs8mgIWu!Nf5l*3fA+hHH}}VYW|k}`McqU83_@0 zWFZR49~Jh0m!3dzEP1@6*t9b%!5hg4yU(Rb+=l54vW;P_OPeE2*#no7NaBSz6 z0aBFPwZ>J(STL~KsYO^E%O8)v#g+DKvBI!PifOBNeCJoiA{EUOR2#=C)Kpn&M(VtPAL>J}DHTCUONMsFs-E<^ z6Z*Ge+K+Oi6!>yp3G-z^@-|kK2K^{W4&@OjD+Z)6UYn1Jjx=d2!gV6x#~$JUfCHER zxTzV@c*r5gd#Txxc;eE<;0Dwzf9t7ril*XG*DZ#_so!dk0knb)9l%Yw| zdBvvPq7=F~o9QbAzHtnF>tCPxkDC0)?Egxk2gU{!s!pIM?83de-D$;{_=j%lK#%*@ z-C-%a+_lR7loM~x$liH4p$t3ou@aWcib_=59+olY!p#;H{T!uQp>U~-t&}1X0F&8f z>H2G^!?2kkD-0PJInP5cV+8-Gc}q)g|S8G^)+qV(#gBZ?2t-M<;t0s}6=O!^OL z{mn{&v!fN97U4YuVc$C8|#mqm6;k>TB`5L>XcS!qznw3$?n0W5amWN&?n2t_7o z_DXo-v>8ha?~57~R*MSg!A3IBIIw6w6cK6-D0nR*xOLxk#bM-HIss;EA;FMzR6f3? zi?#T`7VQFpd!C0q;R3 z*2Tq#K^OI%O;&bm+pDdVBy_3j8ae~bIh)>}8ST+wK&~C$O>D`xB#ER)Cs*Ef-;Kwc zqyc`;?3eviJx9GCu*8!GVP2?pggFO?RsikthdHH>E5)k`91sXeE#5VI$u*TBJtO)t zBHinOYh!NmC6vi6i2l@9O~uGm~3lEwzTkA&OUQg*KwnuQ*mJL3aqB_*$fm-=HfdEzUs!}=sj!ndR*63JZbY}w7?xBzn|+z>_66L()8ln@>U*9 zoD-i@ zM|}-C+c4F6pA+UwX~lRQqUn%cF$`5$!+@Of!gqf0J{j&U;WtGe=vEdwyeFD=4MU_= zq#TMUu3nCT#4`Bzg}}OaF`hFJ>*V!E6?L$|*KQ8u!!S^Dq+P-|EfU!=(9w%rj@G#A ztMSVZb@YLOF&_-NExvJC3JI||)D*P}b5mDK6#EQYzUw_6pQ-{f*m6ZWv~fi3VMKje zC9d-Ei|;%())j7gFr$5`ca#j~tgyUe(%{Mpt$e+TAg1DKW^zzIp(!RV;%Am(L><`N4f5 zNJ)+e@a2UFvhqXZ1YmHmizQW(q+5vIb=;V)rnwK#xr?@7A*1nUr7i4?nud^FmXBI2 zPEU+ZM1aqIx=^6fM@Q1`bM7zbZWG#{P@n6>s zNt3G*4t91WH6d>t&8#bef%Fv|qSb;J%PAjOS_%#ud(fBeGORWAb2q_Te1uVn17R~3 zIM>_}@5tkKh(CTXgg9cHUG@O2p3!-VuyqsYXG$Z&SLa+Ix1*^w z4h~o1l5og)#`{C7f;A4dAEdC~yiaRAc@-|J$ym*`3%2dBAQdWNC0i9-O0ftS)N}ns zr83BD9AcPdYKiXh-YI00;av_Xx4-km^^C_~77^S|PnPHlD!HChX$6o_I!Vma z)AezzwE{{Sp1bRX93hJrq59?1Mw6aBsVc>gFyxpIdY!BlI9~5Kb5W|bBU}wa4lg2o zkbwQC8n>-ggt6$(}OlCTlVZ-w@H3K z*kk{3(Z=O*MFO}caYlDiSo|Z`8sGy#j7^~OW1EME%>b-QV`uR!kcnfk7n81WSCP9Y z{3_}!U^RSbc*X%B+A08lZqRcm9D6@iR|X{+SRtcPa7H2uE8Xd9P%2Je?a+qBo^jcj9vHgEFUO5;?*ju8SzI zh!2Qgpk&6J6d%e=6y$1=sy(zjxVDq5G^j;6u|WGMJ0>LY5*Q@hV{ z8Vn}OQ`^`VV@!KPttwV|nWD~$kFH*Ha(u6A%c@MYE8)Op(qq4CrO%Y@RAV=kW*VULxSP2%fJ zdXKZ%Py{dv#Z)yY(-^uN1`e^uYZzknA|z5%FJg%x7s6(I@Na-`z$`aKp&8<=KtzJkY%@XrpQGy&J>qj_9(K`!I;YuwN-gZG}=OAg=k& znr6lCy<&nOLGBrWw|WxrpREFrl{6xrG2wufm&MiKEg5GsybENMf}V?<5l*iuZ-lw~ z_0-n`^ozQBl!q(XR;~e=#qmdK@TM}F8F=qqx@dVm%w%r#VSa$}MxFM!=<4opznxT( zTVS9n*{d2XiXmVhxLf5Sx8{9M-Pgj9mbFHMcoW_C-LOl0&h;5}G)I<^(CroUgYP_R z15C<}HK0sI#HgUb!b-U<&nf~RC4fggiqv^sO_43qnZ4_M=m9bMgjB$FlCxI$>Rw_W zZYM{nGD*855NE!a-OI|zSei5pD?VCmH5$w*{N(i|CTHe5kKiB|rf>aI$Qi3bcsfF8 z=RM3=@Fis!c6rH%d%waiNkI6jam)4Ig|NZyg;Pk8I_+bM@mjc3j~6R2#Hx(uC%QZ2 zQ%B4(9C-!;FDFG^h@T(xYrGO2r5V$lh%S1^9%eynoEM$g%zbk)qUe0MN|`>UOt4)c zKzaA9(X}?$lJUbqyTi*e*7V`8_Z#3t=3~>nQ)Xt^&yzr~&=@kzoxQHUqcs}X41)32 z*Uw(=p*`<$a|ZJ*lw*-m)vXuAuyFI#58>^5r|-Pxd6%_wy@C)t0H3I=9$g+_Lf!a!@c z6b_xbz$%JcY$a9CXZYPm)q9yVgP+%-lx9#v4aByZo4|=q7Jbb7;u-)jtDctR(Dvxh zPM=Z1_m5e@nn-PP29Z0F0~IwrlCu@u_%qPqJ#VJOFV{9Q8o(B=ygu1V@624yl+{@# z4|?W?S*$`x_W0I4=25NFdp)gfO>-ep8kl(G8A4ft9y^s%2-%ESS)GqvCj?9>C#-83 zmbyKz`I_bQ0m=dskU{37EiWQp4s$%FpinbkzP}lz9O;?1^`|S_A-k_njb4bQ z{)+T$ba7*u0lR{H{{TV!aMy1G67``Yh^j>Wkd|Q zfl^uKqCh!QvPwC^>gxg#;kJc99q(dzr8V!Cl~SI|9e04DFjH>7^F?LGmK9mjvhFo{ zrkOF&JlH}H=(5Mpxbhq?|GP3q(W34)Ost?}G6STNfZ4CO+lGntY-NbBYW2yN2Tf|@ z9ID+_FZm7(dgN8t)QGrzhOU{5e^HDC`h zi2S;5E>oUn%43ow7ZarK#KtY0+Iru`fC3Tag7wyXn-p>dj%MZNu&!rh;gLKU>OAn} zw9SV@UFQ-_$>*=EeVxX9=kdWeySj4*98Ge5<^WwN?1fx+)61Vcx?4WrT$Q!QUoN!v z*V=vOffR|={-Ed;T@R!_F9j^aV11AzPaRcNx&1TIJps+HOIN)rTq~6oDf2l0;6}Z& zxV#$2?%_sQPqvQdVVO44k$!!FfU?KKpClVyeJaqj4}nDuBHDwN^dECwhw*e6IFcb? zDH+;+;gqCMWMp&P6=Mq}ZCu(i@(h02!q7@KSUmmP)S-Nn_^LyQcZhc~XZAIDK!5af zq?p~pv7Xxd{WXdNMlT|_ml`3z%?yNrTDK27uy-$XcGYB4`g+sNt07LS(8H3p+QFD z2HWzZQCHs>bu&-3!Z4uu{?qdsohy#61`o{| z8Umned(D%a;1C#g*%srVVYUlw}U)1+nZ@nhphN4_G;CxFX?{cS#0)^+cfoHaey)7*$K zi1-An%(vR4&4yMPJOu8z`Z=1n%-oBLnP!=OdZ!0^-f%Kn2$4CM{=9-4M=bb0Qp~ly zAe8LG=WIW285x}@O#?Yr8&WbnZciCGQ!_gvN8Ji&jPbW+wG+|wpYlqu5Igg_KCjL5 zd=Kwv35Bca^MgZMpf9JW*W?W;TK-AE|46bL{ZrZ{{&Fs|Am1G|9{QIOwioz6O+HC)M*~(9Vd7(f zFts*0s*Z{{8ckk*3;&lEmoO}@gE$37e}#lPvq-+w&%e=GTh6j%tx&ksk>Rm}dT zizj~9(DSJ*5RA)A__l56Pq*coHGa$buY%&~5k6v-p!h{<5q7bK+l3@c%i_f2H%ks_$W0+)npi63WGa zrRJSBSURX}<#9&}Zs$Tm6vjoUcQ!52Q1ti1JBK5+p02nh?DPq(N4I!ax1=BR`fUz% zO20Me>*7OXRei#l?zW8y&5Je=#o*(~OiXp32}*ogak06IIv#&v9xKM{dSmNhJvQpT z*g9zhW@juO($F{R>X;@rupUl@!R-#sX0;G+*Soe5*641OAvfjeysUK{sq(TUifIj8 zYKkae8%ry7eAh1sk+Y*u=tJsGJCQe|msR;Ggz^b}M=10yxy)dwQn#dhd{0Zv1DhVn zawOGsg^bmekg&b1(NfF1%OH2Xx`l{-^d3$*%eShcZms@hcP;E9V(mlcBp{@3U|>E2 z!`Q%L{9TLpDB6}~oM*g08>$9gC#`+!E8TfFVm~rfQhZjrp$j_9oSLu9fI+gR*jVf9 zCo&TWwO2r46*!J1(>0QM!iSd4|JYD1uQmRDUr5KPLsetc9ew$)ShtJ{3gb%M(a@+_ zvS^2r6{IRYlTG#@j>8(?un<+pCB8P-u4`&tQqb4EQY$R>OBMU85vHCuE37%fi#6i= zaz0!MnVOnjxGQ&>b;`GS{YKvyt!}S}zA8RukB9B8tkKW_N=F%7ac!AJbJ>l{9TST%o3Tu1f_B4sX#3vu(HE z(V#FtN`zsO5tU@`(P<094?g zD^1)j`m2&Wy=vb4Z#v7x0vmds#C;!?alT5K!=GSM77vqe=+5d=?YVE*qJK)`J5S9n=L;y98!alX zF#|a4u(0uPeEQinXlJ<&6E)aa`EYl#2WE*UB~FkNBCl z9*_RD2JfDHR&~Gm!g2G<{j7gEx*PdH#lt*sdW1`>qVDe$%))-gnq*(N0n1;=T})fo zNa&1z8OzD`kb=Y+HH{#s!2`ok)|G)VaW$&bFg2S=#lB@Ay=$`!-tA6jw$+@)wV~_Z z7uTGaGxU)f7#Y68-F{Ylj7H~!mBC;v~Xf~JAhn*as*xv3dX`b z)IJWDqcL3Xh{xKBnPWSA;|~3PjYNjxg@&fLO=9Q1xyipk!Nw=2>`<_m4?ylMfU<}{ zEv!~l-f)+(Ocbo07*LX;>wI1iZ6UZW;I*^k z!_{JvBngO=kU9q_1g@MfgyR;}#o$-MS{$CPv|?-A3as2@ah7AQlHDhig6(4b^^>KoE3st@hO&Z8WR)$pi=3AXmZU(a-HGv(fOaFR>U&i z&Kg??W@X+nmL<;;;^&z!ija$R7b**vt-B!LR1-KgI5fYgXbm0yFgX|?U4{lt551Po z(S2pQV$HbV{Vwg-zPqygv3CCT(cgKvJnKJ|k`wB^)dnN~td&v3n!5 zyz$^`ozbQsYJ$-^Vn@v2 z$l)e`b7$xh0vSVt5(jq|_^}C~TCvM{6G65cLx)1uUQ3Pia-$1Zs;>c-tpPyw7FcXd z{nnhvwA7X;Q&(tBKh#}oB8Ns`K>>y%L&37JPagVoa zB~1Akf(zyfA_ayi77la85Ty&toP8#66#F*t3~EUtQknd?(oFhz3pR;XM?WlLQ@4Pf{NHYUKc2<pQ z>jWbrR|+&{DOKsFi$|7CrG$?aM@z6CY{L&Av~DtmY})HzuIBak2Gz~fq+KVn|pzY)YZBKU$ z?9*r^^K}G3sUzNx&2;kh(gTfTobV!u4&%Lh_ZC@FlRUqRNi3(3>R8>ux7k9t$mgO_t+A3-)f(8npKcG0HM2qEuD?cU zDid9F9e5-Brsm5)iP)yHS^QonQ!r9olHZya_W=ykgm` zU;Frq-63{OSH@ejjPTaJ?U@cpCG4h@>-Z|~R*c&U*~1Jn=%6!~G0^Ipsl^bsg6r7o zhQFJ82Nb$C3edvHYqj$oe+*FC`MhXHO{r*p)v14Wp)dXyEsTOz$KH=`;CedRRS&rl zQ+WsOM(a`wcOU3~#Vx?YDHVY>Q9*`6ny)QHJu}NQEyOE>;Raxs&aMupwmyD5VWGZ_ zc0>!AiY%kXK%}z=s7_CET++Vt{0fzDZRBCz`a^2m{Y5cn?o)n=>~4&*G1)XR$F1u* z=YryC!at(t?U0lUd5mGR;0BmuU#1iJZDf@x2?i!(yPgh4)1D>fk02?|@|1W;3F_?9 zbd8yqG?L)b^f;}M5gRtQrp4E@fz_hyA46V;E1z+-Pfg>She9120F;L#baiBj0DWD2 zS18TKBgnd_B3 z?oWQovf8-L zd!w5UQ!#8^36vFZIITtut_=IeuHCFide^O_)Rb5)5EJ0g%*xWQV`?jOJv2HCY_`Md zuYYRR;LY;Cwn%HWw{uNfJ)^Vy!C`5;@GcArU9(<*PyzDs9}Td@uwd-twOwG9zgObM zoSSoCIRRH?98gwOww|!Ey60D9^4U2<#ts}pn1)LYutyZos-9uWb>^Zh2;VAMxplVaQC0DfAgsG& z2Ta4IG7P~me({b~sK>@G=Z?7Kv%CYs^5~+(NB^6aGDzsTnWu2QXsTQ}SBvDxhF%%; zkRDWY8dfZ1nY4!?=n-Sj>$Wd_wx7QFyngcw2bUpW=rFr`&B%dkoIk<&qR)NL=!;{j z?%hqh%2n)57eUT;62_uC&v7Gk@zl0y zxtLN_N{Z0v5fASE7w&c|`oPFC-^!!kHi=tdY5Nm}SLb;SzS`TWdG9%{n1~7o zlG4tO`^mKB6}fksk4+_qGsdFt`x-+q>WW zZ*|%~#RV}*UlT^f61luBRL{JlTz^l@oD$ehuwq(cNZ*p~Do*A|MxbH=yBw`c0kf?h zl!t~)`6-B)TSz^C;4v_y_#`}nEgK~j|KRQ$G)++Z>Vbc0kuPTSz6h6jxHeAmz1^#f zo8~T}#Yq*GtrHE3a#JJ8Ll6BzW92YS6lP*8=HWHfk%{0zD7YEl9IP0!cXOwiLV8S`3WNUmS{_}!ckW3JsLe5Q3xni3^POc_66;5V^<6FZi zl$M;^zizdECiQEP-k;dK=Geq_>W_y2y&LR-lo#&UyL1-{4Hof{qQWom!=371dV@80 z3%naXOt^TcL^p{Uo&Y?}8&6L5(EQw`R0aL~GPM&Aygq&X_;;Qhs|DZB-E7>tdz$j~ zy0Ltx)HLY~7?Khd$DI7m12#q-Nsm)4uWE49x=|1~QLow1wB&v_vAtD>P@aF5mcXBp zJq{B9u)<{{Ir%f=`qBgId;Q~FlES(~^|;{$L>7bjv4oH@=~@Cmm%y$tS!0jN$G_&e z^@l_cWQ>bF9CPGYzx=>f*Oe?w*6mSj1U}hze*v;rzdWqPi*i77PW07SHa$SiXAJ7X zARE;%vC&D&>~X)DDU?qp9yb7%(U;ebfc1-Eg;m0(B)RK(AXeK8lU3!Y()yhfqPtNQ z4ROY#@P4_mgjCOBWvddZ;ylWyOL`mkqTee$Xt<`Csfj~MTe*NV`Ast2PDWb3hhz5i~>0bj{KYfv=OJ}BMUjf^Ino%l7vuq z#k?|=LwkDfVRh6%d=wX@tC?HfufgsMq*u*1#u)y{4T`;6^~2k|ozMf=Kcx4++W!R? z%WXd8)CVzH_EaD>EF*jP zihvYOo8U?me~~R5y%ySY)5z}~&vsdy5)mJjJpoK$ILp+(uB`UCEtNeqM9p3qK_T(_ zjxh1GXRuh&S2wIx@+wJVmWB_T;jahAm4)nUR$ji1FU`uQnHpw-xIB=3D=VNr`@P4m z=QN_Xe4UNMMqTuI6FZ7d*0j~vzbk0499u|(NyqDIQglBe8^jwEtXBcP{1>W-b zlV~`6GS{g5d{Q8pvd`;fLWVK zK1`w5@gO9y0tpo0#zXqs*Qr{z@x7nsn*AxiMEAo^UOuT`A78E=xNASpBPl7%fAJR2 z;Yzp1_2`KTdTra27|k_GaT?vuGJxb)q#7C5UiE<8CF4i}+ms&2B-J`VW>6s~%$k~L z@^2ER*9hKzL~dAmi~G88I^2>sdT}bZz5#h-+GgJ|Vrml} zRTa}b9k5#$*ZN?kU%=o`3Z3E04c);ZVcoNP#R%cm%n?rrU$U)yOI&eK?X)#uX%9;u zX(iB_UUS)DA;yQwUxasy>yz62rND?v5Y*Nnjt1R%r*y3*5H|K1b#8#hW|dZV?)zPN zT{4)vnO&pg+Se%-==0@{f_xO7t#b==R$tn#1Ls60(XQu z)k7}VYxZnBZ=)>lB`1Aw^(5zI)CnnaBMV!Uc+oOXfd=+%l%5|=oO`F!)~|pGB;7wrgfpd_jcEY>sR7Sy@C;o zRAB46NhHk_Ko^>#w0z8;Q0eOkdNRGKZYN_C)%zabx|DjaLoD=V!3LgMvoegL;t414%G}KiMnC3y{P2~{jkdc}pjU^(bh#Q)j1@i`jBW1%MylIoU zq}nXv*IYv!95j=FZkXwnkU9SHj89j`4g_0XRn&4p6BR}7o1Fj&$QOt~ehuFqe@25PidvD|jB#Q})v;%b*q4}^i9Sy$==%oJRLZ{6jsKI+IDTqVE+21I& z=ND6+@^)YT*MFKnsn$Oq{?(-SpY!||I{*Jbb?p#EGaFp+IYa-+B(Qtx+uK`5S96-b zKL79J|CY&r4Tbpk_4mKB=luJw{6EQcFo~H;NzDu|Gmw+oPWa~fjmKVK`a4e^H{8lU zc3e-4Cks#BX#e)b^|sjj6!P)JZ(m$*jo`6lO2cYPe zu+NmTtcjMw>2gY7{b<071D6jgSczKYG*yD7p9?3<+?A?Xi=HiwKJ(nw8T@#!ph$L# z=!PMQl&0YGe!O;V>U5Z6Pjk=chJ0!!EL_!qWM!Y_oYiyqnGs3n(y*SN;r@VxbhuiE zqrZbLlZAr!x*sd2J{Mv73)64FE01qQ)#)2oVk z81rTzK+B!;HGS>t@jSih2UF!i)~t=v==V~~cCRvn)iVaNbYs0oZ+!9{>*;NuT>u6R zCnQ^;D)TxoKZY1C>yO#9!?llc6kYNW;aBVpZ~C?114%<}W%W**7h&+9+3_zj5`aMA zV~dK}{Ofyk*pFjhNNZ7xU-A-y_KM>_=)TG*?;^-V#1W(&3Fq26DaELGQd8XJ@vlXh z1GOI5h$67cuZ7dcv97-& z@b6%Az3O))NLv{-32G5mQ5)}tfPT3N1oQcvX!d-SBM_Y0N_&>PbqYzTTPy0*+ZS1) z$hJ6ISK@>+`jOp>&A)lyq&{$eZoWmf3Ts#=etY5{^x4@Wrm{5cZeB_nc1IY55)$ox zPzm)jTA@@a(e!$HkI)X{jx(+%Xdqi0*({uh^EZ&&_O6YlM3?UVj4l;wK#ThF#V-(_)aLD>H&GIi^|j zoPc1pLM~;lwy2$AeZj_bAo0F0V3{Tu1EgAZwJAvDNwDqJpsnk z^NVA%>x3u43fn}gmQG%q?gCmZ1R29#vO#8)Pm{3#&xlOg;X(Vua>!*QD*6Tuhc2nx zJ&gxuMX*!b!hIRG3S{-kvH;3}qy5_4hQL=W995Z;ysdoZ(|O*Xfsj5_j&3*r@1Ha~ znQP+W`uaer^IqmsZL8u=_d&9UUf;e5*}E(OENoS{vyI~h)#$n-RcS_#5p>zJQiAE! z+qfMkgJksi*Sc_1C{6=9Y<62kom)zoSyTcR0lQ~9+|X;fvOP(e zpUYi+W4SUhvDvIEz`>`ysi2}r-r^s3NOyJ5QVL;e&viiVNK57BgZwok$I>(E%^XV; z0mXQi07q#ks<4Y1C8?_PW>|hR!5G3zi6l4B8x-bA-i}4g$IEb9%gXls z-$FVFV>nTakq(AJ-NkEr1sl7A-iRs*H?NuJumgdjT0m@W%$MR| zVa$8I$PvC)TFQ5xh$I&?d+_9n(9UKCY^x!Tm9HC0t45-?>I;-&`KxF4sVCm0TNE7b0T zu6z_YJz)+E%4%Ux0{oI@VF$|sZL(RbRx+Be8E~{Bf=X^_nTG@j2tNAz`-^Sa*B9xd zY^eM{j5<8{@XKP|f32VYl=@dpXvZ}hK6+PA2BVIrO9L4L&xB${JNCSJevn4EUwcX)!~Da z0Wv-|@Ry60Y*U~v<^7vGUNfNt#+9}P)^o^d73_68Gf7qdlhwvG$Mv<)QxZzylZp!R zQ<4nPF~C`dNqmDmqkZe!!`Ex>A%czx@s1;w8MobsrLrMtM(BfYH>Dk3zdU%tqArh# z{!~6p`;ik!77V;z2`W%R4Y=gLO|LpXV7sbPcdD`JHnwh~66t%@Lck@ITk|yIN{iaP z6&pfT+cSZJa2i@iF58`w=@U1Ii<1M+`*LG_3?@x~%=d7$v;G7>ZsB7}eWH}6Qpb-F zxgry!k8;lsc!OKok+o5{HYwr2Om1xSw-iq#{ z6iO3L#J~N;2s5<`w2<_A^WZ6)Ixf-WoB7HVE&J}KqCrE^40rLkE#}-dNdNTodYHYL zt8#mGT{tbzx~9SSezmYcoa$5imj?;{jOUT{T1~3z!g?m4@G{v}7iw|8ww!H2KqU-@ z>9BLQQ&PKJ+igGQQeDkI+D?m&MvFZ11Vod14++*PUq5XO)|$=ASSru#gd4^S*G538*wifkk!0dLSGMDW{$0r4*syrq`*GgKPrJvgL{(U!u^?*Y#CV3&73oc!g= z6~jbVZ;R4ETa=iX5KKUY3rL0uA6)|*Up#IVjUF8bxS*TLmoNl9DN4BM4io_Z0CrV* zSCkD`#_XKNEj9$J*}5-TXQMebxOncqt$9`I^<}`B2#gj4*LaHui24B(?)W5pi5&0L$2o=1VH zRd&m-O6hfvcQV#Zte4g-{8;lg6L6jIJvrPH-h+=%Vy1%D8?};A0T_CB?k};pk(9WI zg%^;iO12PW=!%<-jERV|cOaL*NOwW+v3$%GRs&)w*g9c-H~_+uu=3NeDKwhfO;u+8YDoE-P$VxWR%g(Ovf7)6}ZbQYqz(|6k*lW`y%0- zq^NJ8l7d6>r~vBnV7zE5b~U*()y+lvai1cTewz)D?dC3^KG@<}e1j^fhSOuUHYG`3O=1@9}#Vv95MV zQX*d|^%p$5#;m3+c53E9CCGYjVMazD8&lvb{Wlv}C=o64@x=!wD==+5y;$Kzo&KB7 z5f5cnFh&>oX8Cy9;D&?A@FJbE0JZBD0-ab%p*fKUNr&M^n%OL>l%uIHKTJp;B3*(x z;(+HE+c^tK?}nPeR5X9);0?$6E*6epz8oEN55i{m?Ykx`y(+V2az`xlAv)YZAnE9JLplMbaQUShw8!&8>M`o;hwXx)J zL&pKvlE!COB&;=H-Flao2-%b+whQ&XoI&WPk_l3?>k7rcj6@^l#cH%>U?dq2-Z1RE z^6o^QO~3JaVgnAsH*md{Qyt+J$5R(c;%UiU9^$Fd=t(ZBwhj5Aii$LH*4tF$zF(z$=BsoR7OS?Js)sl4q71zCmx(E~t#0f+snj7n=~w2!B!O zUD%ZKo#$BDqNT_!?`75KrtYZ|GM$(JS2E(|0*cEGR>eD?uZMMo`Ia?UxZFP$N1n^B z*7eM$JuhsuebReV8m4UC`%5Jxlt9E=HsKz45uqRp_Z?GOT-aWb^X48p)(e+VBpzlx z71lZgo&e5c2khMQnWsBG&;vSW}I- zYwiY++674By^CQ&Mv`}~pN;>DZ*YKvK!cun^lf#a+Zfq;B*GU`ETHgSBiU3!U%~#SD}GRe9vG)Lc;RpCAzSRp}|k$!%=rd9Hmf`;`7il6ErQ6IcJd(-*rD<8)y^SRMe7S@#Q@9$NXp?I=1L zctFqQopjmFEMW+v4XmF=BEzWWi~bY>{K4Dp-GMI_e!o@0PWTLR1$($@sIAC8ZDSaMU8B6mY0jhv|fh)ksS87 z=F0p_R&AF>9Ye?|Avj4qE;va9}QL4S7@`^T#n z0tm(SF6EBv|B>vw^Uq9oeEt?Y6Y&=b`81UY87wgqHuoLW>M!C+f0x()^83$h9zW!H z1U9jf<^s@!LQV|#*ehI`-U&UcjVP?!16md!_1^Q3lVR7Yb%Cm(l6nFis>+0)?f^t# zpm9)s0;?O`Bk;)Sd?u~|u{(PY)iYM)cx550UOluXK2Gti{0N2JwkGcjwl(3tPc7Qs zG0?Pj)U}%7vzkceHYc<$CF#bfe}2V%o$C&~T6T__`oR18!(MM@2!lz)xbNT_N-^%a zI=h0)c2gGhr-Y#w4mTEr8=%jWY=Son3Di5=6pZ3G5nsx0c&$@PFybv&h%p02?bK7s z8GY$X^G+_5pnj1rl&WUuZO%bDtMB*&4Ht1S$A>K|v}=ge5B4rEVRAJ0?w%W)<@djd zd;!zKdF(e@+ROZHRbpjMb*+Yt{LQIIQXa{uc~MKo_ae6GJFGnn845eZZtq2;9*ZHl zI*0B6#nlR|gn%=P3m-c228sd}#^MUShP!P|{bCZ@Ii;Zig(kc*;X0!ETtmB|bi)bo zrL2I;>t!bWjDbdJsQ5U#SPo*X9GxyifR+#rCE(TR*x(ArX4_3&&hePX9Oq{alLRM(CDskXV<%;@-!+rRZ zATyUYWfaSn?vQv%&F4zD;OCsVwJ~+WH;l~uG6{z|5&1@(a@^hvMDMEp zH8|QEp~3x;FNL4YRIt3FXcB%~=AuU~8Wmv%33x5!9;DciKegUbjZhFosts5)ULRtJ zRXo@HY5j_=YI&$e%Q_djrOzS=Q9>-$jzJ;5W6}j^3ds(q3WMPQht}0aFeS)M6R5904RB)GaziqJ3JPk$zS)J5suthh7TT{PY+9n zMi!0P+cZZ#kkblougRIZbc@_DqFOABZ#-eC?wB7vZHTXHlstq+W1f>9zxhtX-J;JO zmDKt}^}^w*ob->K{gYPiJQ>4b!pJ9^Q#r#gx@}zja5v2UypzYpHHE7n7pHB%Kk6#1?W+=WVQ+=-0 zd3^{q!PvL~3d8ZZ!v$CF8AWw53m(;d zBdR3JorLFDkeubtzHQ`JN&MJQBW0zpe{hakFynYH)fHQk_aNe`<+CV^(zFnT$YCtF zVKDT3uZ(@toj`oBQ-Jl!-JpgjDLSVEKn1LHT!=Fe0I72(!=?3?_V$m?de~KFdWwqY zH#@k(i6x&Bu8iDKFDHHmb>>^~(XRPNjA#66O#>`DUj{!O;4a%x6SRevX)`u2#$6nSy##z!x=KM-;%L+-UsT%NBi;9?5hq|aikJ=D_& z(uz!_(Dd<#Aoki+mP9y@cF(j&7c2AWkr0d8a11={zr!^>T`<4)u@N5iCH=jq>df_! zmT8JoXu!kKg^+A~kq*2K>(vogfpn6Vn)8zR7(k_Oi4rgGr1{Ir)ll>1)3K+wYV`ei z5OX`Fo18D((pOF$Z{w->6Ddut7K~G1*V>M<+pOyLx29>7%5yn|g+hSy9-hyAD3BTN zc8=VNoo};4k@iPyeztEGb&s2DQjMHly!SeKk^G2Mo=2xz@jEQWG%e$(mErEbn5gZ% zjK=`|jHr@SUh{c|1XLn6sJ;C5g+rm&{8jDM7LkRrd_0SQIh37T&h+H7A9$W`WI<^U z0+VKT3N1rg*aRud_Awy?x~qTEp}VQ_uvc^E&~$ih&p>nQ{OMmsv35Un^Gcae2+KlS zKE_n{Fa%Y$9s+@|%Ok!N8X@i5Wl3%yQXluRiq~xOGA*poZRRf39+Zrp^JMbFUXy%R z#q6i>kU!Pe)vKrppEJD!pRXPd-+leD0+6tur;b*JDZ#HHG49G>AD~t zT~hS@|P;j5HkdvE0e`HV4*`dF>*#ksnyR$r0J#8Jb*P*LW4R_m_SHsA! zOXt&O%5QAMY($e$mp}Bej$Cfc#~aT1Qrrvep$XAMM#`uV$QE=pL#OXfWG22XCwaW- zGJA)A2w{+SIXmS|nI*OHbQ&i$dF|OigIib>*NZYyTJD~JgE%WXcilH1Vx&;%Lic~f z+(hr=vG3H&2@4=DgRZ8Bxj>B!PD%UItbt2YmUDJMG4W*awR!!g!Vy$26GNOpmX8h4sPee1pim3J z3RP~!_c4W{GyDv!RBykJkKPWm4w_q_d4!XP+7jIEMt4Kjuw*VuLX0hBA*h;pl4PE} z)`_y2*ehoKnyY7;26N^ zjc6CYf6OU7YJc+3wp-I!Y0We5)VgC3w+U9ymixTssRXY38|>p1WaO~_0{@|zM!<+h z72m$>@kns+>|z?&KeUGhn0J#^cMMq&Qa(@_U;{hEWE#O_pW5*M#vaGvAKneR2#I*6 z@w^BO&jYAal3Z!6ADX)GOu}>b?Q7EUo-2nxnaKN{E#|3xxx#S`8_}v$3nfQKC?cR& z#!l$T7t9y~sGAeciQMOvP~mbee}rC%$?^#|zWmvIv#=DKO>?x~!_zPpYI_|eI5wu| z9_OK*5IVrWc=72+LHcS*UG7L(CZq00LVrw!tE}}AB86soh=$FiJC_SSLBcAO3_eaa zFfzjRp4(^;TH$AcGJ7LhHS##sgDti4p2p)Ltc?j3x}&~KmLdD2Q#euxa1m3N;8ZuV zKdQNO=#^f~@I&%=X-K$DwqMKxgjEfqy)5cRy@PI*&aAigFMs935{bys>k^;90<~x9 zktpJwSEo!)uqo>Ry_G&jOms!InQx^bDKVAg_m9l4^)|oyy{oQ12s2YGa7@)C?J}?VF(w1~M z4ET1*oYu=L`l`5bd|{`duacS1O_D%kbHSGuBTA|pqq-;$ya=L(UrLP$!Xj|`k&*VU z<_;!?n#Vmxo{cwY)#rPg{_yIm=5n2Na4URhXx=sWEzcJkP3FbWxP~2V4O5s_&(EjF zLiZA`92>rslf;!_41&$VXpacWw&GDunNkH6k%rgPNC$`Sm<>JsM-bpYdghONM@IJK z*Nxc6^NsCMeQv`x&m!wabvZozt1r_`iAXs$m=-&bpH?7*-f8)*^OUgFZ}4N?^z`OJ zJZ|dQcsu|OtH`P|q&DPZf?amk&7mlRhz}jfN8ZA!B*}UxQFFU`t_weB272BDps9kf z_B>|sYJ5B$rp>N|zDF0{$qwlSUA1j8D3nB$Kc1DG?NXxkvZ;_Vl@!hb)P_r zWJA1~# z_gDPs8lsX7Csx-|4iGr5g8Gxre@*WzzNXe^S$C-G0ZPaUUMK^Mv~CJ<^g2LYUfT3G zD384Vd&owsY!OSb)fhVHiy|h3nt6NS235i|eZ}A#o^G)EW#v&wuY?yU<@A>ykMMS> ze9^0!W9jaN#l;@!Ts%*AT|xrqka{w$?Dux+oEN$^itC<&E;w3vVkp#wJNl)NYZi$f zR^QlIk2zN+li+}P%|xP43T;Q zIM`^zY-`hLSGDRy<1cf$p>eK2Qhpzbit3qEZ1Bqeg{-6t5};UYM}=sjXf z<8C#)=yqbLNhV#;*Gd}zcz>*xmVQWS!q!6(Rpl}5T(|}n~UwA`EXnCtAF|#;Jt%_4oLVx-2bqJC_W}0@tDfJ#}Ta~wf z^9O;aScBbQM+t=mTlgnuSKolU>94rY8Q@c(#d-Zc_28}B$~ngOc}t|ZAqqLzIr$VF z=SaG3>ELAO-WqkoUo1}R`1up*zt_2IW-~TF`JoZWcn>)c7M}Y3MK4K1D13uD`R2QS z@gV*4rVJB|J+%FBJ=>p8`9<$QbU<9+w9;!PG_?+p~ zWo@m&X4zmK4A&oev!N-K_DrYuhl`!PnN7EPuY?HfI}tN14^^!6o&|-R973%Au?s5P z>u4ImV1X@n`k`o~Zr9bU1c{pEBcbw@8p(uM$Y5V|r;)<3Cc`MXp6t&(A3D!Bm>XYN zRLqf9WO823%;tPXd;Schml+@aj&{< zZQh9IG@&&QN0U|WC6^XwQkF~0aJ;VZRG6(0hNRZyT-Z}r&d;l{-lR}U@B|s1;P1z% z8?1p^OU=P6vDB83$i_(Sha}W(rzGW2*to=|G?jcNFmqwK*q$Cpv^dWO_x-@2V<lPfo%8d!X&Gyu!UwP$79Bp|Yhn0>|j~2bm$}gM4WTssLpK zu{0x9(zY-gQH3A|DrkDyY{q}^fSL3*GNtf!RhgMa!_1;mgr#~@^Mo@sVt%~*lc+B{ zs5x^09xm+&tIMFdYQkWUGN4AtgXHqPNL{@EC9yY=rnjqfxUNIh=}0lDp>i>?b1C;{ z75}Ff^I!F`bY$R{0-pM?9$dfPI30e4Mc6+|^P|kJG@hHbz`%hL2B`O=ze*jyV6zC> z{G6zv;W=NlIH!%#tMtshMY0b`sS>H!@uYdXIRF z=sI6C`uJta$q6bUX=oO^uTe_2x6`E?9tMnwQfy0wLils4C3NiL=(~}cTd{>k!!j^g z1PW=A|AwZO(de*{mZu0tmKM)tM!PA8Y0DFPBqCE<3dHX;-tm3>e?J@_8`XKYQpF*E z?~$DF6}XfB!t|Sn)UM61iVcBJCa#kIbM-&1@gLdvZ|e*Hm?HkmgFmy^2!Xf(4Zp+$ z-Xi%@L`q2Jlj!1F;u^^8lx0LQ-R53NMLF3fPWE`$ar&(}!St~A4L}TW#c1(~+hkcO z1+x$y1BB<}rT?;LW>I!*dTI z4#oLfol;n~0QN$l7E|#b(>_WrY|sMdh~$=0Rr0kQNV0yK-sP38!^L_eC$?8;Sq z6zt3LSKm-eK{AsL1sZd?hv$ms3ciV$b)eXD5hM24R84w<;v05+JVM~MR;fkz6D7O@ zNf#Ea9Wi&mz}nwq>|d7v1v8|5frcqvei!PNvL$TqH6Kl)&`!3HvkSrO0}KVbAMLqm{T^ zDJwvEW{qc1hmjt&MEqb`b6m_C6e@naVz)j>L(&!((>fAaN~IS=Y#w@JPtQW5?xGB} z_I)~~!)AWp8~%9Po8J{fz(^UXghn5DMGO-&=)!q%y%=62$pet{##{YyN<9@xD3fV2iQ2ACd1_+sl`zKEDy}@mv#EYCY1-eL#y? z@WmLuKm1;s0b777PGkDP5oH)5Jzey13|7BJPL4{|C!9;`cti3CV%9vXjr^r+cRcy+ z_G&eg=;zZYj{gv!RsZ{`#}Q}?TWN(!o?k`KfeOH_BUz=|W)-X+416?O5g`a+< zV5#O&;GdXxJ~plFNDj{_q6rGJHro?pt6MoRR36cG8)*LZQS5i=IlBmS>zJA1^E_Pa z3B&d4w0>D*_c5Cahh}B*LjUOzP;@bqJM>H<5!IUJutmHOdN4Owcs4}xOidw!Jnfizl;%s#4#3`}aPN{|{NtuGe;BJw{@5Nle zT`*I1u*J?pvC14>6#OMBddjHw-X~cp*^=7fw=|98u>iM9NJ^Tiu_7ozhfnBf?K!mv za5PtG^IVeZ z4Dt+MQ(Gc(>dUXozZHANF-8`G$qP#m2sg&I{D4wn#m%la?bNAveapZ8+q+6PpF8F) zuIcrtT5|TWb{5=OF>KSdgwqgKX3pMJvWcc7fZy8{x4MvS${xjOA#U!s$z@#lz6aj) zzC*8i*{-@fGLju(_?K+zQ(dixW9@FW80FJWb$A{11bA^ii5f-<%QH|aaJ&;AUl=y9 zZXWBhuXOeyN@U+PA@$ITR%P$^yB3NYy!Thh$N!frF-&^k=d^l0Ro!w76>Z zq`=Bw!$mFhVwOfG6_W|1l<)(b!R*Pdk~9G!M zy~j(7C7|-Y_QZ>V&BA$EA}yz!U`-*$$a>755|v}W=B9e~QeuL=4n6=7BP0}S zC5DXc6Xwr&A|Fu_&Fzx{?HxKQuna!F!{2}*iI)2a8jq+8*(G!Xg{yN~{`T0L_eCj3)HP>N!r6cLcet3>owx#^3hg zbQphzQO)|HtF4jU2=v9H(3vAge&}2jG$KzVy+D+^w^zg%%5#idcN_xq7g6E*DLezYh*JR-h@3{Cg zy*-BgSCPEPib7{q6@|*O`>3tPtEV?lf?v;@TN)d5$Ta18QZ$$yX4VqX+-m2iY#x5d z_R5=n1uVTMJF1Z?G$b_FkgjnzV}mp$-QSFUj<0#Aw`QKwp_HzcJP&;@bhtKuzKN^x z(gf<=(2U_!L<0{a^Nz^NY)cau?1?iSb=e-iKhPLZhR$NQr@sPjb7nA(4}a-fIyHjp zI9(xqrR+~Ej=~XXPg#PEf6Z`N)H_+Z!OqqM9ZJOj3wI9P*dBLwOyuU=V~J>{hH(E` z7MLF-IK~|q(Ryv(|pRn{syZe6gzB^lLr2w8$GDD}sg_%o`;5}wkkXB!}6itdz} zRQdio9Uv>0Af|nHs)K)Rb1Do-09RG;_@9Bya#P_xe`EORD5dmeT(_&xRiYI|0?=+w zPtkrGsLYqF8Ai+@!}5WC^om^R&iNoYAy1x|xMma-b|9rFQ#OtH`c3$e@=$(PNTrS6 z%Y$H>PJQb)V^$3gvX{B!keGHt$*XQ6fju+m((Uc_o2~Av+RBtVB1sV?=~$4G+Ggm- z6-4n#)o~}$Pd~|e%B-k)94cpC&dyYHsEBYhKCql18(5QA2pasEHXA(#AOO*)>g|Vx zQPW7z2SiQrB-B}M{h}aeB$C|VAvm$)zEiMFy;kn4kfny>OxjhQHUVYqed1MGF+w}t z_f9?EY+n>A$td2ykMTM^IUVbeGNf5xWGzFH$Lc&2Gd#Crat|sC^q`DE?GG=JuuAv* zd<=d6ItZa;DA^PDyavYo{w^!u$AH^x?KMDl>F5IwYpk|3#pvd2YL(B~%fLDzEo+vu zP$>bazjm=b{Mxcw;B^Rt_z3Z8+hevGMuFPLj&u#?+bpyz+(Ag3%pUr*)(C>WA0@IZ z6Rs}m_G(G0YF0FPEQIE!>fK7}2g1RFeVrPnP>&C{1U*0TESoCKRC{Wkru4(4en6BhCT zrolp{37Fk*ox{;!7VMkIIBqFi7`1u7N$;HnKLsFd?U42C*o)(-(lM4&6Y7Ww74E>> zgD<)rp6d>IK-D2mU=I~H&|L4!9`DXoyvwH z$8wd&{PwlR$|b zFx|l_B@%-_Fdsn1DCxyvDf0aWPaGm}3+)is6?zscWV%1pILl8>pc}!b0A>Y{YKNXI znA*o0yXhHc3-sGl<5OM3$vPO`wyNgw1bMUS*`ceUn^Lbd52xyO*~>j2TAE(4te8co zECJ|IEu@yaLfQkvw9^{ZFg4L->9fE)eIQxlGh;1L>1eu|nA}V7$M@?Ar&+KJ{M3Rx zb4yx^D;O9XJ3i=D!I|^pj5*SL+IQ7|ONNdN%RepvgJJxHcsZr(1=|ngChluq?)Xh` zGdtQTlzWsTf97_hyQ&XUdoB;*^}Cw;D@P{Dpk%bn1UiEWmv6JpKe3G!OP*3zxru8gH2i)Sk%|aYYk2vmm zN-cWMN;i((?HRujC=-`(I7Z{`Owi*I&q_CGPn#Wtl1+ifg_YP-y=M$DAoxyE-YoaT zCPytEq#HR4ulEpp*&{ZdLSP)JfUqm%3rM{?9_>DcBPnh8w~K| z*xbm&{5;K1KmxC^l#lgKJ z-nht8Xfe!zx(#(|L-e6M>d$s+h3X-8@>l~>0s7&Gq41TF5=`h~bTtzWKqAFt@4+W7 z=IyY5I1j>RzYalDsY3W6Y89~a;RsjEWr3E?<5FWBGMaibF1{;yUE1hqe?6xXy|!7g zSkvl>*Z5252vr&`dK^n4`392BM9W}O`98K=i~h?NAVIA6;Sy2JKox_Lj3Tj7b9=Uicaw{9l_K_c$pPNl}BHiZb?ZgM9}Vf+{S$(;jV@Z2mVIy!ejg3X6=(~>=!#~vrJ{iCAO=RgpbW+j*UkeMaFPM^6cH!V% zWhe6)xe9UiFiH(~<-;17w^8yw;zCoAWA}RO=AG&%<6}kNL{iPZi8R~}l9v9e!CU^6 zeerWf&SWN0pLRweWpWyyJ@F*JCF^DRt*C9!i?An@^)K(h|C;&!U!Ozx+jF)9Z|f&0 zr&@ds+jKmZQScSI>-Cj1xRz+Y`zYj}1wF(6q5Ydk8YdW1C_XuK#M*g><~#Zz6yEc?j@>Nok-OYcw5(O|79r%R*gn1yG%Pj# z;o~QQbX4+i@?=cpo}WjA;_xq@KQ()bfe+>VB9it9G)cYlSS||c{PcS{IdAa;B75l9 zAC)yce?#*QsX$@{p!f917$WJom)eR^X2>pO{o^lnP6}7#mfN}qORDpjqm@=9<>*14IMe!#MjInZ&i-)|d2hdeNmU&u zBaflv`!lSi2_)s?VB*4&YEI|({V%clml@(JbeDQtVR&r0)WZ)Pzo#cRd7<4FL}$xn zhE&0}oT0c7-0iEP!TA?ReH8So zy=1#lba+tehNsTyt*znFUO_+?ba4{9T#6y?r!th%GEEH`36e5)p6z#niiueNkV2*7 z`3H((0hD{6q+6EO>`(LF((CoZg-C7)Q}@QPi)ImOiIib}a#5Xdb9=ZTVz|4{H~dUa zUfdpz?=U^tc8_ie(8uUxvm+c5EvnPvW)k4Rc_}QyTh8E^wKM-NDaNGJXf@rHmAeyV zV06l@d02X+34T!nGSK?=#h}Zw<-VbGw;T5cSm+xHAq*`Xc0LjSWE|imKu2L}G)Jvv zS=%xwt0k9uU|HJ;fk#;Cxl^FsmKHRs={yklLE_w$U8VNR+Im^uVB11o+Thk!KIiSiZ9ePvk+n{)~R8goEExbGnxyAlVYZ%uX@k={b)RpqDO>b-_eg#taYzAOj^{qfi--n69x-j zF9pvYFx`gwJ){PPt!9-$@7k_xB$Q>HdJBwt)-{wW1W7&j!}UMqu_tVe-hTS%`-FE9 z-$cG^**f!42nO}Xsqn;(FNZ{w|2QN4$F=W&J-949^}~O%|EFdCUkbq=Gn4`4SqIR- z;K-h?A70znNnez;Zv|$~ghiF#OYLz1c(0HcPGaD0Z$z-&!r?>Wu*azj|s{^-x$c?1UotM+u9f?awBw4+?Skpq@2zjz{@b+A9?_zE6 zu7}o_%U5@z#9u5q*7lz{@qQct+*SS<^y}LvHQl6*kavIN?AUax`&yR&^{m#e;f1SF zHD7bze-p`=yZ9UEhmGBw(2JCSZz3r|4b9==U+($K(>uDBP#0}K-tJg0dPe*$;VXNy zh$|#fcqt4CBwYk|x^Uu|S3li-vOY1jWqhXW)5VD$$M=2DT1lDd?1opD+k4*UeMhJ`_NTcL3$?Kx%c!Vf{tSTn8PXnd>lL5r zUbt9qKEtt1-+**c-6FcwI>n6eLd8G%_25V?Lpow=yDDX6tsR+~Lf4+9%JjV-#LE^L z?)*b>`5u`4*kz#09wzReHea|lajs*Ygs2im(0w@k z>V$~MpE0wtP`E>HUQo=Z5Njg9@t;pa`|c<{--Ywo8W%OWeIg z=->mXCf-80pRPM%^m$4?AS5=GUVo@FU&E*xP>#V{|6B&!eVJpyaBqW?fD02zlDz;A z4Zmg=51}uRb~i}f3Hpj}eMaKK3O!0zA%%-{XJw(QOg2@DoA!EOdU~s*o%>R$DlUv&~IVJM$ zRF2IJi z)A5MFm%Q_NT@tVGSv1ub3dMx|xv z7tTTr`E4!dmw&v$|Netp$nc7ypDte7rnlDKlJ`gq`0~r{{%Ph;_a`9(c>cAEH zLGK?AXp}Ko3F(lmA$6pDS(wX7F)3C-o`YsP^i_(lib+N*0SSH=k>2mjce6D)&?dCu zUT-})^YsC)b65HeYMv<>yjJt^#r0Zb!BAv zIt)^Zc^>B8DYb~IA{6|n7K`m#L>(zFc?;&4R+5UVfddnI7mp!-AV1j!p43N<)E6V+QlHGtOF} zo`pAf1e14K%sDW8jFg`Zndp?nX7jHq*Q2{Ev-vhE@1d<_@718>3;eN#Kt3WNF-K)F=0wT~OmU!FY5pnP&7rw;-SU&oH`Yw=29&GyOUc~iYiL>^e`#|;(v8I7R zD{PD8ec2%~7Q|RhjtisR zysYn<-u+?;gwrkM8V$>DJVNrPAV@6KfGT%q=O}JqW*_50j2XZS!yZ3y20&B>da5)y zzFu_ZfM7yfusA_j5EiM>Fo7 z&VzZMxwq!^5M&AXaPFaAf(z#=X|>oiX)2_&gkh9Sq~hiPohGTr``a^6CPR!@8<21z z(lF&!KPuc$k5J3@@mz8-4$Udgq^i<1qqB@99@r#mq4nnM?GkV@EL=3z7o87Z|Lpor z1OORsUli!``ugodxSLrqh8YC?<>6v)h} zS*Xpf%Pni{0}n#VppKT&F+Y{`*Cui~!}4Dy9ZbacO1pFj4jWt7=#8w&$CyB)=X6Tq zr+naX-Oz>S#PY=CjLDMlFpoe*nZdD5r|z61UxxU=_@T?_09cNlUtvSprtf%>{=l2d^^uoe8xA+Mf*}xgJSi@L&xgIzOhZ7YmEaIIjzZ zw(DSI9Ib(Bp=f3a`$$(%_>pwEq#yd+iI#}|V;=_CEtrJIg)G;?a=yafZQI{Np+5~c zcY~p7a!DuDGp#mWxN_{M)D5|_;Hx9u`QJo5P+ct!nH7p ze3>X~jpO|;Z>P|1!Ia6KyBH<)O!gOmW-`MV44^yIO4lZ6FcOUGJUnA#`n5D^<@Z$H z7h=IoqKoC-p^Ct~4A#T<UwR=-_CWRCOV+?gbYdYv02)ZD$U0Q?>0Bf zGuT7Di@1I2#|F5-uB%tR(2ApoxE1XNxE%HJh7ht5GU>bUg#a7l1reJ$Wl)oZi?9$1 zB@$Q_Qez~M?mk3R&ToQiu?k5rf2L&6wF(}sZJRUr?$`n_k?&%iZL#jAI*x?V#dNO= zV3LnEp8h*#{*NZB6ef2>$1R}Vy`8rxaCpOD$v9TpBz%~b^brY1zg3y0lJ-5j7RfiQXmt~uaPQQM zdk|v}G6Bvh`aU$mm8$hvvK=omhD)XU-f7 zwRq1C4gD=Byv~mF?oBy~kQ)WI_OO^EV!{mZifqFhhG5$!adbr2qx6Z8@S_zmdOf^i zGCqeVJ5*!TIV1_JLG+7W=dPzUWuZTa9~^n{q|U?t$dgW%z+YO(P3HUpya#cn2~kQN zGs?w?_u4%~x1_dQsR-jWbsH<_(TB&lev%Zh1)GcKRC_FIfy1AZxVeP;hb%-8yFQmLQ zwHgR-jH>SL@u~+*LF2iQqjby)aA4~;$!fwzEqZ03LoN_JIi_c=!@c`f`=V|{*|zD( z=N_)WG$f?{3cV0xVOY&2a{8!34~}&A0;p>W9Rx)*HV*>w!=8Bn==laiqYts8R1%&^ zAy1<66tTT7QwEyZk^1)!1CQu+J%%j{j;RSYnIq?&)Zpy-cu%gqPuOIJ`9nCcHC_!X ze$++(z{5TIml#{F;T%o^2vPPZsRmE1JkjP{)4lgPEwWbMM$TWRz@3`XY_MjOllY|S zYAs?jd3fKEN`V>hIoc}FO(7jytAv~*OL5MwTqYSH^xUsQGU! zNeZcCe!e%CgS0mdl}*OsVEb*aRKHq}G+dBTNr%AXrD>WF5L+2R59bP5PEanl(e+Lb)C9(?Vt6M5u zjI2zSn16A5(zDRN$2_xVpE8;rgfm zCC}5+Gm$RD&p%#DQo6MF`b)fTI=P+XBj>#rbM0T~A&%B9Ln9f6XJ6 zY+^aoz+WQjZuF@8YD?2vT6w@7tZHM?^f>0EZ1>s zihsVf1hJsOJG>>%56c1kh0t9xVC>9oUd!+zCHO?bEl=~h&Y33;@rd5})xL%=pCZS@ zU1%oj32wJoR)RTTOYh)MHIu_8t8Ifa*2)N8w z^Qx2BK{l_XuTz4`v7kALYglTXZY}N$SwHO*qAxl)r$RfMnW=WO{vdU9D!sZo!#9#V zX~%GyCog#+CX`WWVN65~o_RyFFN{nK;Q}@r)5K+{=};k%rp$E* None: + for item in contributions: + if item.table == "LiteLLM_DailyCustomerMemberSpend": + await _upsert_customer_daily(transaction, item) + if item.table == "LiteLLM_CustomerOrganizationTable": + await transaction.litellm_customerorganizationtable.update_many( + where=dict(item.key), data={"spend": {"increment": dict(item.increments)["spend"]}} + ) + if item.table == "LiteLLM_CustomerTeamTable": + await transaction.litellm_customerteamtable.update_many( + where=dict(item.key), data={"spend": {"increment": dict(item.increments)["spend"]}} + ) + if item.table == "LiteLLM_CustomerMemberTable": + await transaction.litellm_customermembertable.update_many( + where=dict(item.key), data={"spend": {"increment": dict(item.increments)["spend"]}} + ) + + +async def _upsert_customer_daily(transaction: LedgerTransaction, contribution: Contribution) -> None: + row_id: Final = _customer_daily_row_id(contribution.key) + await transaction.litellm_dailycustomermemberspend.upsert( + where={"id": row_id}, + data={ + "create": { + "id": row_id, + **dict(contribution.key), + **dict(contribution.values), + **dict(contribution.increments), + }, + "update": {name: {"increment": value} for name, value in contribution.increments}, + }, + ) + + +def _customer_daily_row_id(key: tuple[tuple[str, str], ...]) -> str: + encoded: Final = dumps(sorted(key), separators=(",", ":")).encode("utf-8") + return sha256(encoded).hexdigest() + + def _window(item: Contribution) -> WindowSpendTransaction: key: Final = dict(item.key) values: Final = dict(item.values) diff --git a/litellm/accounting/projection.py b/litellm/accounting/projection.py index d1e461efca9..2c11372d7cb 100644 --- a/litellm/accounting/projection.py +++ b/litellm/accounting/projection.py @@ -14,6 +14,7 @@ from litellm.constants import ( MODEL_INSIGHTS_TASK_TAG_PREFIX, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import GuardrailCostByUnitEntry +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, daily_spend_entity_ids from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks from litellm.proxy.db.model_usage_rollup import date_from_start_time @@ -202,10 +203,53 @@ def project(row: Mapping[str, object]) -> tuple[Contribution, ...]: *_window_contributions(row, metadata), *_model_contributions(row, metadata), *_guardrail_contributions(row, metadata), + *_customer_usage_contributions(row, metadata), ) return tuple(_capped(contribution) for contribution in contributions if _indexable(contribution)) +def _customer_usage_contributions( + row: Mapping[str, object], metadata: Mapping[str, object] +) -> tuple[Contribution, ...]: + serialized: Final = metadata.get(CUSTOMER_USAGE_METADATA_KEY) + date: Final = _date(row) + if not isinstance(serialized, str) or date is None: + return () + try: + attribution: Final = CustomerUsageAttribution.model_validate_json(serialized) + except ValidationError: + return () + counters: Final = _daily_counters(row, metadata) + identities: Final = ( + ("LiteLLM_CustomerOrganizationTable", "customer_org_id", attribution.customer_org_id), + ("LiteLLM_CustomerTeamTable", "customer_team_id", attribution.customer_team_id), + ("LiteLLM_CustomerMemberTable", "customer_member_id", attribution.customer_member_id), + ) + return ( + Contribution( + "LiteLLM_DailyCustomerMemberSpend", + ( + ("date", date), + ("customer_org_id", attribution.customer_org_id), + ("customer_team_id", attribution.customer_team_id or ""), + ("customer_member_id", attribution.customer_member_id or ""), + ("model", _text(row.get("model"))), + ("model_group", _text(row.get("model_group"))), + ("custom_llm_provider", _text(row.get("custom_llm_provider"))), + ("api_key", _text(row.get("api_key"))), + ("mcp_namespaced_tool_name", _text(row.get("mcp_namespaced_tool_name"))), + ("endpoint", _text(ROUTE_ENDPOINT_MAPPING.get(_text(row.get("call_type"))))), + ), + counters, + ), + *( + _spend(table, ((column, identity),), float(_number(row.get("spend")))) + for table, column, identity in identities + if identity is not None + ), + ) + + def _indexable(contribution: Contribution) -> bool: texts: Final = (value for _, value in (*contribution.key, *contribution.values) if value is not None) return all(len(text.encode("utf-8", "surrogatepass")) <= _MAX_INDEXED_TEXT_BYTES for text in texts) diff --git a/litellm/accounting/snapshot.py b/litellm/accounting/snapshot.py index 874b90110c1..b86f37c3f5e 100644 --- a/litellm/accounting/snapshot.py +++ b/litellm/accounting/snapshot.py @@ -31,6 +31,7 @@ _METADATA_FIELDS: Final = frozenset( "compression_savings", "internal_call_origin", "billed_retry_attempt", + "user_api_key_customer_usage", } ) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index d1ed9f7b62b..2c9d64d50dd 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -308,6 +308,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_DailyUserSpend", "LiteLLM_DailyGlobalSpend", "LiteLLM_DailyOrganizationSpend", + "LiteLLM_DailyCustomerMemberSpend", "LiteLLM_DailyEndUserSpend", "LiteLLM_DailyAgentSpend", "LiteLLM_DailyTeamSpend", diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql new file mode 100644 index 00000000000..f2746f41854 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261113000000_customer_daily_usage/migration.sql @@ -0,0 +1,47 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend" ( + "id" TEXT NOT NULL, + "date" TEXT NOT NULL, + "customer_org_id" TEXT NOT NULL, + "customer_team_id" TEXT NOT NULL DEFAULT '', + "customer_member_id" TEXT NOT NULL DEFAULT '', + "model" TEXT NOT NULL DEFAULT '', + "model_group" TEXT NOT NULL DEFAULT '', + "custom_llm_provider" TEXT NOT NULL DEFAULT '', + "mcp_namespaced_tool_name" TEXT NOT NULL DEFAULT '', + "endpoint" TEXT NOT NULL DEFAULT '', + "api_key" TEXT NOT NULL DEFAULT '', + "prompt_tokens" BIGINT NOT NULL DEFAULT 0, + "completion_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_read_input_tokens" BIGINT NOT NULL DEFAULT 0, + "cache_creation_input_tokens" BIGINT NOT NULL DEFAULT 0, + "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0, + "api_requests" BIGINT NOT NULL DEFAULT 0, + "successful_requests" BIGINT NOT NULL DEFAULT 0, + "failed_requests" BIGINT NOT NULL DEFAULT 0, + "total_response_time_ms" BIGINT NOT NULL DEFAULT 0, + "timed_requests" BIGINT NOT NULL DEFAULT 0, + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "LiteLLM_DailyCustomerMemberSpend_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_org_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_org_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_team_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_team_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_customer_member_id_date_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("customer_member_id", "date"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_api_key_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("api_key"); +CREATE INDEX IF NOT EXISTS "LiteLLM_DailyCustomerMemberSpend_model_idx" + ON "LiteLLM_DailyCustomerMemberSpend"("model"); diff --git a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma index 2c0246bb94e..bac27daa5de 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma @@ -1035,6 +1035,45 @@ model LiteLLM_DailyOrganizationSpend { } // Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyCustomerMemberSpend { + id String @id + date String + customer_org_id String + customer_team_id String @default("") + customer_member_id String @default("") + model String @default("") + model_group String @default("") + custom_llm_provider String @default("") + mcp_namespaced_tool_name String @default("") + endpoint String @default("") + api_key String @default("") + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@index([date]) + @@index([customer_org_id, date]) + @@index([customer_team_id, date]) + @@index([customer_member_id, date]) + @@index([api_key]) + @@index([model]) +} + model LiteLLM_DailyEndUserSpend { id String @id @default(uuid()) end_user_id String? diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3ee3ea92d7b..7a791ff7876 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -726,12 +726,15 @@ class LiteLLMRoutes(enum.Enum): "/jwt/key/mapping/info", "/management/v1/customer_organizations", "/management/v1/customer_organizations/{customer_org_id}", + "/management/v1/customer_organizations/daily/activity", "/management/v1/customer_organizations/{customer_org_id}/default_budgets", "/management/v1/customer_organizations/{customer_org_id}/default_budgets/{budget_id}", "/management/v1/customer_teams", "/management/v1/customer_teams/{customer_team_id}", + "/management/v1/customer_teams/daily/activity", "/management/v1/customer_members", "/management/v1/customer_members/{customer_member_id}", + "/management/v1/customer_members/daily/activity", ] + key_management_routes + mcp_management_routes @@ -3855,6 +3858,7 @@ class SpendLogsRouterMetadata(TypedDict): class SpendLogsMetadata(TypedDict): + user_api_key_customer_usage: ReadOnly[NotRequired[str | None]] actor_agent_id: ReadOnly[NotRequired[str | None]] target_agent_id: ReadOnly[NotRequired[str | None]] billing_agent_id: ReadOnly[NotRequired[str | None]] diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 7d413121b56..16b59657d80 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -354,6 +354,7 @@ async def _sign_in( response = await generate_key_helper_fn( llm_router=None, request_type="key", + actor=None, **{ "user_role": LitellmUserRoles.PROXY_ADMIN, "duration": LITELLM_UI_SESSION_DURATION, diff --git a/litellm/proxy/common_utils/customer_usage.py b/litellm/proxy/common_utils/customer_usage.py new file mode 100644 index 00000000000..f356fc0f2bf --- /dev/null +++ b/litellm/proxy/common_utils/customer_usage.py @@ -0,0 +1,187 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from typing import Annotated, Final, Protocol, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +from litellm.repositories.prisma_protocols import TableActions + +CustomerIdentifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, max_length=256)] +_IDENTITY_FIELDS: Final = ("customer_org_id", "customer_team_id", "customer_member_id") +CUSTOMER_USAGE_METADATA_KEY: Final = "user_api_key_customer_usage" +_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +class CustomerUsageInput(BaseModel): + model_config = ConfigDict(frozen=True) + + customer_org_id: CustomerIdentifier | None = None + customer_team_id: CustomerIdentifier | None = None + customer_member_id: CustomerIdentifier | None = None + + +class CustomerUsageAttribution(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + customer_org_id: CustomerIdentifier + customer_team_id: CustomerIdentifier | None = None + customer_member_id: CustomerIdentifier | None = None + + +class _Team(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + parent_org_id: str + + +class _Member(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + parent_team_id: str + + +class _KeyMetadata(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + metadata: object = None + + +class CustomerUsageDatabase(Protocol): + @property + def litellm_customerorganizationtable(self) -> TableActions[object]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[object]: ... + + @property + def litellm_customermembertable(self) -> TableActions[object]: ... + + +class CustomerUsageError(str, Enum): + INVALID = "Customer usage metadata contains invalid identifiers." + MISMATCH = "Customer usage identifiers must belong to the same organization and team." + FORBIDDEN = "Customer usage must match the customer organization assigned to this API key." + UNBOUND = "Assign a customer organization to the API key before tracking customer usage." + NOT_FOUND = "The customer usage organization, team, or member does not exist." + UNAVAILABLE = "Customer usage tracking requires a database and spend accounting ledger." + KEY_BINDING_FORBIDDEN = "Only proxy admins can change a key's customer organization, team, or member binding." + + +@dataclass(frozen=True, slots=True) +class CustomerUsageFailure: + error: CustomerUsageError + + +def normalize_customer_key_metadata( + metadata: Mapping[str, object] | None, + existing_metadata: Mapping[str, object] | None, + *, + administrator: bool, +) -> Mapping[str, object] | CustomerUsageFailure | None: + if administrator: + return metadata + stored: Final = existing_metadata or {} + requested: Final = metadata or {} + if any(name in requested and requested[name] != stored.get(name) for name in _IDENTITY_FIELDS): + return CustomerUsageFailure(CustomerUsageError.KEY_BINDING_FORBIDDEN) + bindings: Final = {name: stored[name] for name in _IDENTITY_FIELDS if stored.get(name) is not None} + if not bindings: + return metadata + return {**requested, **bindings} + + +def _input(metadata: Mapping[str, object], binding: Mapping[str, object]) -> CustomerUsageInput | CustomerUsageFailure: + if any( + name in metadata and name in binding and metadata[name] != binding[name] + for name in _IDENTITY_FIELDS + if binding.get(name) is not None + ): + return CustomerUsageFailure(CustomerUsageError.FORBIDDEN) + try: + return CustomerUsageInput.model_validate( + {**metadata, **{name: binding[name] for name in _IDENTITY_FIELDS if binding.get(name) is not None}} + ) + except ValidationError: + return CustomerUsageFailure(CustomerUsageError.INVALID) + + +def _hierarchy_failure(error: CustomerUsageError, *, administrator: bool) -> CustomerUsageFailure: + if administrator: + return CustomerUsageFailure(error) + return CustomerUsageFailure(CustomerUsageError.FORBIDDEN) + + +async def resolve_customer_usage( + *, + metadata: Mapping[str, object], + key_metadata: Mapping[str, object], + administrator: bool, + database: CustomerUsageDatabase | None, + ledger_enabled: bool, +) -> CustomerUsageAttribution | CustomerUsageFailure | None: + if not customer_usage_requested(metadata, key_metadata): + return None + candidate: Final = _input(metadata, key_metadata) + if isinstance(candidate, CustomerUsageFailure): + return candidate + if not administrator and key_metadata.get("customer_org_id") is None: + return CustomerUsageFailure(CustomerUsageError.UNBOUND) + if database is None or not ledger_enabled: + return CustomerUsageFailure(CustomerUsageError.UNAVAILABLE) + member: Final = ( + await database.litellm_customermembertable.find_unique( + where={"customer_member_id": candidate.customer_member_id} + ) + if candidate.customer_member_id is not None + else None + ) + if candidate.customer_member_id is not None and member is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + member_team: Final = _Member.model_validate(member).parent_team_id if member is not None else None + if candidate.customer_team_id is not None and member_team is not None and candidate.customer_team_id != member_team: + return _hierarchy_failure(CustomerUsageError.MISMATCH, administrator=administrator) + team_id: Final = candidate.customer_team_id or member_team + team: Final = ( + await database.litellm_customerteamtable.find_unique(where={"customer_team_id": team_id}) + if team_id is not None + else None + ) + if team_id is not None and team is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + team_org: Final = _Team.model_validate(team).parent_org_id if team is not None else None + if candidate.customer_org_id is not None and team_org is not None and candidate.customer_org_id != team_org: + return _hierarchy_failure(CustomerUsageError.MISMATCH, administrator=administrator) + organization_id: Final = candidate.customer_org_id or team_org + if organization_id is None: + return CustomerUsageFailure(CustomerUsageError.INVALID) + organization: Final = await database.litellm_customerorganizationtable.find_unique( + where={"customer_org_id": organization_id} + ) + if organization is None: + return _hierarchy_failure(CustomerUsageError.NOT_FOUND, administrator=administrator) + return CustomerUsageAttribution( + customer_org_id=organization_id, + customer_team_id=team_id, + customer_member_id=candidate.customer_member_id, + ) + + +def customer_usage_requested(metadata: Mapping[str, object], key_metadata: Mapping[str, object]) -> bool: + return any(metadata.get(name) is not None or key_metadata.get(name) is not None for name in _IDENTITY_FIELDS) + + +def customer_usage_fields(value: object) -> Mapping[str, object]: + return _FIELDS.validate_python(value) if isinstance(value, Mapping) else {} + + +def customer_usage_key_fields(key: object) -> Mapping[str, object]: + return customer_usage_fields(_KeyMetadata.model_validate(key).metadata) + + +def customer_usage_http_status(error: CustomerUsageError) -> int: + if error == CustomerUsageError.UNAVAILABLE: + return 503 + if error in (CustomerUsageError.UNBOUND, CustomerUsageError.FORBIDDEN, CustomerUsageError.KEY_BINDING_FORBIDDEN): + return 403 + return 400 diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8abc91a72ed..5ba324e7154 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -59,6 +59,7 @@ from litellm.litellm_core_utils.core_helpers import ( iter_items, keyword_arguments, normalize_drop_params, + require_attributes, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -93,6 +94,16 @@ from litellm.proxy.common_utils.callback_utils import ( key_logging_entries, strip_callback_config, ) +from litellm.proxy.common_utils.customer_usage import ( + CUSTOMER_USAGE_METADATA_KEY, + CustomerUsageDatabase, + CustomerUsageFailure, + customer_usage_fields, + customer_usage_http_status, + customer_usage_key_fields, + customer_usage_requested, + resolve_customer_usage, +) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export detach_request_parsed_body, @@ -2712,7 +2723,7 @@ async def add_litellm_data_to_request( """ - from litellm.proxy.proxy_server import llm_router, premium_user + from litellm.proxy.proxy_server import llm_router, premium_user, prisma_client from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields # Strip internal-only keys from user input before the proxy sets its own. @@ -2989,6 +3000,33 @@ async def add_litellm_data_to_request( if _metadata_variable_name == "metadata": data.pop("litellm_metadata", None) + _customer_metadata: Final = customer_usage_fields(_REQUEST_FIELDS.validate_python(data)[_metadata_variable_name]) + _customer_key_metadata: Final = customer_usage_key_fields(user_api_key_dict) + _customer_usage: Final = await resolve_customer_usage( + metadata=_customer_metadata, + key_metadata=_customer_key_metadata, + administrator=user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN, + database=require_attributes( + prisma_client.db, + CustomerUsageDatabase, + "litellm_customerorganizationtable", + "litellm_customerteamtable", + "litellm_customermembertable", + ) + if prisma_client is not None and customer_usage_requested(_customer_metadata, _customer_key_metadata) + else None, + ledger_enabled=general_settings_view is None + or general_settings_view.get("use_spend_logs_ledger", True) is not False, + ) + if isinstance(_customer_usage, CustomerUsageFailure): + raise HTTPException( + status_code=customer_usage_http_status(_customer_usage.error), detail=_customer_usage.error.value + ) + if _customer_usage is not None: + data[_metadata_variable_name][CUSTOMER_USAGE_METADATA_KEY] = ( # rebind-ok: data is an out-param + _customer_usage.model_dump_json() + ) + data = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( data=data, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index f973fadb53c..a2c32cdb008 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -79,9 +79,11 @@ from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401 # l from litellm.proxy.management_endpoints.key_management_endpoints import ( # noqa: F401 # legacy module exports _check_permissions_caller_permission, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export check_permissions_caller_permission, + customer_key_actor, generate_key_helper_fn, prepare_metadata_fields, raise_on_logging_credential_change, + validated_customer_key_metadata, ) from litellm.proxy.management_helpers.bulk_user_deletion import delete_users_from_current_teams from litellm.proxy.management_helpers.object_permission_utils import ( # noqa: F401 # legacy module exports @@ -560,17 +562,15 @@ async def new_user( }' ``` """ + actor: Final = customer_key_actor(user_api_key_dict) + if data.auto_create_key and actor is not None: + validated_customer_key_metadata(data.metadata, None, actor) try: from litellm.proxy.proxy_server import _license_check, prisma_client if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) - if prisma_client is None: - raise HTTPException( - status_code=500, - detail=CommonProxyErrors.db_not_connected_error.value, - ) validate_budget_duration(data.budget_duration) # Check for duplicate user_id or email @@ -629,7 +629,12 @@ async def new_user( teams = check_if_default_team_set() organization_ids: Final = cast(list[str] | None, data_json.pop("organizations", None)) - response: Final = await generate_key_helper_fn(request_type="user", **data_json, llm_router=None) + response: Final = await generate_key_helper_fn( + request_type="user", + **data_json, + llm_router=None, + actor=actor, + ) created_user_id: Final = cast(str | None, response.get("user_id", None)) if created_user_id is not None: forget_missing_user(created_user_id) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 3bd17c4b5d8..b6f9dd0ef55 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -82,6 +82,12 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) +from litellm.proxy.common_utils.customer_usage import ( + CustomerUsageFailure, + customer_usage_fields, + customer_usage_http_status, + normalize_customer_key_metadata, +) from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time, keeps_due_budget_reset @@ -1479,6 +1485,25 @@ async def _stored_user_role(prisma_client: PrismaClient | None, user_id: str | N return user_row.user_role if user_row is not None else None +def validated_customer_key_metadata( + metadata: object, + existing_metadata: object, + actor: UserAPIKeyAuth | None, +) -> Mapping[str, object] | None: + binding: Final = normalize_customer_key_metadata( + customer_usage_fields(metadata) if metadata is not None else None, + customer_usage_fields(existing_metadata) if existing_metadata is not None else None, + administrator=actor is not None and actor.user_role == LitellmUserRoles.PROXY_ADMIN.value, + ) + if isinstance(binding, CustomerUsageFailure): + raise HTTPException(status_code=customer_usage_http_status(binding.error), detail=binding.error.value) + return binding + + +def customer_key_actor(value: object) -> UserAPIKeyAuth | None: + return value if isinstance(value, UserAPIKeyAuth) else None + + async def _common_key_generation_helper( data: GenerateKeyRequest, user_api_key_dict: UserAPIKeyAuth, @@ -1493,6 +1518,8 @@ async def _common_key_generation_helper( prisma_client, ) + validated_customer_key_metadata(data.metadata, None, user_api_key_dict) + acting_as_team_admin: Final = _team_admin_requests_member_key( data=data, team_table=team_table, @@ -1857,7 +1884,7 @@ async def _common_key_generation_helper( try: response = await generate_key_helper_fn( - request_type="key", **data_json, table_name="key", llm_router=llm_router + request_type="key", **data_json, table_name="key", llm_router=llm_router, actor=user_api_key_dict ) except KeyProjectBindingError: if prisma_client is not None: @@ -3062,6 +3089,7 @@ async def prepare_key_update_data( *, prisma_client: PrismaClient | None = None, llm_router: Router | None = None, + user_api_key_dict: UserAPIKeyAuth | None = None, ): if data.router_settings is not None or ( "router_settings" not in data.model_fields_set @@ -3080,6 +3108,13 @@ async def prepare_key_update_data( llm_router=llm_router, ) data_json: Final[dict] = data.model_dump(exclude_unset=True) + if "metadata" in data_json: + customer_binding: Final = validated_customer_key_metadata( + data.metadata, + _StoredKeyMetadata.model_validate(existing_key_row, from_attributes=True).metadata, + user_api_key_dict, + ) + data_json["metadata"] = dict(customer_binding) if customer_binding is not None else None data_json.pop("key", None) data_json.pop("new_key", None) data_json.pop("grace_period", None) # Request-only param, not a DB column @@ -3477,7 +3512,11 @@ async def _process_single_key_update( # Prepare update data raise_on_logging_credential_change(key_request.metadata, existing_key_row, user_api_key_dict) non_default_values = await prepare_key_update_data( - data=key_request, existing_key_row=existing_key_row, prisma_client=prisma_client, llm_router=llm_router + data=key_request, + existing_key_row=existing_key_row, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) await _enforce_custom_key_policy( @@ -4231,7 +4270,11 @@ async def update_key_fn( _enforce_upperbound_key_params(data, fill_defaults=False) raise_on_logging_credential_change(data.metadata, existing_key_row, user_api_key_dict) non_default_values: Final = await prepare_key_update_data( - data=data, existing_key_row=existing_key_row, prisma_client=prisma_client, llm_router=llm_router + data=data, + existing_key_row=existing_key_row, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) # Only validate key_alias format if it's actually being changed @@ -5384,7 +5427,9 @@ async def generate_key_helper_fn( budget_limits: list | None = None, # multiple concurrent budget windows *, llm_router: Router | None = None, + actor: UserAPIKeyAuth | None = None, ): + customer_metadata: Final = validated_customer_key_metadata(_DECODED_JSON.validate_python(metadata), None, actor) from litellm.proxy.proxy_server import premium_user, prisma_client if prisma_client is None: @@ -5436,7 +5481,7 @@ async def generate_key_helper_fn( router_settings_json: Final = safe_dumps(router_settings) if router_settings is not None else safe_dumps({}) metadata_json: Final = metadata_json_with_limits( - metadata, + customer_metadata, model_rpm_limit=model_rpm_limit, model_tpm_limit=model_tpm_limit, mcp_rpm_limit=mcp_rpm_limit, @@ -6419,7 +6464,11 @@ async def _execute_virtual_key_regeneration( _enforce_upperbound_key_params(data, fill_defaults=False) raise_on_logging_credential_change(data.metadata, key_in_db, user_api_key_dict) non_default_values = await prepare_key_update_data( - data=data, existing_key_row=key_in_db, prisma_client=prisma_client, llm_router=llm_router + data=data, + existing_key_row=key_in_db, + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, ) # Only validate key_alias format if it's actually being changed new_key_alias: Final = non_default_values.get("key_alias") diff --git a/litellm/proxy/management_endpoints/management_v1/__init__.py b/litellm/proxy/management_endpoints/management_v1/__init__.py index c42a786db0d..c4483ef7ff1 100644 --- a/litellm/proxy/management_endpoints/management_v1/__init__.py +++ b/litellm/proxy/management_endpoints/management_v1/__init__.py @@ -8,6 +8,9 @@ from litellm.proxy.management.users.endpoints import router as current_user_rout from litellm.proxy.management_endpoints.management_v1.budgets import ( router as budgets_router, ) +from litellm.proxy.management_endpoints.management_v1.customer_usage import ( + router as customer_usage_router, +) from litellm.proxy.management_endpoints.management_v1.customers import ( router as customers_router, ) @@ -24,6 +27,7 @@ from litellm.proxy.management_endpoints.management_v1.users import ( router: Final = APIRouter() router.include_router(budgets_router) router.include_router(customers_router) +router.include_router(customer_usage_router) router.include_router(spend_logs_router) router.include_router(teams_router) router.include_router(users_router) diff --git a/litellm/proxy/management_endpoints/management_v1/customer_usage.py b/litellm/proxy/management_endpoints/management_v1/customer_usage.py new file mode 100644 index 00000000000..4f0a7fc154e --- /dev/null +++ b/litellm/proxy/management_endpoints/management_v1/customer_usage.py @@ -0,0 +1,220 @@ +from collections.abc import Sequence +from enum import Enum +from functools import partial +from typing import Annotated, Final, Protocol + +from fastapi import APIRouter, Depends +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import require_attributes +from litellm.proxy.list_api.common import reject_unknown_query_params +from litellm.proxy.management_endpoints.common_daily_activity import DailySpendRecord, get_daily_activity +from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.management_endpoints.management_v1.customers import ( + raise_public, + reject_duplicate_list_params, + require_customer_admin, +) +from litellm.proxy.management_helpers.customer_hierarchy import ErrorCode, Failure, PrismaRecord +from litellm.proxy.management_helpers.utils import ( + management_endpoint_wrapper, # pyright: ignore[reportUnknownVariableType] # legacy decorator is untyped +) +from litellm.proxy.utils import PrismaClient +from litellm.repositories.chunked_in import find_many_in +from litellm.repositories.prisma_protocols import DatabaseClient, TableActions +from litellm.types.proxy.management_endpoints.common_daily_activity import SpendAnalyticsPaginatedResponse + + +class CustomerEntity(str, Enum): + ORGANIZATION = "customer_org_id" + TEAM = "customer_team_id" + MEMBER = "customer_member_id" + + +class CustomerMetadataDatabase(Protocol): + @property + def litellm_customerorganizationtable(self) -> TableActions[PrismaRecord]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[PrismaRecord]: ... + + @property + def litellm_customermembertable(self) -> TableActions[PrismaRecord]: ... + + +_TEXT: Final = TypeAdapter(str) +_OPTIONAL_TEXT: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) + + +def get_customer_usage_prisma() -> PrismaClient: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise_public(Failure(ErrorCode.DATABASE_UNAVAILABLE)) + return prisma_client + + +def metadata_table(prisma_client: PrismaClient, entity: CustomerEntity) -> TableActions[PrismaRecord]: + database: Final = require_attributes(prisma_client, DatabaseClient, "db").db + tables: Final = require_attributes( + database, + CustomerMetadataDatabase, + "litellm_customerorganizationtable", + "litellm_customerteamtable", + "litellm_customermembertable", + ) + if entity is CustomerEntity.ORGANIZATION: + return tables.litellm_customerorganizationtable + if entity is CustomerEntity.TEAM: + return tables.litellm_customerteamtable + return tables.litellm_customermembertable + + +async def page_aliases( + table: TableActions[PrismaRecord], entity: CustomerEntity, records: Sequence[DailySpendRecord] +) -> dict[str, dict[str, object]]: # mutable-ok: get_daily_activity requires dictionary-returning metadata callbacks + identifiers: Final = tuple(_OPTIONAL_TEXT.validate_python(getattr(row, entity.value, None)) for row in records) + rows: Final = await find_many_in(table, entity.value, tuple(value for value in identifiers if value)) + values: Final = tuple(row.model_dump() for row in rows) + return { + _TEXT.validate_python(row[entity.value]): { + "alias": _OPTIONAL_TEXT.validate_python(row.get("alias")) or _TEXT.validate_python(row[entity.value]) + } + for row in values + } + + +async def customer_activity( + prisma_client: PrismaClient, + entity: CustomerEntity, + entity_ids: str | None, + excluded_ids: str | None, + start_date: str | None, + end_date: str | None, + model: str | None, + api_key: str | None, + page: int, + page_size: int, +) -> SpendAnalyticsPaginatedResponse: + return await get_daily_activity( + prisma_client=prisma_client, + table_name="litellm_dailycustomermemberspend", + entity_id_field=entity.value, + entity_id=entity_ids.split(",") if entity_ids else None, + entity_metadata_field=None, + exclude_entity_ids=excluded_ids.split(",") if excluded_ids else None, + start_date=start_date, + end_date=end_date, + model=model, + api_key=api_key, + page=page, + page_size=page_size, + resolve_entity_metadata=partial(page_aliases, metadata_table(prisma_client, entity), entity), + ) + + +router: Final = APIRouter( + prefix=MANAGEMENT_V1_PREFIX, + tags=["customer usage"], + dependencies=( + Depends(require_customer_admin), + Depends(reject_unknown_query_params), + Depends(reject_duplicate_list_params), + ), +) + + +@router.get( + "/customer_organizations/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer organization activity with model, provider, key and organization breakdowns. " + "Filters accept comma-separated organization IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_organization_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_org_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_org_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.ORGANIZATION, + customer_org_ids, + exclude_customer_org_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) + + +@router.get( + "/customer_teams/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer team activity with model, provider, key and team breakdowns. " + "Filters accept comma-separated team IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_team_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_team_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_team_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.TEAM, + customer_team_ids, + exclude_customer_team_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) + + +@router.get( + "/customer_members/daily/activity", + response_model=SpendAnalyticsPaginatedResponse, + description="Get paginated daily customer member activity with model, provider, key and member breakdowns. " + "Filters accept comma-separated member IDs. Proxy admins only.", +) +@management_endpoint_wrapper +async def customer_member_daily_activity( + prisma_client: Annotated[PrismaClient, Depends(get_customer_usage_prisma)], + customer_member_ids: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + model: str | None = None, + api_key: str | None = None, + page: int = 1, + page_size: int = 10, + exclude_customer_member_ids: str | None = None, +) -> SpendAnalyticsPaginatedResponse: + return await customer_activity( + prisma_client, + CustomerEntity.MEMBER, + customer_member_ids, + exclude_customer_member_ids, + start_date, + end_date, + model, + api_key, + page, + page_size, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b37eede1745..f9d4d5704d8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -19970,6 +19970,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str: response: Final = await generate_key_helper_fn( llm_router=llm_router, request_type="key", + actor=None, **{ "user_role": user_obj.user_role, "duration": LITELLM_UI_SESSION_DURATION, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 2c0246bb94e..bac27daa5de 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1035,6 +1035,45 @@ model LiteLLM_DailyOrganizationSpend { } // Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyCustomerMemberSpend { + id String @id + date String + customer_org_id String + customer_team_id String @default("") + customer_member_id String @default("") + model String @default("") + model_group String @default("") + custom_llm_provider String @default("") + mcp_namespaced_tool_name String @default("") + endpoint String @default("") + api_key String @default("") + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@index([date]) + @@index([customer_org_id, date]) + @@index([customer_team_id, date]) + @@index([customer_member_id, date]) + @@index([api_key]) + @@index([model]) +} + model LiteLLM_DailyEndUserSpend { id String @id @default(uuid()) end_user_id String? diff --git a/litellm/repositories/accounting_delivery_repository.py b/litellm/repositories/accounting_delivery_repository.py index 624c2b8851a..e172e863b63 100644 --- a/litellm/repositories/accounting_delivery_repository.py +++ b/litellm/repositories/accounting_delivery_repository.py @@ -14,6 +14,18 @@ if TYPE_CHECKING: class LedgerTransaction(Protocol): + @property + def litellm_dailycustomermemberspend(self) -> TableActions[object]: ... + + @property + def litellm_customerorganizationtable(self) -> TableActions[object]: ... + + @property + def litellm_customerteamtable(self) -> TableActions[object]: ... + + @property + def litellm_customermembertable(self) -> TableActions[object]: ... + @property def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ... diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 9aa0950785b..d8b4d1eb8d5 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -96,6 +96,9 @@ class _DailyActivityDatabase(Protocol): @property def litellm_dailyenduserspend(self) -> TableActions[DailyActivityRow]: ... + @property + def litellm_dailycustomermemberspend(self) -> TableActions[DailyActivityRow]: ... + @property def litellm_dailyagentspend(self) -> TableActions[DailyActivityRow]: ... @@ -178,6 +181,8 @@ def _daily_rows_table( return prisma_client.db.litellm_dailyorganizationspend if table is DailyActivityTable.CUSTOMER: return prisma_client.db.litellm_dailyenduserspend + if table is DailyActivityTable.CUSTOMER_MEMBER: + return prisma_client.db.litellm_dailycustomermemberspend if table is DailyActivityTable.AGENT: return prisma_client.db.litellm_dailyagentspend assert_never(table) diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index dc3ca19bb30..4b575761e19 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -43,6 +43,7 @@ PRISMA_TO_PG_TABLE: Final[Mapping[DailyActivityTable, str]] = MappingProxyType( DailyActivityTable.TAG: "LiteLLM_DailyTagSpend", DailyActivityTable.ORGANIZATION: "LiteLLM_DailyOrganizationSpend", DailyActivityTable.CUSTOMER: "LiteLLM_DailyEndUserSpend", + DailyActivityTable.CUSTOMER_MEMBER: "LiteLLM_DailyCustomerMemberSpend", DailyActivityTable.AGENT: "LiteLLM_DailyAgentSpend", } ) diff --git a/litellm/schema.prisma b/litellm/schema.prisma index 2c0246bb94e..bac27daa5de 100644 --- a/litellm/schema.prisma +++ b/litellm/schema.prisma @@ -1035,6 +1035,45 @@ model LiteLLM_DailyOrganizationSpend { } // Track daily end user (customer) spend metrics per model and key +model LiteLLM_DailyCustomerMemberSpend { + id String @id + date String + customer_org_id String + customer_team_id String @default("") + customer_member_id String @default("") + model String @default("") + model_group String @default("") + custom_llm_provider String @default("") + mcp_namespaced_tool_name String @default("") + endpoint String @default("") + api_key String @default("") + spend Float @default(0.0) + prompt_tokens BigInt @default(0) + completion_tokens BigInt @default(0) + cache_read_input_tokens BigInt @default(0) + cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) + api_requests BigInt @default(0) + successful_requests BigInt @default(0) + failed_requests BigInt @default(0) + total_response_time_ms BigInt @default(0) + timed_requests BigInt @default(0) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + + @@index([date]) + @@index([customer_org_id, date]) + @@index([customer_team_id, date]) + @@index([customer_member_id, date]) + @@index([api_key]) + @@index([model]) +} + model LiteLLM_DailyEndUserSpend { id String @id @default(uuid()) end_user_id String? diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index 8d68d92a336..babf4a6c8fd 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -12,6 +12,7 @@ class DailyActivityTable(str, Enum): TAG = "litellm_dailytagspend" ORGANIZATION = "litellm_dailyorganizationspend" CUSTOMER = "litellm_dailyenduserspend" + CUSTOMER_MEMBER = "litellm_dailycustomermemberspend" AGENT = "litellm_dailyagentspend" @@ -22,6 +23,7 @@ _ENTITY_FIELDS: Mapping[DailyActivityTable, frozenset[str]] = MappingProxyType( DailyActivityTable.TAG: frozenset(("tag",)), DailyActivityTable.ORGANIZATION: frozenset(("organization_id",)), DailyActivityTable.CUSTOMER: frozenset(("end_user_id",)), + DailyActivityTable.CUSTOMER_MEMBER: frozenset(("customer_org_id", "customer_team_id", "customer_member_id")), DailyActivityTable.AGENT: frozenset(("agent_id",)), } ) diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerMemberSpend.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerMemberSpend.tsx new file mode 100644 index 00000000000..dfcd9470764 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerMemberSpend.tsx @@ -0,0 +1,166 @@ +"use client"; + +import { useMemo } from "react"; +import { parseAsString, parseAsInteger, useQueryStates } from "nuqs"; +import { Button } from "@/components/ui/button"; +import { useTranslation } from "react-i18next"; +import { Search } from "lucide-react"; +import { formatCompact, formatUsd } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { Input } from "@/components/ui/input"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { cn } from "@/lib/cva.config"; +import type { CustomerHierarchyUsage, CustomerUsageEntity } from "./customerUsageApi"; + +const MEMBER_TABLE_PARSERS = { memberSearch: parseAsString.withDefault(""), memberPage: parseAsInteger.withDefault(1) }; +const PAGE_SIZE = 20; +const memberStatus = (member: CustomerUsageEntity) => { + if (member.budgetLimit === null) return "customerPortal.analytics.noCap" as const; + if (member.spend > member.budgetLimit) return "customerPortal.analytics.overBudget" as const; + if (member.spend === member.budgetLimit) return "customerPortal.analytics.atBudget" as const; + return "customerPortal.analytics.withinBudget" as const; +}; + +export default function CustomerMemberSpend({ + data, + onMemberClick, +}: { + data: CustomerHierarchyUsage; + onMemberClick: (id: string) => void; +}) { + const { t } = useTranslation(); + const [table, setTable] = useQueryStates(MEMBER_TABLE_PARSERS); + const search = table.memberSearch; + const usage = useMemo(() => new Map(data.members.map((member) => [member.id, member])), [data.members]); + const members = useMemo( + () => + data.memberBudgets + .filter((member) => member.label.toLocaleLowerCase().includes(search.toLocaleLowerCase())) + .map((member) => ({ ...member, rangeUsage: usage.get(member.id) })) + .sort((a, b) => (b.rangeUsage?.spend ?? 0) - (a.rangeUsage?.spend ?? 0) || a.label.localeCompare(b.label)), + [data.memberBudgets, search, usage], + ); + + const lastPage = Math.max(1, Math.ceil(members.length / PAGE_SIZE)); + const page = Math.max(1, Math.min(table.memberPage, lastPage)); + const start = (page - 1) * PAGE_SIZE; + return ( +
+
+
+

{t("customerPortal.analytics.memberSpend")}

+

{t("customerPortal.analytics.memberSpendHint")}

+
+
+
+
+ + + + {t("customerPortal.analytics.member")} + {t("customerPortal.analytics.spend")} + {t("customerPortal.analytics.tokens")} + {t("customerPortal.analytics.currentBudget")} + {t("customerPortal.analytics.status")} + + + + {members.slice(start, start + PAGE_SIZE).map((member) => { + const atLimit = member.budgetLimit !== null && member.spend >= member.budgetLimit; + return ( + + +
+ +
+ + {member.id} +
+
+ + {formatUsd(member.rangeUsage?.spend ?? 0)} + + {formatCompact(member.rangeUsage?.tokens ?? 0)} + + + {member.budgetLimit === null + ? t("customerPortal.analytics.noBudget") + : `${formatUsd(member.spend)} / ${formatUsd(member.budgetLimit)}`} + + + + {t(memberStatus(member))} + + + + ); + })} + {members.length === 0 && ( + + + {t(search ? "customerPortal.analytics.noMatches" : "customerPortal.analytics.noMembers")} + + + )} + +
+ {members.length > PAGE_SIZE && ( +
+ + {t("customerPortal.analytics.pagination", { + from: start + 1, + to: Math.min(start + PAGE_SIZE, members.length), + total: members.length, + })} + +
+ + +
+
+ )} +
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageHeader.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageHeader.tsx new file mode 100644 index 00000000000..618aeb0b80b --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageHeader.tsx @@ -0,0 +1,112 @@ +import type { ParseKeys } from "i18next"; +import { ChevronRight } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { formatUsd } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; +import { uiHref } from "@/utils/uiHref"; +import type { CustomerHierarchyUsage, CustomerUsageEntity, CustomerUsageScope } from "./customerUsageApi"; + +interface CustomerUsageHeaderProps { + scope: CustomerUsageScope; + data: CustomerHierarchyUsage | undefined; + dateValue: DateRangePickerValue; + preset: string | null; + onOrganizationClick: () => void; + onTeamClick: () => void; + onDateRangeChange: (value: DateRangePickerValue, preset: string | null) => void; +} + +const scopeLabel = (scope: CustomerUsageScope): ParseKeys => { + if (scope.member) return "customerPortal.analytics.memberUsage"; + if (scope.team) return "customerPortal.analytics.teamUsage"; + return "customerPortal.analytics.organizationUsage"; +}; + +function CustomerUsageBreadcrumbs({ + scope, + data, + onOrganizationClick, + onTeamClick, +}: Pick) { + const { t } = useTranslation(); + return ( + + ); +} + +function CustomerCurrentSpend({ entity }: { entity: CustomerUsageEntity }) { + const { t } = useTranslation(); + return ( +

+ {t("customerPortal.analytics.currentSpend")}{" "} + {formatUsd(entity.spend)} + {entity.budgetLimit === null + ? ` · ${t("customerPortal.analytics.noBudget")}` + : ` / ${formatUsd(entity.budgetLimit)}`} +

+ ); +} + +export default function CustomerUsageHeader({ + scope, + data, + dateValue, + preset, + onOrganizationClick, + onTeamClick, + onDateRangeChange, +}: CustomerUsageHeaderProps) { + const { t } = useTranslation(); + const entity = data?.member ?? data?.team ?? data?.organization; + const label = entity?.label ?? scope.member ?? scope.team ?? scope.org; + return ( +
+ +
+
+

{t(scopeLabel(scope))}

+

{label}

+

{t("customerPortal.analytics.subtitle")}

+ {entity && } +
+ +
+
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageRankings.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageRankings.tsx new file mode 100644 index 00000000000..aa881cb997f --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageRankings.tsx @@ -0,0 +1,200 @@ +"use client"; + +import type { ParseKeys } from "i18next"; +import { useTranslation } from "react-i18next"; +import { ChevronRight, Search } from "lucide-react"; +import { + type UsageMetric, + formatCompact, + formatUsd, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { cn } from "@/lib/cva.config"; +import type { CustomerUsageRow } from "./customerUsageData"; +import { ModelMark } from "@/app/(dashboard)/usage/_components/components/overview/BreakdownChart"; +import { colorForModel } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { providerForModel } from "@/app/(dashboard)/usage/_components/components/overview/modelProvider"; + +export const CUSTOMER_USAGE_DIMENSIONS = ["models", "api_keys", "teams"] as const; +export type CustomerUsageDimension = (typeof CUSTOMER_USAGE_DIMENSIONS)[number]; + +const DIMENSION_LABELS: Record = { + models: "customerPortal.analytics.models", + api_keys: "customerPortal.analytics.keys", + teams: "customerPortal.analytics.teams", +}; +const PAGE_SIZE = 10; + +interface CustomerUsageRankingsProps { + rows: readonly CustomerUsageRow[]; + dimension: CustomerUsageDimension; + metric: UsageMetric; + search: string; + page: number; + colorByModel: ReadonlyMap; + onSearchChange: (search: string) => void; + onPageChange: (page: number) => void; + onDrillDown: (dimension: "teams", id: string) => void; +} + +export default function CustomerUsageRankings({ + rows, + dimension, + metric, + search, + page, + colorByModel, + onSearchChange, + onPageChange, + onDrillDown, +}: CustomerUsageRankingsProps) { + const { t } = useTranslation(); + const filtered = rows.filter((row) => row.label.toLocaleLowerCase().includes(search.toLocaleLowerCase())); + const lastPage = Math.max(1, Math.ceil(filtered.length / PAGE_SIZE)); + const currentPage = Math.min(Math.max(1, page), lastPage); + const start = (currentPage - 1) * PAGE_SIZE; + const canDrillDown = dimension === "teams"; + + return ( +
+
+
+
+
+
+

{t("customerPortal.analytics.rankingHint")}

+ + + + {t("customerPortal.analytics.rank")} + {t(DIMENSION_LABELS[dimension])} + {t("customerPortal.analytics.share")} + + {t("customerPortal.analytics.spend")} + + + {t("customerPortal.analytics.tokens")} + + + {t("customerPortal.analytics.requests")} + + + + + {filtered.slice(start, start + PAGE_SIZE).map((row, index) => ( + + {start + index + 1} + + {canDrillDown && row.id ? ( + + ) : ( + + {dimension === "models" && ( + + )} + + {row.label} + {dimension === "models" && providerForModel(row.id) && ( + + {t("usage.breakdown.byProvider", { provider: providerForModel(row.id) })} + + )} + + + )} + + +
+ + + + {formatUsd(row.spend)} + + + {formatCompact(row.tokens)} + + + {formatCompact(row.requests)} + + + ))} + {filtered.length === 0 && ( + + + {t(search ? "customerPortal.analytics.noMatches" : "customerPortal.analytics.noBreakdown")} + + + )} + +
+ {filtered.length > PAGE_SIZE && ( +
+ + {t("customerPortal.analytics.pagination", { + from: start + 1, + to: Math.min(start + PAGE_SIZE, filtered.length), + total: filtered.length, + })} + +
+ + +
+
+ )} +
+
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageTrend.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageTrend.tsx new file mode 100644 index 00000000000..0ba88ea4d6d --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/CustomerUsageTrend.tsx @@ -0,0 +1,152 @@ +"use client"; + +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; +import { + bucketSeries, + bucketTotals, + formatMetricValue, + labelForDate, + type UsageMetric, + type Granularity, + type OverviewTotals, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { ChartSkeleton, Segmented } from "@/app/(dashboard)/usage/_components/components/overview/Primitives"; +import { StackedUsageChart } from "@/components/shared/charts"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import type { CustomerHierarchyUsage } from "./customerUsageApi"; +import { customerUsageSeries } from "./customerUsageChartData"; +import { CUSTOMER_USAGE_DIMENSIONS, type CustomerUsageDimension } from "./CustomerUsageRankings"; +import { ModelMark } from "@/app/(dashboard)/usage/_components/components/overview/BreakdownChart"; + +const DIMENSION_LABELS = { + models: "customerPortal.analytics.models", + api_keys: "customerPortal.analytics.keys", + teams: "customerPortal.analytics.teams", +} as const; + +interface CustomerUsageTrendProps { + data: CustomerHierarchyUsage | undefined; + dimension: CustomerUsageDimension; + onDimensionChange: (dimension: CustomerUsageDimension) => void; + totals: OverviewTotals; + metric: UsageMetric; + bucket: Granularity; + loading: boolean; + colorByModel: ReadonlyMap; + onMetricChange: (metric: UsageMetric) => void; + onBucketChange: (bucket: Granularity) => void; +} + +export default function CustomerUsageTrend({ + data, + dimension, + onDimensionChange, + totals, + metric, + bucket, + loading, + colorByModel, + onMetricChange, + onBucketChange, +}: CustomerUsageTrendProps) { + const { t } = useTranslation(); + const dailySeries = useMemo( + () => + data + ? customerUsageSeries(data, dimension, metric, colorByModel) + : { data: [], keys: [], labels: [], colors: [] }, + [data, dimension, metric, colorByModel], + ); + const series = useMemo(() => bucketSeries(dailySeries, bucket), [dailySeries, bucket]); + const bucketValues = useMemo(() => bucketTotals(series), [series]); + const empty = totals.requests === 0 && totals.tokens === 0 && totals.spend === 0; + return ( +
+
+
+

{t("customerPortal.analytics.trend")}

+

+ {t("customerPortal.analytics.trendHint", { dimension: t(DIMENSION_LABELS[dimension]).toLocaleLowerCase() })} +

+
+
+ + + label={t("customerPortal.analytics.metric")} + value={metric} + options={[ + { value: "spend", label: t("customerPortal.analytics.spend") }, + { value: "tokens", label: t("customerPortal.analytics.tokens") }, + { value: "requests", label: t("customerPortal.analytics.requests") }, + ]} + onChange={(metric) => onMetricChange(metric)} + /> + +
+ {loading && ( +
+ +
+ )} + {!loading && empty && ( +
+ {t("customerPortal.analytics.empty")} +
+ )} + {!loading && !empty && ( + <> + labelForDate(series, date)} + format={(value) => formatMetricValue(value, metric)} + totalFor={(date) => bucketValues.get(date)} + className="h-72 sm:h-80" + /> +
+ {series.labels.map((label, index) => ( + + {dimension === "models" ? ( + + ) : ( + + ))} +
+ + )} +
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.test.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.test.ts new file mode 100644 index 00000000000..72443540bab --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.test.ts @@ -0,0 +1,336 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { registerBaseUrlGetter } from "@/lib/http/runtime"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import { customerHierarchyUsageCall } from "./customerUsageApi"; + +const audit = { + created_at: "2026-11-01T00:00:00Z", + created_by: "admin", + updated_at: "2026-11-01T00:00:00Z", + updated_by: "admin", + spend: 40, +}; +const organization = { + ...audit, + customer_org_id: "org-1", + alias: "Customer organization", + budget: { max_budget: 100 }, + budget_reset_at: "2026-12-01T00:00:00Z", + default_budgets: [], +}; +const teams = [ + { ...audit, customer_team_id: "team-a", parent_org_id: "org-1", alias: "Product", budget: { max_budget: 50 } }, + { ...audit, customer_team_id: "team-b", parent_org_id: "org-1", alias: "Operations", budget: null }, +]; +const members = [ + { + ...audit, + customer_member_id: "member-a", + parent_team_id: "team-a", + alias: "Member One", + effective_budget: { max_budget: 10 }, + }, + { + ...audit, + customer_member_id: "member-b", + parent_team_id: "team-b", + alias: "Member Two", + effective_budget: { max_budget: 20 }, + }, +]; +const metrics = (spend: number, tokens: number, requests: number): SpendMetrics => ({ + spend, + prompt_tokens: tokens, + completion_tokens: 0, + total_tokens: tokens, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); +const records = [ + { + organization: "org-1", + team: "team-a", + member: "member-a", + model: "model-a", + key: "hash-a", + alias: "Assistant key", + metrics: metrics(3, 100, 8), + }, + { + organization: "org-1", + team: "team-b", + member: "member-b", + model: "model-b", + key: "hash-b", + alias: "Search key", + metrics: metrics(1, 500, 2), + }, +]; +type UsageRecord = (typeof records)[number]; +const daily = (record: UsageRecord, entityId: string): DailyData => { + const key = { metrics: record.metrics, metadata: { key_alias: record.alias, team_id: null } }; + const entity = { metrics: record.metrics, metadata: {}, api_key_breakdown: { [record.key]: key } }; + return { + date: "2026-11-01", + metrics: record.metrics, + breakdown: { + models: { [record.model]: entity }, + model_groups: { [record.model]: entity }, + providers: {}, + mcp_servers: {}, + api_keys: { [record.key]: key }, + entities: { [entityId]: entity }, + }, + }; +}; +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } }); +const requestUrl = (input: Parameters[0]) => + new URL(input instanceof Request ? input.url : String(input), "http://localhost"); +const fetchMock = vi.fn(); +const urls = () => fetchMock.mock.calls.map(([input]) => requestUrl(input)); +const dailyUrls = () => urls().filter((url) => url.pathname.endsWith("/daily/activity")); +const request = { + scope: { org: "org-1", team: null, member: null }, + accessToken: "test-session", + startDate: "2026-11-01", + endDate: "2026-11-03", +}; +const list = (data: T[], page = 1, totalPages = 1) => ({ + data, + meta: { total_count: data.length, page, page_size: 100, total_pages: totalPages }, +}); + +const activityDimension = (resource: string | undefined): "organization" | "team" | "member" => { + if (resource === "customer_organizations") return "organization"; + if (resource === "customer_teams") return "team"; + return "member"; +}; +const filterFor = (resource: string | undefined) => + ({ + organization: "customer_org_ids", + team: "customer_team_ids", + member: "customer_member_ids", + })[activityDimension(resource)]; +const activityResponse = (url: URL) => { + const collection = url.pathname.split("/").at(-3); + const dimension = activityDimension(collection); + const filter = filterFor(collection); + const ids = url.searchParams.get(filter)?.split(",") ?? []; + const selected = records.filter((row) => ids.includes(row[dimension])); + const page = Number(url.searchParams.get("page")); + const current = selected.slice(page - 1, page); + const values = current[0]?.metrics ?? metrics(0, 0, 0); + return json({ + results: current.map((row) => daily(row, row[dimension])), + metadata: { + ...EMPTY_DAILY_ACTIVITY_METADATA, + total_spend: values.spend, + total_tokens: values.total_tokens, + total_api_requests: values.api_requests, + page, + total_pages: selected.length, + has_more: page < selected.length, + }, + }); +}; + +const serve = (input: Parameters[0]): Response => { + const url = requestUrl(input); + if (url.pathname.endsWith("/daily/activity")) return activityResponse(url); + if (url.pathname === "/management/v1/customer_organizations/org-1") return json(organization); + if (url.pathname === "/management/v1/customer_teams") { + const page = Number(url.searchParams.get("page")); + return json(list([teams[page - 1]], page, 2)); + } + if (url.pathname === "/management/v1/customer_members") + return json(list(members.filter((member) => member.parent_team_id === url.searchParams.get("parent_team_id")))); + const team = teams.find((team) => url.pathname.endsWith(`/customer_teams/${team.customer_team_id}`)); + if (team) return json(team); + const member = members.find((member) => url.pathname.endsWith(`/customer_members/${member.customer_member_id}`)); + if (member) return json(member); + throw new Error(`Unexpected usage request ${url.pathname}`); +}; + +beforeEach(() => { + registerBaseUrlGetter(() => "http://localhost"); + fetchMock.mockReset(); + fetchMock.mockImplementation((input) => Promise.resolve(serve(input))); + vi.stubGlobal("fetch", fetchMock); +}); +afterEach(() => vi.unstubAllGlobals()); + +describe("customerHierarchyUsageCall", () => { + it("should read exact daily activity routes, all pages, and scoped descendant lists", async () => { + fetchMock.mockImplementation((input) => { + const url = requestUrl(input); + if (url.pathname === "/management/v1/customer_members" && url.searchParams.get("parent_team_id") === "team-a") { + const page = Number(url.searchParams.get("page")); + const member = page === 1 ? members[0] : { ...members[0], customer_member_id: "member-c" }; + return Promise.resolve(json(list([member], page, 2))); + } + return Promise.resolve(serve(input)); + }); + const result = await customerHierarchyUsageCall(request); + expect(result.metadata.total_spend).toBe(4); + expect(result.metadata.total_tokens).toBe(600); + expect(result.metadata.total_api_requests).toBe(10); + expect(result.results).toHaveLength(1); + expect(result.results[0].metrics.spend).toBe(4); + expect(result.results[0].breakdown.model_groups["model-b"].metrics.total_tokens).toBe(500); + expect(result.api_keys.map((row) => row.label).sort()).toEqual(["Assistant key", "Search key"]); + expect(result.teams.map((row) => [row.id, row.spend])).toEqual([ + ["team-a", 3], + ["team-b", 1], + ]); + expect(result.members.map((row) => [row.id, row.tokens])).toEqual([ + ["member-a", 100], + ["member-b", 500], + ]); + expect(result.teamResults).toHaveLength(1); + expect(result.teamResults[0].metrics.spend).toBe(4); + expect(result.teamResults[0].breakdown.entities["team-a"]).toMatchObject({ + metadata: { alias: "Product" }, + metrics: { spend: 3 }, + }); + expect(result.memberResults).toHaveLength(1); + expect(result.memberResults[0].breakdown.entities["member-b"].metrics.total_tokens).toBe(500); + expect(result.memberBudgets[0]).toMatchObject({ id: "member-a", spend: 40, budgetLimit: 10 }); + for (const url of dailyUrls()) { + expect(url.pathname).toMatch(/^\/management\/v1\/customer_(organizations|teams|members)\/daily\/activity$/); + expect(url.searchParams.get("start_date")).toBe(request.startDate); + expect(url.searchParams.get("end_date")).toBe(request.endDate); + expect(url.searchParams.get("page_size")).toBe("100"); + expect([...url.searchParams.keys()].filter((name) => name.endsWith("_ids"))).toHaveLength(1); + } + const organizationPages = dailyUrls().filter((url) => url.searchParams.get("customer_org_ids") === "org-1"); + expect(organizationPages.map((url) => url.searchParams.get("page"))).toEqual(["1", "2"]); + expect( + urls() + .filter((url) => url.pathname === "/management/v1/customer_teams") + .map((url) => [url.searchParams.get("parent_org_id"), url.searchParams.get("page")]), + ).toEqual([ + ["org-1", "1"], + ["org-1", "2"], + ]); + expect( + dailyUrls() + .find((url) => url.searchParams.has("customer_team_ids")) + ?.searchParams.get("customer_team_ids"), + ).toBe("team-a,team-b"); + expect( + dailyUrls() + .find((url) => url.searchParams.has("customer_member_ids")) + ?.searchParams.get("customer_member_ids"), + ).toBe("member-a,member-c,member-b"); + expect( + urls().some( + (url) => + url.pathname === "/management/v1/customer_members" && + url.searchParams.get("parent_team_id") === "team-a" && + url.searchParams.get("page") === "2", + ), + ).toBe(true); + }); + + it("should infer member ancestry and retain live spend and effective budget separately from range totals", async () => { + const result = await customerHierarchyUsageCall({ + ...request, + scope: { org: "org-1", team: null, member: "member-a" }, + }); + expect(result.team?.id).toBe("team-a"); + expect(result.member).toMatchObject({ id: "member-a", spend: 40, budgetLimit: 10 }); + expect(result.organization).toMatchObject({ spend: 40, budgetLimit: 100, budgetResetAt: "2026-12-01T00:00:00Z" }); + expect(result.metadata.total_spend).toBe(3); + expect(result.teams).toEqual([{ id: "team-a", label: "Product", spend: 3, tokens: 100, requests: 8 }]); + expect(Object.keys(result.teamResults[0].breakdown.entities)).toEqual(["team-a"]); + expect(result.teamResults[0].breakdown.entities["team-a"].metrics.spend).toBe(3); + expect(result.memberResults[0].breakdown.entities["member-a"].metrics.total_tokens).toBe(100); + expect(dailyUrls()).toHaveLength(1); + expect(dailyUrls()[0].searchParams.get("customer_member_ids")).toBe("member-a"); + expect(urls().some((url) => url.pathname === "/management/v1/customer_members")).toBe(false); + }); + + it("should reject mismatched ancestry before requesting any usage", async () => { + fetchMock.mockImplementation((input) => + Promise.resolve( + requestUrl(input).pathname.endsWith("/customer_teams/team-a") + ? json({ ...teams[0], parent_org_id: "another-org" }) + : serve(input), + ), + ); + await expect( + customerHierarchyUsageCall({ ...request, scope: { org: "org-1", team: "team-a", member: null } }), + ).rejects.toThrow("does not belong"); + expect(dailyUrls()).toHaveLength(0); + }); + + it("should never send an empty descendant filter and should retain unassigned usage", async () => { + fetchMock.mockImplementation((input) => + Promise.resolve(requestUrl(input).pathname === "/management/v1/customer_teams" ? json(list([])) : serve(input)), + ); + const result = await customerHierarchyUsageCall(request); + expect(dailyUrls().every((url) => url.searchParams.get("customer_org_ids") === "org-1")).toBe(true); + expect(result.teams).toEqual([ + { id: "", label: "Unassigned or removed team", spend: 4, tokens: 600, requests: 10 }, + ]); + expect(result.members).toEqual([ + { id: "", label: "Unassigned or removed member", spend: 4, tokens: 600, requests: 10 }, + ]); + }); + + it("should fail a snapshot when a later page fails instead of presenting partial totals", async () => { + fetchMock.mockImplementation((input) => { + const url = requestUrl(input); + return Promise.resolve( + url.searchParams.get("customer_org_ids") === "org-1" && url.searchParams.get("page") === "2" + ? json({ error: "Second page unavailable" }, 503) + : serve(input), + ); + }); + await expect(customerHierarchyUsageCall(request)).rejects.toThrow("Second page unavailable"); + }); + + it("should reject identifiers the comma-separated API cannot represent instead of broadening filters", async () => { + await expect( + customerHierarchyUsageCall({ ...request, scope: { org: "org-1,org-2", team: null, member: null } }), + ).rejects.toThrow(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("should chunk descendant filters and keep every request phase within four concurrent reads", async () => { + const manyTeams = Array.from({ length: 205 }, (_, index) => ({ ...teams[0], customer_team_id: `team-${index}` })); + let active = 0; + let maximumActive = 0; + fetchMock.mockImplementation(async (input) => { + active++; + maximumActive = Math.max(maximumActive, active); + await Promise.resolve(); + const url = requestUrl(input); + active--; + if (url.pathname === "/management/v1/customer_teams") { + const page = Number(url.searchParams.get("page")); + return json(list(manyTeams.slice((page - 1) * 100, page * 100), page, 3)); + } + if (url.pathname === "/management/v1/customer_members") return json(list([])); + if (url.pathname === "/management/v1/customer_teams/daily/activity") { + return json({ results: [], metadata: EMPTY_DAILY_ACTIVITY_METADATA }); + } + return serve(input); + }); + + await customerHierarchyUsageCall(request); + + expect(maximumActive).toBeLessThanOrEqual(4); + const teamFilters = dailyUrls() + .filter((url) => url.searchParams.has("customer_team_ids")) + .map((url) => url.searchParams.get("customer_team_ids")!.split(",")); + expect(teamFilters.map((ids) => ids.length)).toEqual([100, 100, 5]); + expect(new Set(teamFilters.flat()).size).toBe(manyTeams.length); + expect(dailyUrls().some((url) => url.searchParams.has("customer_member_ids"))).toBe(false); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.ts new file mode 100644 index 00000000000..3bfeeaebef5 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageApi.ts @@ -0,0 +1,265 @@ +import { apiClient } from "@/components/networking"; +import { type DailyActivityAggregatedResponse } from "@/components/UsagePage/dailyActivityApi"; +import type { components } from "@/lib/http/schema"; +import i18n from "@/i18n/i18n"; +import { + mergeCustomerActivityPages, + customerEntityRankings, + customerKeyRankings, + customerEntityDailyData, + type CustomerUsageRow, +} from "./customerUsageData"; + +type Organization = components["schemas"]["Organization"]; +type Team = components["schemas"]["Team"]; +type Member = components["schemas"]["CustomerMember"]; +type Collection = "customer_organizations" | "customer_teams" | "customer_members"; +type UsageRow = Omit; + +export interface CustomerUsageEntity { + id: string; + label: string; + spend: number; + budgetLimit: number | null; + budgetResetAt: string | null; +} + +export interface CustomerHierarchyUsage extends ReturnType { + organization: CustomerUsageEntity; + team: CustomerUsageEntity | null; + member: CustomerUsageEntity | null; + teams: UsageRow[]; + members: UsageRow[]; + api_keys: UsageRow[]; + teamResults: ReturnType["results"]; + memberResults: ReturnType["results"]; + memberBudgets: CustomerUsageEntity[]; +} + +export interface CustomerUsageScope { + org: string; + team: string | null; + member: string | null; +} + +interface CustomerUsageRequest { + scope: CustomerUsageScope; + accessToken: string; + startDate: string; + endDate: string; + signal?: AbortSignal; +} + +interface ResolvedScope { + organization: Organization; + team: Team | null; + member: Member | null; +} + +const PREFIX = "/management/v1"; +const PAGE_SIZE = 100; +const MAX_PAGES = 1_000; +const MAX_CONCURRENT = 4; +const ID_FILTER: Record = { + customer_organizations: "customer_org_ids", + customer_teams: "customer_team_ids", + customer_members: "customer_member_ids", +}; + +const assertIdentifiers = (ids: readonly string[]) => { + if (ids.some((id) => !id || id.includes(","))) throw new Error(i18n.t("customerPortal.analytics.invalidIdentifier")); +}; + +const assertScope = (valid: boolean) => { + if (!valid) throw new Error(i18n.t("customerPortal.analytics.scopeMismatch")); +}; + +const batches = async (items: readonly T[], task: (item: T) => Promise): Promise => { + const results: R[] = []; + for (let start = 0; start < items.length; start += MAX_CONCURRENT) { + results.push(...(await Promise.all(items.slice(start, start + MAX_CONCURRENT).map(task)))); + } + return results; +}; + +const info = (resource: Collection, id: string, request: CustomerUsageRequest) => + apiClient.get(`${PREFIX}/${resource}/${encodeURIComponent(id)}`, { + accessToken: request.accessToken, + signal: request.signal, + }); + +const resolveScope = async (request: CustomerUsageRequest): Promise => { + const { scope } = request; + assertIdentifiers([scope.org, ...[scope.team, scope.member].filter((id): id is string => id !== null)]); + const [organization, requestedTeam, member] = await Promise.all([ + info("customer_organizations", scope.org, request), + scope.team ? info("customer_teams", scope.team, request) : null, + scope.member ? info("customer_members", scope.member, request) : null, + ]); + const team = requestedTeam ?? (member ? await info("customer_teams", member.parent_team_id, request) : null); + assertScope(organization.customer_org_id === scope.org); + assertScope(!team || team.parent_org_id === scope.org); + assertScope(!scope.team || team?.customer_team_id === scope.team); + assertScope(!member || member.customer_member_id === scope.member); + assertScope(!member || member.parent_team_id === team?.customer_team_id); + return { organization, team, member }; +}; + +const listAll = async ( + resource: Collection, + parent: Record, + request: CustomerUsageRequest, +): Promise => { + const rows: T[] = []; + for (let page = 1; page <= MAX_PAGES; page++) { + const result = await apiClient.get<{ data: T[]; meta: components["schemas"]["ListMeta"] }>( + `${PREFIX}/${resource}`, + { + accessToken: request.accessToken, + signal: request.signal, + query: { ...parent, page, page_size: PAGE_SIZE }, + }, + ); + rows.push(...result.data); + if (page >= result.meta.total_pages) return rows; + } + throw new Error(i18n.t("customerPortal.analytics.incompletePages")); +}; + +const descendants = async (scope: ResolvedScope, request: CustomerUsageRequest) => { + const teams = scope.team + ? [scope.team] + : await listAll("customer_teams", { parent_org_id: request.scope.org }, request); + if (teams.some((team) => team.parent_org_id !== request.scope.org)) + throw new Error(i18n.t("customerPortal.analytics.scopeMismatch")); + const memberLists = scope.member + ? [[scope.member]] + : await batches(teams, async (team) => { + const members = await listAll("customer_members", { parent_team_id: team.customer_team_id }, request); + if (members.some((member) => member.parent_team_id !== team.customer_team_id)) + throw new Error(i18n.t("customerPortal.analytics.scopeMismatch")); + return members; + }); + return { + teams: [...new Map(teams.map((team) => [team.customer_team_id, team])).values()], + members: [...new Map(memberLists.flat().map((member) => [member.customer_member_id, member])).values()], + }; +}; + +const activityPages = async (resource: Collection, ids: readonly string[], request: CustomerUsageRequest) => { + assertIdentifiers(ids); + const pages: DailyActivityAggregatedResponse[] = []; + for (let page = 1; page <= MAX_PAGES; page++) { + const result = await apiClient.get(`${PREFIX}/${resource}/daily/activity`, { + accessToken: request.accessToken, + signal: request.signal, + query: { + start_date: request.startDate, + end_date: request.endDate, + [ID_FILTER[resource]]: ids.join(","), + page, + page_size: PAGE_SIZE, + }, + }); + if (!result.metadata) throw new Error(i18n.t("customerPortal.analytics.incompletePages")); + pages.push(result); + if (!result.metadata.has_more && page >= result.metadata.total_pages) return pages; + } + throw new Error(i18n.t("customerPortal.analytics.incompletePages")); +}; + +interface ActivityJob { + dimension: "scope" | "teams" | "members"; + resource: Collection; + ids: readonly string[]; +} + +const chunkJobs = (resource: Collection, dimension: ActivityJob["dimension"], ids: readonly string[]): ActivityJob[] => + Array.from({ length: Math.ceil(ids.length / PAGE_SIZE) }, (_, index) => ({ + resource, + dimension, + ids: ids.slice(index * PAGE_SIZE, (index + 1) * PAGE_SIZE), + })); + +const entity = (id: string, record: Organization | Team | Member): CustomerUsageEntity => ({ + id, + label: record.alias || id, + spend: record.spend, + budgetLimit: ("effective_budget" in record ? record.effective_budget : record.budget)?.max_budget ?? null, + budgetResetAt: record.budget_reset_at ?? null, +}); + +const selectedResource = (scope: ResolvedScope): Collection => { + if (scope.member) return "customer_members"; + if (scope.team) return "customer_teams"; + return "customer_organizations"; +}; + +export const customerHierarchyUsageCall = async (request: CustomerUsageRequest): Promise => { + const selected = await resolveScope(request); + const children = await descendants(selected, request); + const resource = selectedResource(selected); + const selectedId = selected.member?.customer_member_id ?? selected.team?.customer_team_id ?? request.scope.org; + const jobs: ActivityJob[] = [ + { dimension: "scope", resource, ids: [selectedId] }, + ...(!selected.team + ? chunkJobs( + "customer_teams", + "teams", + children.teams.map((team) => team.customer_team_id), + ) + : []), + ...(!selected.member + ? chunkJobs( + "customer_members", + "members", + children.members.map((member) => member.customer_member_id), + ) + : []), + ]; + const activity = await batches(jobs, async (job) => ({ + dimension: job.dimension, + pages: await activityPages(job.resource, job.ids, request), + })); + const primary = mergeCustomerActivityPages( + activity.filter((result) => result.dimension === "scope").flatMap((result) => result.pages), + ); + const teamDays = selected.team + ? primary.results + : mergeCustomerActivityPages( + activity.filter((result) => result.dimension === "teams").flatMap((result) => result.pages), + ).results; + const memberDays = selected.member + ? primary.results + : mergeCustomerActivityPages( + activity.filter((result) => result.dimension === "members").flatMap((result) => result.pages), + ).results; + const teamLabels = new Map( + children.teams.map((team) => [team.customer_team_id, team.alias || team.customer_team_id]), + ); + const memberLabels = new Map( + children.members.map((member) => [member.customer_member_id, member.alias || member.customer_member_id]), + ); + return { + ...primary, + organization: entity(selected.organization.customer_org_id, selected.organization), + team: selected.team ? entity(selected.team.customer_team_id, selected.team) : null, + member: selected.member ? entity(selected.member.customer_member_id, selected.member) : null, + teams: customerEntityRankings(teamDays, teamLabels, primary.metadata, { + unassignedLabel: i18n.t("customerPortal.analytics.unassignedTeam"), + onlyEntityId: selected.member ? selected.team?.customer_team_id : undefined, + }), + members: customerEntityRankings(memberDays, memberLabels, primary.metadata, { + unassignedLabel: i18n.t("customerPortal.analytics.unassignedMember"), + }), + api_keys: customerKeyRankings(primary.results, i18n.t("customerPortal.analytics.unnamedKey")), + teamResults: customerEntityDailyData( + primary.results, + teamDays, + teamLabels, + selected.member ? selected.team?.customer_team_id : undefined, + ), + memberResults: customerEntityDailyData(primary.results, memberDays, memberLabels), + memberBudgets: children.members.map((member) => entity(member.customer_member_id, member)), + }; +}; diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.test.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.test.ts new file mode 100644 index 00000000000..2d2a46178e4 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, it } from "vitest"; +import { + bucketTotals, + OTHER_COLOR, + type Series, + type UsageMetric, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { stackedUsageColor } from "@/components/shared/charts"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData, MetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; +import i18n from "@/i18n/i18n"; +import type { CustomerHierarchyUsage } from "./customerUsageApi"; +import { customerUsageSeries } from "./customerUsageChartData"; +import type { CustomerUsageDimension } from "./CustomerUsageRankings"; + +const metrics = (spend: number, tokens: number, requests: number): SpendMetrics => ({ + spend, + prompt_tokens: tokens, + completion_tokens: 0, + total_tokens: tokens, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); + +const entry = (spend: number, tokens: number, requests: number): MetricWithMetadata => ({ + metrics: metrics(spend, tokens, requests), + metadata: {}, + api_key_breakdown: {}, +}); + +const day = (date: string, total: SpendMetrics, breakdown: Partial = {}): DailyData => ({ + date, + metrics: total, + breakdown: { models: {}, model_groups: {}, providers: {}, mcp_servers: {}, api_keys: {}, entities: {}, ...breakdown }, +}); + +const snapshot = (): CustomerHierarchyUsage => ({ + organization: { id: "org", label: "Organization", spend: 0, budgetLimit: null, budgetResetAt: null }, + team: null, + member: null, + memberBudgets: [], + metadata: { ...EMPTY_DAILY_ACTIVITY_METADATA }, + results: [ + day("2026-11-01", metrics(100, 1_000, 100), { + models: { deployed: entry(100, 1_000, 100) }, + model_groups: { alpha: entry(60, 300, 20), beta: entry(40, 700, 80) }, + api_keys: { + "hash-a": { metrics: metrics(30, 800, 70), metadata: { key_alias: "Support", team_id: null } }, + "hash-b": { metrics: metrics(70, 200, 30), metadata: { key_alias: "Search", team_id: null } }, + }, + }), + day("2026-11-02", metrics(60, 900, 30)), + day("2026-11-03", metrics(50, 500, 40)), + ], + teamResults: [ + day("2026-11-01", metrics(75, 800, 60), { + entities: { "team-a": entry(25, 100, 40), "team-b": entry(50, 700, 20) }, + }), + day("2026-11-02", metrics(10, 90, 5), { entities: { "team-a": entry(10, 90, 5) } }), + day("2026-11-09", metrics(999, 999, 999), { entities: { "team-a": entry(999, 999, 999) } }), + ], + memberResults: [ + day("2026-11-01", metrics(80, 700, 80), { + entities: { "member-a": entry(20, 200, 30), "member-b": entry(60, 500, 50) }, + }), + ], + teams: [ + { id: "team-a", label: "Product", spend: 35, tokens: 190, requests: 45 }, + { id: "team-b", label: "Operations", spend: 50, tokens: 700, requests: 20 }, + ], + members: [ + { id: "member-a", label: "Member A", spend: 20, tokens: 200, requests: 30 }, + { id: "member-b", label: "Member B", spend: 60, tokens: 500, requests: 50 }, + ], + api_keys: [ + { id: "hash-a", label: "Support", spend: 30, tokens: 800, requests: 70 }, + { id: "hash-b", label: "Search", spend: 70, tokens: 200, requests: 30 }, + ], +}); + +const modelColors = new Map([ + ["alpha", stackedUsageColor(3)], + ["beta", stackedUsageColor(4)], +]); +const valueAt = (series: Series, label: string, date = "2026-11-01"): number => { + const index = series.labels.indexOf(label); + return Number(series.data.find((row) => row.date === date)?.[series.keys[index]] ?? 0); +}; +const colorOf = (series: Series, label: string): string => series.colors[series.labels.indexOf(label)]; +const other = () => i18n.t("usage.overview.otherSeries"); + +describe("customerUsageSeries", () => { + it.each<[CustomerUsageDimension, string, number]>([ + ["models", "alpha", 60], + ["api_keys", "Support", 30], + ["teams", "Product", 25], + ])("should use %s values without adding the other breakdowns", (dimension, label, value) => { + const series = customerUsageSeries(snapshot(), dimension, "spend", modelColors); + + expect(valueAt(series, label)).toBe(value); + expect(bucketTotals(series).get("2026-11-01")).toBe(100); + expect(series.labels).not.toContain("deployed"); + }); + + it.each<[UsageMetric, number, number, number]>([ + ["spend", 25, 50, 25], + ["tokens", 100, 700, 200], + ["requests", 40, 20, 40], + ])("should use %s for both named teams and the unattributed remainder", (metric, first, second, remainder) => { + const series = customerUsageSeries(snapshot(), "teams", metric, modelColors); + + expect(valueAt(series, "Product")).toBe(first); + expect(valueAt(series, "Operations")).toBe(second); + expect(valueAt(series, other())).toBe(remainder); + expect(colorOf(series, other())).toBe(OTHER_COLOR); + }); + + it("should preserve all selected-scope dates and totals when companion records are missing", () => { + const series = customerUsageSeries(snapshot(), "teams", "spend", modelColors); + + expect(series.data.map((row) => row.date)).toEqual(["2026-11-01", "2026-11-02", "2026-11-03"]); + expect(valueAt(series, "Product", "2026-11-02")).toBe(10); + expect(valueAt(series, other(), "2026-11-02")).toBe(50); + expect(valueAt(series, "Product", "2026-11-03")).toBe(0); + expect(valueAt(series, other(), "2026-11-03")).toBe(50); + expect([...bucketTotals(series).values()]).toEqual([100, 60, 50]); + }); + + it("should include removed and unnamed descendants in Other rather than a misleading named bucket", () => { + const data = snapshot(); + const removed = day("2026-11-01", metrics(100, 1_000, 100), { + entities: { "team-a": entry(25, 100, 40), removed: entry(10, 150, 10), "": entry(5, 50, 5) }, + }); + const series = customerUsageSeries({ ...data, teamResults: [removed] }, "teams", "tokens", modelColors); + + expect(series.labels).toEqual(["Product", other()]); + expect(valueAt(series, "Product")).toBe(100); + expect(valueAt(series, other())).toBe(900); + }); + + it("should keep spend-ranked palette colors attached to API keys when the selected metric changes", () => { + const spend = customerUsageSeries(snapshot(), "api_keys", "spend", modelColors); + const tokens = customerUsageSeries(snapshot(), "api_keys", "tokens", modelColors); + + expect(spend.labels.slice(0, 2)).toEqual(["Search", "Support"]); + expect(tokens.labels.slice(0, 2)).toEqual(["Support", "Search"]); + expect(colorOf(spend, "Search")).toBe(stackedUsageColor(0)); + expect(colorOf(tokens, "Search")).toBe(stackedUsageColor(0)); + expect(colorOf(spend, "Support")).toBe(stackedUsageColor(1)); + expect(colorOf(tokens, "Support")).toBe(stackedUsageColor(1)); + }); + + it("should use key aliases and hide hashes even when a label is missing or repeats the hash", () => { + const data = snapshot(); + const series = customerUsageSeries( + { ...data, api_keys: [{ ...data.api_keys[0], label: " hash-a " }] }, + "api_keys", + "spend", + modelColors, + ); + + expect(series.labels).not.toContain("hash-a"); + expect(series.labels).not.toContain("hash-b"); + expect(series.labels.filter((label) => label === i18n.t("customerPortal.analytics.unnamedKey"))).toHaveLength(2); + expect(bucketTotals(series).get("2026-11-01")).toBe(100); + }); + + it("should preserve model colors and fall back to the models breakdown when groups are absent", () => { + const data = snapshot(); + const color = stackedUsageColor(5); + const series = customerUsageSeries( + { ...data, results: [day("2026-11-01", metrics(7, 70, 2), { models: { deployed: entry(7, 70, 2) } })] }, + "models", + "tokens", + new Map([["deployed", color]]), + ); + + expect(series.labels).toEqual(["deployed"]); + expect(valueAt(series, "deployed")).toBe(70); + expect(colorOf(series, "deployed")).toBe(color); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.ts new file mode 100644 index 00000000000..804d50bfbe8 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageChartData.ts @@ -0,0 +1,71 @@ +import { + rankValue, + rollUpBreakdown, + seriesBy, + type Series, + type UsageMetric, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { stackedUsageColor } from "@/components/shared/charts"; +import type { DailyData } from "@/components/UsagePage/types"; +import i18n from "@/i18n/i18n"; +import type { CustomerHierarchyUsage } from "./customerUsageApi"; +import type { CustomerUsageDimension } from "./CustomerUsageRankings"; + +const TOP_SERIES = 8; + +const entityDays = ( + primary: readonly DailyData[], + entities: readonly DailyData[], + labels: ReadonlyMap, +): DailyData[] => { + const byDate = new Map(entities.map((day) => [day.date, day])); + return primary.map((day) => ({ + ...day, + breakdown: { + ...day.breakdown, + model_groups: Object.fromEntries( + Object.entries(byDate.get(day.date)?.breakdown.entities ?? {}).filter(([id]) => id && labels.has(id)), + ), + }, + })); +}; + +export const customerUsageSeries = ( + data: CustomerHierarchyUsage, + dimension: CustomerUsageDimension, + metric: UsageMetric, + colorByModel: ReadonlyMap, +): Series => { + if (dimension === "models") { + const days = data.results.map((day) => ({ + ...day, + breakdown: { + ...day.breakdown, + model_groups: + Object.keys(day.breakdown.model_groups).length > 0 ? day.breakdown.model_groups : day.breakdown.models, + }, + })); + return seriesBy(days, "model_groups", metric, { top: TOP_SERIES, colorByKey: colorByModel }); + } + + const unnamedKey = i18n.t("customerPortal.analytics.unnamedKey"); + const rows = data[dimension]; + const labels = new Map( + rows + .filter((row) => row.id) + .map((row) => { + const label = row.label.trim(); + return [row.id, dimension === "api_keys" && (!label || label === row.id) ? unnamedKey : label]; + }), + ); + const days = dimension === "api_keys" ? data.results : entityDays(data.results, data.teamResults, labels); + const breakdown = dimension === "api_keys" ? "api_keys" : "model_groups"; + const ranking = rollUpBreakdown(days, breakdown).sort((a, b) => b.spend - a.spend || a.key.localeCompare(b.key)); + const colors = new Map(ranking.map((row, index) => [row.key, stackedUsageColor(index)])); + const series = seriesBy(days, breakdown, metric, { top: TOP_SERIES, colorByKey: colors }); + const namedCount = Math.min(TOP_SERIES, ranking.filter((row) => rankValue(row, metric) > 0).length); + return { + ...series, + labels: series.labels.map((id, index) => (index < namedCount ? labels.get(id) ?? unnamedKey : id)), + }; +}; diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.test.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.test.ts new file mode 100644 index 00000000000..c8e6d638c79 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; +import type { DailyData, MetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; +import { rankCustomerModels, rankCustomerUsage, type CustomerUsageRow } from "./customerUsageData"; + +const metrics = (spend: number, tokens: number, requests: number): SpendMetrics => ({ + spend, + prompt_tokens: tokens, + completion_tokens: 0, + total_tokens: tokens, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); + +const model = (spend: number, tokens: number, requests: number): MetricWithMetadata => ({ + metrics: metrics(spend, tokens, requests), + metadata: {}, + api_key_breakdown: {}, +}); + +const day = ( + date: string, + modelGroups: Record, + models: Record = {}, +): DailyData => ({ + date, + metrics: metrics(0, 0, 0), + breakdown: { + models, + model_groups: modelGroups, + providers: {}, + mcp_servers: {}, + api_keys: {}, + entities: {}, + }, +}); + +const totals = { spend: 10, tokens: 1_000, requests: 20 }; + +describe("rankCustomerUsage", () => { + it("should sort by the selected metric and calculate shares against overall customer totals", () => { + const rows = [ + { id: "assistant", label: "Assistant", spend: 3, tokens: 100, requests: 8 }, + { id: "search", label: "Search", spend: 1, tokens: 500, requests: 2 }, + ]; + + expect(rankCustomerUsage(rows, "spend", totals).map((row) => [row.label, row.share])).toEqual([ + ["Assistant", 0.3], + ["Search", 0.1], + ]); + expect(rankCustomerUsage(rows, "tokens", totals).map((row) => [row.label, row.share])).toEqual([ + ["Search", 0.5], + ["Assistant", 0.1], + ]); + expect(rankCustomerUsage(rows, "requests", totals).map((row) => [row.label, row.share])).toEqual([ + ["Assistant", 0.4], + ["Search", 0.1], + ]); + }); + + it("should preserve zero-cost token usage while removing rows with no activity", () => { + const freeUsage = { id: "assistant", label: "Assistant", spend: 0, tokens: 200, requests: 3 }; + const inactive = { id: "inactive", label: "Inactive", spend: 0, tokens: 0, requests: 0 }; + + expect(rankCustomerUsage([inactive, freeUsage], "tokens", totals)).toEqual([{ ...freeUsage, share: 0.2 }]); + expect(rankCustomerUsage([freeUsage], "spend", totals)).toEqual([{ ...freeUsage, share: 0 }]); + }); + + it("should use stable label and id order for ties without modifying the source rows", () => { + const rows: readonly Omit[] = Object.freeze([ + { id: "b", label: "Search", spend: 1, tokens: 100, requests: 1 }, + { id: "c", label: "Assistant", spend: 1, tokens: 100, requests: 1 }, + { id: "a", label: "Assistant", spend: 1, tokens: 100, requests: 1 }, + ]); + expect(rankCustomerUsage(rows, "tokens", totals).map((row) => row.id)).toEqual(["a", "c", "b"]); + expect(rows.map((row) => row.id)).toEqual(["b", "c", "a"]); + }); + + it("should return finite zero shares when the overall metric is zero", () => { + const rows = [{ id: "assistant", label: "Assistant", spend: 0, tokens: 200, requests: 1 }]; + expect(rankCustomerUsage(rows, "spend", { spend: 0, tokens: 200, requests: 1 })[0].share).toBe(0); + expect(rankCustomerUsage([], "spend", totals)).toEqual([]); + }); +}); + +describe("rankCustomerModels", () => { + it("should sum models across days without adding model and model-group totals together", () => { + const days = [ + day("2026-11-01", { assistant: model(2, 100, 3) }, { assistant: model(2, 100, 3) }), + day("2026-11-02", { assistant: model(1, 200, 2), search: model(0, 400, 1) }), + day("2026-11-03", {}, { assistant: model(1, 100, 1) }), + ]; + expect(rankCustomerModels(days, "tokens", totals)).toEqual([ + { id: "assistant", label: "assistant", spend: 4, tokens: 400, requests: 6, share: 0.4 }, + { id: "search", label: "search", spend: 0, tokens: 400, requests: 1, share: 0.4 }, + ]); + }); + + it("should return no model rows for an empty date range", () => { + expect(rankCustomerModels([], "spend", totals)).toEqual([]); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.ts new file mode 100644 index 00000000000..afa0d3c7767 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/customerUsageData.ts @@ -0,0 +1,262 @@ +import type { DailyData, SpendMetrics, BreakdownMetrics, MetricWithMetadata } from "@/components/UsagePage/types"; +import { + EMPTY_DAILY_ACTIVITY_METADATA, + toDailyData, + type DailyActivityAggregatedResponse, + type DailyActivityMetadata, +} from "@/components/UsagePage/dailyActivityApi"; +import { + rollUpBreakdown, + type UsageMetric, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; + +export interface CustomerUsageTotals { + spend: number; + tokens: number; + requests: number; +} + +export interface CustomerUsageRow extends CustomerUsageTotals { + id: string; + label: string; + share: number; +} + +export const rankCustomerUsage = ( + rows: readonly Omit[], + metric: UsageMetric, + totals: CustomerUsageTotals, +): CustomerUsageRow[] => + rows + .filter((row) => row.spend > 0 || row.tokens > 0 || row.requests > 0) + .map((row) => ({ ...row, share: totals[metric] > 0 ? row[metric] / totals[metric] : 0 })) + .sort((a, b) => b[metric] - a[metric] || a.label.localeCompare(b.label) || a.id.localeCompare(b.id)); + +export const rankCustomerModels = ( + days: readonly DailyData[], + metric: UsageMetric, + totals: CustomerUsageTotals, +): CustomerUsageRow[] => { + const withModelGroups = days.map((day) => ({ + ...day, + breakdown: { + ...day.breakdown, + model_groups: + Object.keys(day.breakdown.model_groups).length > 0 ? day.breakdown.model_groups : day.breakdown.models, + }, + })); + return rankCustomerUsage( + rollUpBreakdown(withModelGroups, "model_groups").map((model) => ({ + id: model.key, + label: model.key, + spend: model.spend, + tokens: model.tokens, + requests: model.requests, + })), + metric, + totals, + ); +}; + +const ZERO_METRICS: SpendMetrics = { + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}; + +const sumMetrics = (values: readonly SpendMetrics[]): SpendMetrics => + values.reduce( + (total, value) => ({ + ...total, + ...Object.fromEntries( + (Object.keys(value) as (keyof SpendMetrics)[]).map((field) => [ + field, + (total[field] ?? 0) + (value[field] ?? 0), + ]), + ), + }), + ZERO_METRICS, + ); + +const mergeKeys = ( + left: BreakdownMetrics["api_keys"], + right: BreakdownMetrics["api_keys"], +): BreakdownMetrics["api_keys"] => { + const result = { ...left }; + for (const [id, entry] of Object.entries(right)) { + const previous = result[id]; + result[id] = { + ...entry, + metrics: sumMetrics([...(previous ? [previous.metrics] : []), entry.metrics]), + metadata: { + ...previous?.metadata, + ...entry.metadata, + key_alias: entry.metadata.key_alias ?? previous?.metadata.key_alias ?? null, + }, + }; + } + return result; +}; + +const mergeMetrics = ( + left: Record, + right: Record, +): Record => { + const result = { ...left }; + for (const [id, entry] of Object.entries(right)) { + const previous = result[id]; + result[id] = { + ...entry, + metrics: sumMetrics([...(previous ? [previous.metrics] : []), entry.metrics]), + metadata: { ...previous?.metadata, ...entry.metadata }, + api_key_breakdown: mergeKeys(previous?.api_key_breakdown ?? {}, entry.api_key_breakdown), + }; + } + return result; +}; + +const mergeBreakdowns = (left: BreakdownMetrics, right: BreakdownMetrics): BreakdownMetrics => ({ + models: mergeMetrics(left.models, right.models), + model_groups: mergeMetrics(left.model_groups, right.model_groups), + providers: mergeMetrics(left.providers, right.providers), + mcp_servers: mergeMetrics(left.mcp_servers, right.mcp_servers), + endpoints: mergeMetrics(left.endpoints ?? {}, right.endpoints ?? {}), + entities: mergeMetrics(left.entities, right.entities), + api_keys: mergeKeys(left.api_keys, right.api_keys), +}); + +export const mergeCustomerActivityPages = ( + pages: readonly DailyActivityAggregatedResponse[], +): { results: DailyData[]; metadata: DailyActivityMetadata } => { + const days = new Map(); + for (const day of pages.flatMap(toDailyData)) { + const previous = days.get(day.date); + days.set( + day.date, + previous + ? { + date: day.date, + metrics: sumMetrics([previous.metrics, day.metrics]), + breakdown: mergeBreakdowns(previous.breakdown, day.breakdown), + } + : day, + ); + } + const results = [...days.values()].sort((a, b) => a.date.localeCompare(b.date)); + const total = sumMetrics(results.map((day) => day.metrics)); + return { + results, + metadata: { + ...EMPTY_DAILY_ACTIVITY_METADATA, + total_spend: total.spend, + total_flat_cost: total.flat_cost ?? 0, + total_ptu_hours: total.ptu_hours ?? 0, + total_prompt_tokens: total.prompt_tokens, + total_completion_tokens: total.completion_tokens, + total_tokens: total.total_tokens, + total_api_requests: total.api_requests, + total_successful_requests: total.successful_requests, + total_failed_requests: total.failed_requests, + total_cache_read_input_tokens: total.cache_read_input_tokens, + total_cache_creation_input_tokens: total.cache_creation_input_tokens, + total_compression_saved_tokens: total.compression_saved_tokens ?? 0, + total_compression_savings_spend: total.compression_savings_spend ?? 0, + total_prompt_caching_savings_spend: total.prompt_caching_savings_spend ?? 0, + total_gateway_injected_caching_savings_spend: total.gateway_injected_caching_savings_spend ?? 0, + total_autorouter_savings_spend: total.autorouter_savings_spend ?? 0, + total_response_time_ms: total.total_response_time_ms ?? 0, + total_timed_requests: total.timed_requests ?? 0, + total_api_keys: new Set(results.flatMap((day) => Object.keys(day.breakdown.api_keys))).size, + }, + }; +}; + +const usageTotals = (value: SpendMetrics): CustomerUsageTotals => ({ + spend: value.spend, + tokens: value.total_tokens, + requests: value.api_requests, +}); +const addTotals = (left: CustomerUsageTotals, right: CustomerUsageTotals): CustomerUsageTotals => ({ + spend: left.spend + right.spend, + tokens: left.tokens + right.tokens, + requests: left.requests + right.requests, +}); + +export const customerKeyRankings = ( + days: readonly DailyData[], + unnamedKey: string, +): Omit[] => { + const keys = new Map>(); + for (const [id, entry] of days.flatMap((day) => Object.entries(day.breakdown.api_keys))) { + const previous = keys.get(id); + keys.set(id, { + id, + label: entry.metadata.key_alias?.trim() || previous?.label || unnamedKey, + ...addTotals(previous ?? { spend: 0, tokens: 0, requests: 0 }, usageTotals(entry.metrics)), + }); + } + return [...keys.values()]; +}; + +interface EntityRankingOptions { + unassignedLabel: string; + onlyEntityId?: string; +} + +export const customerEntityDailyData = ( + primary: readonly DailyData[], + days: readonly DailyData[], + labels: ReadonlyMap, + onlyEntityId?: string, +): DailyData[] => { + const byDate = new Map(days.map((day) => [day.date, day])); + return primary.map((day) => { + const entities = onlyEntityId + ? { + [onlyEntityId]: { + metrics: day.metrics, + metadata: { alias: labels.get(onlyEntityId) }, + api_key_breakdown: day.breakdown.api_keys, + }, + } + : Object.fromEntries( + Object.entries(byDate.get(day.date)?.breakdown.entities ?? {}) + .filter(([id]) => labels.has(id)) + .map(([id, entry]) => [id, { ...entry, metadata: { ...entry.metadata, alias: labels.get(id) } }]), + ); + return { ...day, breakdown: { ...day.breakdown, entities } }; + }); +}; + +export const customerEntityRankings = ( + days: readonly DailyData[], + labels: ReadonlyMap, + metadata: DailyActivityMetadata, + options: EntityRankingOptions, +): Omit[] => { + const entries = new Map(); + for (const [id, entry] of days.flatMap((day) => Object.entries(day.breakdown.entities))) { + const identity = options.onlyEntityId ?? id; + if (labels.has(identity)) + entries.set( + identity, + addTotals(entries.get(identity) ?? { spend: 0, tokens: 0, requests: 0 }, usageTotals(entry.metrics)), + ); + } + const rows = [...entries].map(([id, total]) => ({ id, label: labels.get(id)!, ...total })); + const assigned = rows.reduce(addTotals, { spend: 0, tokens: 0, requests: 0 }); + const unassigned = { + spend: Math.max(0, metadata.total_spend - assigned.spend), + tokens: Math.max(0, metadata.total_tokens - assigned.tokens), + requests: Math.max(0, metadata.total_api_requests - assigned.requests), + }; + return unassigned.spend > 0.000001 || unassigned.tokens > 0 || unassigned.requests > 0 + ? [...rows, { id: "", label: options.unassignedLabel, ...unassigned }] + : rows; +}; diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/useCustomerHierarchyUsage.ts b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/useCustomerHierarchyUsage.ts new file mode 100644 index 00000000000..05fa467d0e5 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/_components/useCustomerHierarchyUsage.ts @@ -0,0 +1,122 @@ +import { useMemo } from "react"; +import { skipToken, useQuery } from "@tanstack/react-query"; +import { parseAsInteger, parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { + dateRangeFromParams, + dateRangePatch, + defaultUsageDateRange, + usageUrlParsers, +} from "@/app/(dashboard)/usage/_components/usageUrlState"; +import { + modelColorMap, + overviewTotals, + type Granularity, + type UsageMetric, +} from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import { formatDate } from "@/components/networking"; +import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData } from "@/components/UsagePage/types"; +import { customerHierarchyUsageCall, type CustomerUsageScope } from "./customerUsageApi"; +import { rankCustomerModels, rankCustomerUsage } from "./customerUsageData"; +import { CUSTOMER_USAGE_DIMENSIONS, type CustomerUsageDimension } from "./CustomerUsageRankings"; + +const usageParsers = { + org: parseAsString, + team: parseAsString, + member: parseAsString, + metric: parseAsStringLiteral(["spend", "tokens", "requests"] as const).withDefault("spend"), + breakdown: parseAsStringLiteral(CUSTOMER_USAGE_DIMENSIONS).withDefault("models"), + bucket: parseAsStringLiteral(["day", "week"] as const).withDefault("day"), + search: parseAsString.withDefault(""), + usagePage: parseAsInteger.withDefault(1), + range: usageUrlParsers.range, + from: usageUrlParsers.from, + to: usageUrlParsers.to, +}; +const EMPTY_RESULTS: readonly DailyData[] = []; +const ORGANIZATION_PATCH = { team: null, member: null, search: null, usagePage: null }; +const TEAM_PATCH = { member: null, search: null, usagePage: null }; + +const usageRequest = ( + scope: CustomerUsageScope | null, + accessToken: string | null, + startDate: string | undefined, + endDate: string | undefined, +) => { + if (!scope || !accessToken) return null; + if (!startDate || !endDate) return null; + return { scope, accessToken, startDate, endDate }; +}; + +export default function useCustomerHierarchyUsage() { + const { accessToken } = useAuthorized(); + const [params, setParams] = useQueryStates(usageParsers); + const defaultRange = useMemo(() => defaultUsageDateRange("month_to_date"), []); + const dateValue = useMemo(() => dateRangeFromParams(params) ?? defaultRange.value, [params, defaultRange]); + const preset = params.range ?? (params.from === null && params.to === null ? defaultRange.preset : null); + const startDate = dateValue.from ? formatDate(dateValue.from) : undefined; + const endDate = dateValue.to ? formatDate(dateValue.to) : undefined; + const scope = params.org ? { org: params.org, team: params.team, member: params.member } : null; + const request = usageRequest(scope, accessToken, startDate, endDate); + const usage = useQuery({ + queryKey: ["customer-hierarchy", "usage", params.org, params.team, params.member, startDate, endDate], + queryFn: request ? ({ signal }) => customerHierarchyUsageCall({ ...request, signal }) : skipToken, + }); + const data = usage.data; + const results = data?.results ?? EMPTY_RESULTS; + const totals = useMemo(() => overviewTotals(data?.metadata ?? EMPTY_DAILY_ACTIVITY_METADATA, null), [data]); + const modelColors = useMemo(() => modelColorMap(results), [results]); + const rows = useMemo( + () => + params.breakdown === "models" + ? rankCustomerModels(results, params.metric, totals) + : rankCustomerUsage(data?.[params.breakdown] ?? [], params.metric, totals), + [data, results, params.breakdown, params.metric, totals], + ); + const drillDown = (dimension: "teams" | "members", id: string) => { + const patch = { + ...(dimension === "teams" ? { team: id, member: null } : { member: id }), + search: null, + usagePage: null, + breakdown: null, + }; + void setParams(patch, { history: "push" }); + }; + const setDateRange = (value: DateRangePickerValue, selectedPreset: string | null) => { + const { range, from, to } = dateRangePatch(value, selectedPreset); + const patch = { range, from, to, usagePage: null }; + void setParams(patch); + }; + const setDimension = (breakdown: CustomerUsageDimension) => + void setParams({ breakdown, search: null, usagePage: null }); + const selectOrganization = () => void setParams(ORGANIZATION_PATCH, { history: "push" }); + const selectTeam = () => void setParams(TEAM_PATCH, { history: "push" }); + const setMetric = (metric: UsageMetric) => void setParams({ metric, usagePage: null }); + const setBucket = (bucket: Granularity) => void setParams({ bucket }); + const setSearch = (search: string) => void setParams({ search, usagePage: null }); + const setPage = (usagePage: number) => void setParams({ usagePage }); + + return { + params, + scope, + dateValue, + preset, + usage, + data, + results, + totals, + modelColors, + rows, + drillDown, + setDateRange, + setDimension, + selectOrganization, + selectTeam, + setMetric, + setBucket, + setSearch, + setPage, + }; +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.integration.test.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.integration.test.tsx new file mode 100644 index 00000000000..852d85f3320 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.integration.test.tsx @@ -0,0 +1,379 @@ +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { registerBaseUrlGetter } from "@/lib/http/runtime"; +import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; +import type { SpendMetrics } from "@/components/UsagePage/types"; +import { + act, + chooseSelectOption, + fireEvent, + renderWithProviders, + screen, + testQueryClient, + waitFor, + within, +} from "../../../../../tests/test-utils"; +import CustomerHierarchyUsagePage from "./page"; + +const metrics = (spend: number, tokens: number, requests: number): SpendMetrics => ({ + spend, + api_requests: requests, + successful_requests: requests, + failed_requests: 0, + total_tokens: tokens, + prompt_tokens: tokens, + completion_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}); +const audit = { + created_at: "2026-11-01T00:00:00Z", + created_by: "admin", + updated_at: "2026-11-01T00:00:00Z", + updated_by: "admin", + spend: 40, +}; +const organization = { + ...audit, + customer_org_id: "org-1", + alias: "Customer organization", + budget: { max_budget: 100 }, + default_budgets: [], +}; +const teams = [ + { ...audit, customer_team_id: "team-a", parent_org_id: "org-1", alias: "Product", budget: null }, + { ...audit, customer_team_id: "team-b", parent_org_id: "org-1", alias: "Operations", budget: null }, +]; +const members = [ + { + ...audit, + customer_member_id: "member-a", + parent_team_id: "team-a", + alias: "Member One", + effective_budget: { max_budget: 10 }, + }, + { + ...audit, + customer_member_id: "member-b", + parent_team_id: "team-b", + alias: "Member Two", + effective_budget: { max_budget: 20 }, + }, +]; +const records = [ + { + organization: "org-1", + team: "team-a", + member: "member-a", + model: "model-a", + key: "hash-a", + alias: "Assistant key", + metrics: metrics(12, 100, 2), + date: "2026-11-01", + }, + { + organization: "org-1", + team: "team-b", + member: "member-b", + model: "model-b", + key: "hash-b", + alias: "Search key", + metrics: metrics(3, 600, 5), + date: "2026-11-02", + }, +]; +const fetchMock = vi.fn(); +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } }); +const requestUrl = (input: Parameters[0]) => + new URL(input instanceof Request ? input.url : String(input), "http://localhost"); +const requestedUrls = () => fetchMock.mock.calls.map(([input]) => requestUrl(input)); +const initialUrl = "?org=org-1&from=2026-11-01&to=2026-11-03&source=portal"; +const tableRows = (name: string) => within(screen.getByRole("table", { name })).getAllByRole("row").slice(1); +const list = (data: T[]) => ({ data, meta: { total_count: data.length, page: 1, page_size: 100, total_pages: 1 } }); +const selectBreakdown = (user: ReturnType, name: string) => + chooseSelectOption(user, screen.getByRole("combobox", { name: "Break down by" }), name); +const activityDimension = (resource: string | undefined): "organization" | "team" | "member" => { + if (resource === "customer_organizations") return "organization"; + if (resource === "customer_teams") return "team"; + return "member"; +}; +const filterFor = (resource: string | undefined) => + ({ + organization: "customer_org_ids", + team: "customer_team_ids", + member: "customer_member_ids", + })[activityDimension(resource)]; +const activityUrl = (resource: string, id: string) => { + const filter = filterFor(resource); + return requestedUrls().findLast( + (url) => url.pathname === `/management/v1/${resource}/daily/activity` && url.searchParams.get(filter) === id, + ); +}; +const serve = (input: Parameters[0]): Response => { + const url = requestUrl(input); + if (url.pathname.endsWith("/daily/activity")) { + const resource = url.pathname.split("/").at(-3); + const dimension = activityDimension(resource); + const filter = filterFor(resource); + const ids = url.searchParams.get(filter)?.split(",") ?? []; + const selected = records.filter((record) => ids.includes(record[dimension])); + const results = selected.map((record) => { + const key = { metrics: record.metrics, metadata: { key_alias: record.alias, team_id: null } }; + const entry = { metrics: record.metrics, metadata: {}, api_key_breakdown: { [record.key]: key } }; + return { + date: record.date, + metrics: record.metrics, + breakdown: { + model_groups: { [record.model]: entry }, + api_keys: { [record.key]: key }, + entities: { [record[dimension]]: entry }, + }, + }; + }); + return json({ results, metadata: EMPTY_DAILY_ACTIVITY_METADATA }); + } + if (url.pathname === "/management/v1/customer_organizations/org-1") return json(organization); + if (url.pathname === "/management/v1/customer_teams") return json(list(teams)); + if (url.pathname === "/management/v1/customer_members") + return json(list(members.filter((member) => member.parent_team_id === url.searchParams.get("parent_team_id")))); + const team = teams.find((team) => url.pathname.endsWith(`/customer_teams/${team.customer_team_id}`)); + if (team) return json(team); + const member = members.find((member) => url.pathname.endsWith(`/customer_members/${member.customer_member_id}`)); + if (member) return json(member); + throw new Error(`Unexpected usage request ${url.pathname}`); +}; + +describe("CustomerHierarchyUsagePage", () => { + beforeEach(() => { + testQueryClient.clear(); + registerBaseUrlGetter(() => "http://localhost"); + fetchMock.mockReset(); + fetchMock.mockImplementation((input) => Promise.resolve(serve(input))); + vi.stubGlobal("fetch", fetchMock); + vi.stubGlobal("requestIdleCallback", (callback: IdleRequestCallback) => { + callback({ didTimeout: false, timeRemaining: () => 50 }); + return 1; + }); + }); + + afterEach(() => { + testQueryClient.clear(); + vi.unstubAllGlobals(); + }); + + it("should render organization totals and model rankings from the selected UTC date range", async () => { + renderWithProviders(, { searchParams: initialUrl }); + + expect(await screen.findByRole("heading", { name: "Customer organization" })).toBeInTheDocument(); + expect(screen.getByText("$15.00")).toBeInTheDocument(); + expect(screen.getByTitle("700 tokens")).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Usage over time" })).toBeInTheDocument(); + expect(tableRows("Models")[0]).toHaveTextContent("model-a"); + expect(within(tableRows("Models")[0]).getByText("80%")).toBeInTheDocument(); + expect(tableRows("Models")[1]).toHaveTextContent("model-b"); + expect(activityUrl("customer_organizations", "org-1")?.pathname).toBe( + "/management/v1/customer_organizations/daily/activity", + ); + expect(activityUrl("customer_organizations", "org-1")?.searchParams.get("start_date")).toBe("2026-11-01"); + expect(activityUrl("customer_organizations", "org-1")?.searchParams.get("end_date")).toBe("2026-11-03"); + }); + + it("should apply the global metric to sorting and shares across models, keys and teams", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: initialUrl }); + await screen.findByRole("table", { name: "Models" }); + + await user.click(screen.getByRole("radio", { name: "Tokens" })); + expect(tableRows("Models")[0]).toHaveTextContent("model-b"); + expect(within(tableRows("Models")[0]).getByText("85.7%")).toBeInTheDocument(); + + for (const [tab, leader] of [ + ["API keys", "Search key"], + ["Teams", "Operations"], + ]) { + await selectBreakdown(user, tab); + expect(tableRows(tab)[0]).toHaveTextContent(leader); + expect(within(tableRows(tab)[0]).getByText("85.7%")).toBeInTheDocument(); + expect(within(screen.getByRole("region", { name: "Usage over time" })).getByText(leader)).toBeInTheDocument(); + } + + await user.click(screen.getByRole("radio", { name: "Requests" })); + expect(tableRows("Teams")[0]).toHaveTextContent("Operations"); + expect(within(tableRows("Teams")[0]).getByText("71.4%")).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Requests" })).toHaveAttribute("aria-sort", "descending"); + }); + + it("should drill into customer teams and members and retain dates and unrelated URL state", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { searchParams: initialUrl, onUrlUpdate }); + await screen.findByRole("heading", { name: "Customer organization" }); + + await selectBreakdown(user, "Teams"); + await user.click(within(screen.getByRole("table", { name: "Teams" })).getByRole("button", { name: "Product" })); + expect(await screen.findByRole("heading", { name: "Product" })).toBeInTheDocument(); + expect(activityUrl("customer_teams", "team-a")?.pathname).toBe("/management/v1/customer_teams/daily/activity"); + expect(activityUrl("customer_teams", "team-a")?.searchParams.get("customer_team_ids")).toBe("team-a"); + + await user.click( + within(screen.getByRole("table", { name: "Member spend" })).getByRole("button", { name: "Member One" }), + ); + expect(await screen.findByRole("heading", { name: "Member One" })).toBeInTheDocument(); + const memberUrl = activityUrl("customer_members", "member-a")!; + expect(memberUrl.pathname).toBe("/management/v1/customer_members/daily/activity"); + expect(memberUrl.searchParams.get("customer_member_ids")).toBe("member-a"); + expect(memberUrl.searchParams.has("customer_org_ids")).toBe(false); + expect(memberUrl.searchParams.has("customer_team_ids")).toBe(false); + expect(memberUrl.searchParams.get("start_date")).toBe("2026-11-01"); + expect(memberUrl.searchParams.get("end_date")).toBe("2026-11-03"); + expect(onUrlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ + queryString: "?org=org-1&from=2026-11-01&to=2026-11-03&source=portal&team=team-a&member=member-a", + }), + ); + + await user.click(screen.getByRole("button", { name: "Customer organization" })); + expect(await screen.findByRole("heading", { name: "Customer organization" })).toBeInTheDocument(); + expect(onUrlUpdate).toHaveBeenLastCalledWith(expect.objectContaining({ queryString: initialUrl })); + }); + + it("should fall back to Models for a removed Members breakdown and offer only chart breakdowns", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: `${initialUrl}&breakdown=members` }); + + expect(await screen.findByRole("table", { name: "Models" })).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Top models" })).toBeInTheDocument(); + expect(screen.getByRole("table", { name: "Member spend" })).toBeInTheDocument(); + await user.click(screen.getByRole("combobox", { name: "Break down by" })); + expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual([ + "Models", + "API keys", + "Teams", + ]); + }); + + it("should retain a separate member spend table with range usage and current budget status across chart breakdowns", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { searchParams: initialUrl, onUrlUpdate }); + const table = await screen.findByRole("table", { name: "Member spend" }); + const firstMember = within(table).getAllByRole("row")[1]; + expect(firstMember).toHaveTextContent("Member One"); + expect(within(firstMember).getByText("$12.00")).toBeInTheDocument(); + expect(within(firstMember).getByText("$40.00 / $10.00")).toBeInTheDocument(); + expect(within(firstMember).getByText("Over budget")).toBeInTheDocument(); + + await selectBreakdown(user, "Teams"); + await user.click(screen.getByRole("radio", { name: "Tokens" })); + + expect(tableRows("Teams")[0]).toHaveTextContent("Operations"); + expect(within(table).getAllByRole("row")[1]).toHaveTextContent("Member One"); + const requestCount = fetchMock.mock.calls.length; + act(() => + fireEvent.change(screen.getByRole("textbox", { name: "Search members" }), { target: { value: "Member One" } }), + ); + expect(within(table).queryByText("Member Two")).not.toBeInTheDocument(); + expect(within(table).getByText("Member One")).toBeInTheDocument(); + await waitFor(() => + expect(new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString).get("memberSearch")).toBe("Member One"), + ); + const updatedUrl = new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString); + expect(updatedUrl.get("org")).toBe("org-1"); + expect(updatedUrl.get("breakdown")).toBe("teams"); + expect(updatedUrl.get("metric")).toBe("tokens"); + expect(updatedUrl.get("source")).toBe("portal"); + expect(fetchMock.mock.calls).toHaveLength(requestCount); + }); + + it("should clear the previous scope values while customer team usage loads", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: initialUrl }); + await screen.findByRole("heading", { name: "Customer organization" }); + await selectBreakdown(user, "Teams"); + fetchMock.mockImplementation(() => new Promise(() => {})); + + await user.click(within(screen.getByRole("table", { name: "Teams" })).getByRole("button", { name: "Product" })); + + expect(screen.getByRole("status", { name: "Loading customer usage" })).toBeInTheDocument(); + expect(screen.queryByText("$15.00")).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Customer organization" })).not.toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + await waitFor(() => + expect(requestedUrls().some((url) => url.pathname === "/management/v1/customer_teams/team-a")).toBe(true), + ); + }); + + it("should request a changed date range without showing stale totals or losing the customer scope", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { searchParams: initialUrl, onUrlUpdate }); + await screen.findByRole("heading", { name: "Customer organization" }); + fetchMock.mockImplementation((input) => + requestUrl(input).pathname.endsWith("/daily/activity") && + requestUrl(input).searchParams.get("start_date") === "2026-10-01" + ? new Promise(() => {}) + : Promise.resolve(serve(input)), + ); + + await user.click(screen.getByRole("button", { name: / - / })); + act(() => { + fireEvent.change(screen.getByLabelText("Start date"), { target: { value: "2026-10-01" } }); + fireEvent.change(screen.getByLabelText("End date"), { target: { value: "2026-10-07" } }); + }); + await user.click(screen.getByRole("button", { name: "Apply" })); + + await waitFor(() => + expect(activityUrl("customer_organizations", "org-1")?.searchParams.get("start_date")).toBe("2026-10-01"), + ); + expect(activityUrl("customer_organizations", "org-1")?.searchParams.get("end_date")).toBe("2026-10-07"); + expect(activityUrl("customer_organizations", "org-1")?.pathname).toBe( + "/management/v1/customer_organizations/daily/activity", + ); + expect(screen.getByRole("status", { name: "Loading customer usage" })).toBeInTheDocument(); + expect(screen.queryByText("$15.00")).not.toBeInTheDocument(); + expect(onUrlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ queryString: "?org=org-1&from=2026-10-01&to=2026-10-07&source=portal" }), + ); + }); + + it("should show a failed request as an error and load usage when retried", async () => { + const user = userEvent.setup(); + fetchMock.mockResolvedValueOnce(json({ error: "Usage temporarily unavailable" }, 503)); + renderWithProviders(, { searchParams: initialUrl }); + + expect(await screen.findByRole("alert")).toHaveTextContent("Could not load customer usage"); + expect(screen.getByRole("alert")).toHaveTextContent("Usage temporarily unavailable"); + expect(screen.queryByText("$0.00")).not.toBeInTheDocument(); + expect(screen.queryByText("No usage in this range")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Retry" })); + + expect(await screen.findByText("$15.00")).toBeInTheDocument(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); + + it("should show an empty date range after the scoped request succeeds without activity", async () => { + fetchMock.mockImplementation((input) => + Promise.resolve( + requestUrl(input).pathname.endsWith("/daily/activity") + ? json({ results: [], metadata: EMPTY_DAILY_ACTIVITY_METADATA }) + : serve(input), + ), + ); + renderWithProviders(, { searchParams: initialUrl }); + + expect(await screen.findByText("No usage in this range")).toBeInTheDocument(); + expect(screen.getByTitle("0 tokens")).toBeInTheDocument(); + expect(screen.queryByRole("status", { name: "Loading customer usage" })).not.toBeInTheDocument(); + expect(screen.getByText("No usage recorded for this breakdown.")).toBeInTheDocument(); + }); + + it("should request an organization selection without issuing an unscoped usage request", () => { + renderWithProviders(, { searchParams: "?source=portal" }); + + expect(screen.getByText("Open a customer organization to explore its usage.")).toBeInTheDocument(); + expect(screen.getByRole("link", { name: "Back to customers" })).toHaveAttribute("href", "/ui/customers"); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.tsx new file mode 100644 index 00000000000..3fa99627bee --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/customers/usage/page.tsx @@ -0,0 +1,136 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import UsageStatStrip from "@/app/(dashboard)/usage/_components/components/overview/UsageStatStrip"; +import { ErrorState } from "@/components/shared/ErrorState"; +import { Button } from "@/components/ui/button"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { uiHref } from "@/utils/uiHref"; +import CustomerMemberSpend from "../_components/CustomerMemberSpend"; +import CustomerUsageHeader from "../_components/CustomerUsageHeader"; +import CustomerUsageRankings from "../_components/CustomerUsageRankings"; +import CustomerUsageTrend from "../_components/CustomerUsageTrend"; +import useCustomerHierarchyUsage from "../_components/useCustomerHierarchyUsage"; + +const METRIC_LABELS = { + spend: "customerPortal.analytics.spend", + tokens: "customerPortal.analytics.tokens", + requests: "customerPortal.analytics.requests", +} as const; +const TOP_DIMENSION_LABELS = { + models: "customerPortal.analytics.topModels", + api_keys: "customerPortal.analytics.topKeys", + teams: "customerPortal.analytics.topTeams", +} as const; + +export default function CustomerHierarchyUsagePage() { + const { t } = useTranslation(); + const { + params, + scope, + dateValue, + preset, + usage, + data, + results, + totals, + modelColors, + rows, + drillDown, + setDateRange, + setDimension, + selectOrganization, + selectTeam, + setMetric, + setBucket, + setSearch, + setPage, + } = useCustomerHierarchyUsage(); + const metricLabel = t(METRIC_LABELS[params.metric]); + + if (!scope) + return ( +
+

{t("customerPortal.analytics.title")}

+

{t("customerPortal.analytics.selectOrganization")}

+ + {t("customerPortal.analytics.backToCustomers")} + +
+ ); + + return ( +
+ + {usage.isError ? ( + void usage.refetch()} + /> + ) : ( + <> + + + {!usage.isPending && ( + <> +
+

{t(TOP_DIMENSION_LABELS[params.breakdown])}

+

+ {t("customerPortal.analytics.sortedBy", { metric: metricLabel.toLocaleLowerCase() })} +

+
+ + + )} + {data && drillDown("members", id)} />} + {(params.team || params.member) && ( + + )} +

{t("customerPortal.analytics.freshness")}

+ + )} +
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx b/litellm/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx index 79c64e6079b..04b6ddbc2b0 100644 --- a/litellm/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx +++ b/litellm/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx @@ -84,7 +84,7 @@ export function BreakdownControls({ } /** Provider logo on a soft tile with a series-colored dot, so mixed logo styles still read as a set. */ -function ModelMark({ model, color }: { model: string; color: string }) { +export function ModelMark({ model, color }: { model: string; color: string }) { const provider = providerForModel(model); return ( diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index bdade69fef6..a2369c9f077 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -20681,6 +20681,72 @@ "maxBudget": "Max budget (USD)", "maxBudgetPlaceholder": "Leave empty for no budget", "sharedBudgetWarning": "This customer is on budget {{budgetId}}. Changing the budget here changes it for every customer on that budget." + }, + "analytics": { + "title": "Customer usage", + "customers": "Customers", + "breadcrumb": "Customer hierarchy", + "selectOrganization": "Open a customer organization to explore its usage.", + "backToCustomers": "Back to customers", + "organizationUsage": "Organization usage", + "teamUsage": "Team usage", + "memberUsage": "Member usage", + "subtitle": "Understand where usage goes, from your organization to each member.", + "trend": "Usage over time", + "trendHint": "Daily activity, broken down by {{dimension}}.", + "metric": "Usage metric", + "spend": "Spend", + "tokens": "Tokens", + "requests": "Requests", + "bucket": "Time interval", + "daily": "Daily", + "weekly": "Weekly", + "loading": "Loading customer usage", + "empty": "No usage in this range", + "error": "Could not load customer usage", + "breakdown": "Usage breakdown", + "models": "Models", + "keys": "API keys", + "useCases": "Use cases", + "teams": "Teams", + "members": "Members", + "sortedBy": "Ranked by {{metric}}", + "search": "Search breakdown", + "rank": "Rank", + "share": "Share", + "rankingHint": "Share of the selected metric across this customer scope. Click a team or member to explore its usage.", + "useCaseHint": "Use cases recorded with requests. Unattributed activity appears as Unassigned.", + "noMatches": "No results match your search.", + "noBreakdown": "No usage recorded for this breakdown.", + "pagination": "{{from}}–{{to}} of {{total}}", + "previous": "Previous", + "next": "Next", + "allOrganizationUsage": "View all organization usage", + "freshness": "Usage appears after accounting completes. Date ranges use UTC. Customer budgets are managed separately.", + "scopeMismatch": "This team or member does not belong to the selected organization.", + "invalidIdentifier": "Select a valid customer identifier.", + "incompletePages": "Could not load the complete usage range. Please choose fewer days.", + "unassignedTeam": "Unassigned or removed team", + "unassignedMember": "Unassigned or removed member", + "unnamedKey": "Unnamed key", + "currentSpend": "Budget spend", + "noBudget": "No budget set", + "breakDownBy": "Break down by", + "topModels": "Top models", + "topKeys": "Top API keys", + "topTeams": "Top teams", + "topMembers": "Top members", + "memberSpend": "Member spend", + "memberSpendHint": "Usage in the selected date range. Cap status uses each member’s recorded budget spend.", + "searchMembers": "Search members", + "member": "Member", + "currentBudget": "Spend / cap", + "status": "Status", + "noCap": "No cap", + "atBudget": "At budget", + "withinBudget": "Within budget", + "noMembers": "No members in this customer scope.", + "overBudget": "Over budget" } } } diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index 97f169bf9f5..ff97a43c136 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9608,6 +9608,26 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/customer_members/daily/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Customer Member Daily Activity + * @description Get paginated daily customer member activity with model, provider, key and member breakdowns. Filters accept comma-separated member IDs. Proxy admins only. + */ + get: operations["customer_member_daily_activity_management_v1_customer_members_daily_activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/management/v1/customer_members/{customer_member_id}": { parameters: { query?: never; @@ -9660,6 +9680,26 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/customer_organizations/daily/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Customer Organization Daily Activity + * @description Get paginated daily customer organization activity with model, provider, key and organization breakdowns. Filters accept comma-separated organization IDs. Proxy admins only. + */ + get: operations["customer_organization_daily_activity_management_v1_customer_organizations_daily_activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/management/v1/customer_organizations/{customer_org_id}": { parameters: { query?: never; @@ -9756,6 +9796,26 @@ export interface paths { patch?: never; trace?: never; }; + "/management/v1/customer_teams/daily/activity": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Customer Team Daily Activity + * @description Get paginated daily customer team activity with model, provider, key and team breakdowns. Filters accept comma-separated team IDs. Proxy admins only. + */ + get: operations["customer_team_daily_activity_management_v1_customer_teams_daily_activity_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/management/v1/customer_teams/{customer_team_id}": { parameters: { query?: never; @@ -68072,6 +68132,44 @@ export interface operations { }; }; }; + customer_member_daily_activity_management_v1_customer_members_daily_activity_get: { + parameters: { + query?: { + customer_member_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + model?: string | null; + api_key?: string | null; + page?: number; + page_size?: number; + exclude_customer_member_ids?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_customer_member_management_v1_customer_members__customer_member_id__get: { parameters: { query?: never; @@ -68232,6 +68330,44 @@ export interface operations { }; }; }; + customer_organization_daily_activity_management_v1_customer_organizations_daily_activity_get: { + parameters: { + query?: { + customer_org_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + model?: string | null; + api_key?: string | null; + page?: number; + page_size?: number; + exclude_customer_org_ids?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_customer_organization_management_v1_customer_organizations__customer_org_id__get: { parameters: { query?: never; @@ -68494,6 +68630,44 @@ export interface operations { }; }; }; + customer_team_daily_activity_management_v1_customer_teams_daily_activity_get: { + parameters: { + query?: { + customer_team_ids?: string | null; + start_date?: string | null; + end_date?: string | null; + model?: string | null; + api_key?: string | null; + page?: number; + page_size?: number; + exclude_customer_team_ids?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SpendAnalyticsPaginatedResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_customer_team_management_v1_customer_teams__customer_team_id__get: { parameters: { query?: never; diff --git a/tests/unit/accounting/test_application.py b/tests/unit/accounting/test_application.py new file mode 100644 index 00000000000..d60893a147d --- /dev/null +++ b/tests/unit/accounting/test_application.py @@ -0,0 +1,103 @@ +from collections.abc import Mapping +from dataclasses import dataclass, replace +from datetime import datetime, timezone +from typing import Final, cast + +import pytest +from pydantic import TypeAdapter + +from litellm.accounting.application import _apply_customer_usage +from litellm.accounting.delivery_projection import project_deliveries +from litellm.accounting.projection import Contribution +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution +from litellm.repositories.accounting_delivery_repository import DeliveryProjection, LedgerTransaction + +_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_TEXT: Final = TypeAdapter(str) +_UPDATES: Final = TypeAdapter(Mapping[str, Mapping[str, float]]) +_IDENTIFIER: Final = "\U0001f680" * 256 +_ATTRIBUTION: Final = CustomerUsageAttribution( + customer_org_id=_IDENTIFIER, customer_team_id=_IDENTIFIER, customer_member_id=_IDENTIFIER +) + + +@dataclass(frozen=True, slots=True) +class Bucket: + values: Mapping[str, object] + counters: Mapping[str, float] + + +class Buckets: + def __init__(self) -> None: + self.rows: Mapping[str, Bucket] = {} + + async def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: + row_id: Final = _TEXT.validate_python(where["id"]) + create: Final = _FIELDS.validate_python(data["create"]) + previous: Final = self.rows.get(row_id) + increments: Final = ( + {name: fields["increment"] for name, fields in _UPDATES.validate_python(data["update"]).items()} + if previous is not None + else {name: float(value) for name, value in create.items() if isinstance(value, (int, float))} + ) + self.rows = { + **self.rows, + row_id: Bucket( + values=previous.values if previous is not None else create, + counters={ + name: (previous.counters.get(name, 0) if previous is not None else 0) + value + for name, value in increments.items() + }, + ), + } + + +@dataclass(frozen=True, slots=True) +class Transaction: + litellm_dailycustomermemberspend: Buckets + + +def contribution(alias: str, provider: str, spend: float) -> Contribution: + delivery: Final = DeliveryProjection( + { + "request_id": "long-customer-id-request", + "startTime": datetime(2026, 1, 1, tzinfo=timezone.utc).isoformat(), + "api_key": "hashed-key", + "model": "shared-model", + "model_group": alias, + "custom_llm_provider": provider, + "spend": spend, + "prompt_tokens": 10, + "completion_tokens": 5, + "metadata": {CUSTOMER_USAGE_METADATA_KEY: _ATTRIBUTION.model_dump_json()}, + } + ) + return next(item for item in project_deliveries((delivery,)) if item.table == "LiteLLM_DailyCustomerMemberSpend") + + +@pytest.mark.asyncio +async def test_long_customer_ids_roll_up_without_truncation_and_keep_alias_provider_buckets_distinct() -> None: + buckets: Final = Buckets() + transaction: Final = cast(LedgerTransaction, Transaction(buckets)) # cast-ok: in-memory Prisma upsert boundary + first: Final = contribution("alias-a", "provider-a", 1) + second: Final = replace(contribution("alias-a", "provider-a", 2), key=tuple(reversed(first.key))) + + await _apply_customer_usage(transaction, (first,)) + await _apply_customer_usage(transaction, (second, contribution("alias-b", "provider-a", 4))) + await _apply_customer_usage(transaction, (contribution("alias-a", "provider-b", 8),)) + + assert len(buckets.rows) == 3 + assert all(len(row_id.encode("utf-8")) == 64 for row_id in buckets.rows) + assert all( + row.values["customer_org_id"] + == row.values["customer_team_id"] + == row.values["customer_member_id"] + == _IDENTIFIER + for row in buckets.rows.values() + ) + assert { + (row.values["model_group"], row.values["custom_llm_provider"]): row.counters["spend"] + for row in buckets.rows.values() + } == {("alias-a", "provider-a"): 3, ("alias-b", "provider-a"): 4, ("alias-a", "provider-b"): 8} + assert sum(row.counters["api_requests"] for row in buckets.rows.values()) == 4 + assert sum(row.counters["prompt_tokens"] + row.counters["completion_tokens"] for row in buckets.rows.values()) == 60 diff --git a/tests/unit/accounting/test_projection.py b/tests/unit/accounting/test_projection.py index 090825dca75..75b3f76de26 100644 --- a/tests/unit/accounting/test_projection.py +++ b/tests/unit/accounting/test_projection.py @@ -6,20 +6,120 @@ from typing import Final import pytest from pydantic import TypeAdapter +from litellm.accounting.delivery_projection import project_deliveries from litellm.accounting.projection import project from litellm.accounting.snapshot import build_accounting_payload from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.proxy._types import SpendLogsPayload +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.proxy_server import proxy_logging_obj from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.proxy.utils import PrismaClient +from litellm.repositories.accounting_delivery_repository import DeliveryProjection from litellm.types.utils import ModelResponse, Usage _NOW: Final = datetime(2026, 11, 2, tzinfo=timezone.utc) _FIELDS: Final = TypeAdapter(dict[str, object]) +def test_customer_model_aliases_and_providers_keep_separate_daily_totals() -> None: + stamp: Final = CustomerUsageAttribution(customer_org_id="org-1", customer_member_id="member-1").model_dump_json() + base: Final = { + "startTime": _NOW, + "api_key": "hashed-key", + "model": "shared-model", + "metadata": {CUSTOMER_USAGE_METADATA_KEY: stamp}, + } + deliveries: Final = tuple( + DeliveryProjection({**base, "model_group": alias, "custom_llm_provider": provider, "spend": spend}) + for alias, provider, spend in ( + ("alias-a", "provider-a", 1), + ("alias-b", "provider-a", 2), + ("alias-a", "provider-b", 3), + ) + ) + totals: Final = { + (dict(item.key)["model_group"], dict(item.key)["custom_llm_provider"]): dict(item.increments)["spend"] + for item in project_deliveries(deliveries) + if item.table == "LiteLLM_DailyCustomerMemberSpend" + } + assert totals == {("alias-a", "provider-a"): 1, ("alias-b", "provider-a"): 2, ("alias-a", "provider-b"): 3} + + +@pytest.mark.parametrize("retry", [False, True]) +def test_customer_usage_survives_logging_and_accounting_with_correct_request_counts(retry: bool) -> None: + attribution: Final = CustomerUsageAttribution( + customer_org_id="org-1", customer_team_id="team-1", customer_member_id="member-1" + ) + payload: Final = get_logging_payload( + kwargs={ + "model": "usage-model", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "usage-request", + "litellm_params": { + "metadata": { + "user_api_key": "hashed-key", + "status": "success", + CUSTOMER_USAGE_METADATA_KEY: attribution.model_dump_json(), + **({BILLED_RETRY_ATTEMPT_METADATA_KEY: True} if retry else {}), + } + }, + }, + response_obj=ModelResponse( + id="usage-request", model="usage-model", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + ) + accounting: Final = build_accounting_payload( + {**payload, "spend": 0.75}, entities={}, model_access_groups=(), windows=(), router=lambda: None + ) + contributions: Final = project(accounting) + usage: Final = next(item for item in contributions if item.table == "LiteLLM_DailyCustomerMemberSpend") + assert dict(usage.key) == { + "date": _NOW.date().isoformat(), + "customer_org_id": attribution.customer_org_id, + "customer_team_id": attribution.customer_team_id, + "customer_member_id": attribution.customer_member_id, + "model": payload["model"], + "model_group": payload["model_group"], + "custom_llm_provider": payload["custom_llm_provider"], + "api_key": payload["api_key"], + "mcp_namespaced_tool_name": "", + "endpoint": "/chat/completions", + } + assert dict(usage.increments) == { + "spend": 0.75, + "prompt_tokens": 10, + "completion_tokens": 5, + "api_requests": 0 if retry else 1, + "successful_requests": 0 if retry else 1, + "failed_requests": 0, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "compression_saved_tokens": 0, + "compression_savings_spend": 0, + "prompt_caching_savings_spend": 0, + "gateway_injected_caching_savings_spend": 0, + "autorouter_savings_spend": 0, + "guardrail_spend": 0, + "total_response_time_ms": 0, + "timed_requests": 1, + } + assert { + item.table: dict(item.increments)["spend"] + for item in contributions + if item.table + in ("LiteLLM_CustomerOrganizationTable", "LiteLLM_CustomerTeamTable", "LiteLLM_CustomerMemberTable") + } == { + "LiteLLM_CustomerOrganizationTable": 0.75, + "LiteLLM_CustomerTeamTable": 0.75, + "LiteLLM_CustomerMemberTable": 0.75, + } + + @pytest.mark.parametrize("status", ["success", "failure"]) @pytest.mark.parametrize("origin", [None, "auto_router", "billed_retry"]) @pytest.mark.parametrize("spend", [0.0, 0.25]) diff --git a/tests/unit/accounting/test_worker.py b/tests/unit/accounting/test_worker.py index d3b5d71cff9..372ae90b158 100644 --- a/tests/unit/accounting/test_worker.py +++ b/tests/unit/accounting/test_worker.py @@ -1,5 +1,9 @@ import json +from collections.abc import AsyncIterator, Mapping +from contextlib import asynccontextmanager from dataclasses import dataclass +from datetime import datetime, timezone +from functools import partial from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -10,11 +14,18 @@ from fakeredis.aioredis import FakeRedis from pydantic import TypeAdapter from litellm import in_memory_llm_clients_cache +from litellm.accounting.application import apply_deliveries from litellm.accounting.worker import flush_accounting_deliveries from litellm.caching.redis_cache import RedisCache +from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY, CustomerUsageAttribution from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.accounting_delivery_repository import ( + AccountingDeliveryRepository, + DeliveryProjection, + LedgerTransaction, +) _FIELDS: Final = TypeAdapter(dict[str, object]) _INCREMENTS: Final = TypeAdapter(dict[str, dict[str, float]]) @@ -30,7 +41,7 @@ class _Ledger: spend: float = 0.0 attempts: int = 0 - async def query(self, query: str, *args: object) -> tuple[dict[str, object], ...]: + async def query(self, query: str, *args: object) -> tuple[Mapping[str, object], ...]: if query.startswith("SELECT delivery_id, attempts"): if not self.pending or self.attempts or args[1] is True: return () @@ -96,3 +107,119 @@ async def test_delivery_charges_the_creating_key_despite_an_orphaned_maintenance in_memory_llm_clients_cache.delete_cache(key=cache_key) await redis.aclose() sync_redis.close() + + +class _CustomerLedger: + def __init__(self, fail_acknowledgement: bool) -> None: + self.pending = True + self.attempts = 0 + self.fail_acknowledgement = fail_acknowledgement + self.daily: Mapping[str, int | float] = {} + self.entities: Mapping[str, float] = {} + + async def query(self, query: str, *args: object) -> tuple[Mapping[str, object], ...]: + if query.startswith("SELECT delivery_id, attempts"): + if not self.pending or (self.attempts > 0) is not args[1]: + return () + return ( + { + "delivery_id": "customer-delivery", + "attempts": self.attempts, + "accounting_payload": _CUSTOMER_PAYLOAD, + }, + ) + if "SET attempts = attempts + 1" in query: + self.attempts += 1 + return ({"delivery_id": "customer-delivery", "attempts": self.attempts},) + return () + + async def execute(self, query: str, *args: object) -> int: + if "SET state = 'applied'" not in query: + return 0 + if self.fail_acknowledgement: + self.fail_acknowledgement = False + raise OSError("database connection lost before delivery acknowledgement") + self.pending = False + return 1 + + def upsert_daily(self, *, where: object, data: object) -> None: + values: Final = _FIELDS.validate_python(_FIELDS.validate_python(data)["create"]) + counters: Final = {name: value for name, value in values.items() if isinstance(value, (int, float))} + self.daily = {name: self.daily.get(name, 0) + value for name, value in counters.items()} + + def update_entity(self, entity: str, *, where: object, data: object) -> int: + spend: Final = _INCREMENTS.validate_python(data)["spend"]["increment"] + self.entities = {**self.entities, entity: self.entities.get(entity, 0.0) + spend} + return 1 + + +_CUSTOMER_PAYLOAD: Final = json.dumps( + { + "request_id": "customer-completed-request", + "startTime": datetime(2026, 1, 1, tzinfo=timezone.utc).isoformat(), + "api_key": "hashed-key", + "spend": 0.75, + "prompt_tokens": 10, + "completion_tokens": 5, + "metadata": { + CUSTOMER_USAGE_METADATA_KEY: CustomerUsageAttribution( + customer_org_id="customer-org", customer_team_id="customer-team", customer_member_id="customer-member" + ).model_dump_json() + }, + } +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_acknowledgement", (False, True)) +async def test_customer_daily_and_lifetime_totals_commit_once_after_delivery_replay( + fail_acknowledgement: bool, +) -> None: + ledger: Final = _CustomerLedger(fail_acknowledgement) + transaction: Final = MagicMock() + transaction.query_raw = AsyncMock(side_effect=ledger.query) + transaction.execute_raw = AsyncMock(side_effect=ledger.execute) + transaction.litellm_dailycustomermemberspend.upsert = AsyncMock(side_effect=ledger.upsert_daily) + transaction.litellm_customerorganizationtable.update_many = AsyncMock( + side_effect=partial(ledger.update_entity, "organization") + ) + transaction.litellm_customerteamtable.update_many = AsyncMock(side_effect=partial(ledger.update_entity, "team")) + transaction.litellm_customermembertable.update_many = AsyncMock(side_effect=partial(ledger.update_entity, "member")) + transaction.batch_.return_value.__aenter__ = AsyncMock(return_value=MagicMock()) + transaction.batch_.return_value.__aexit__ = AsyncMock(return_value=False) + + @asynccontextmanager + async def database_transaction(*, timeout: object) -> AsyncIterator[LedgerTransaction]: + before: Final = (ledger.pending, ledger.daily, ledger.entities) + try: + yield transaction + except Exception: + ledger.pending, ledger.daily, ledger.entities = before + raise + + database: Final = MagicMock(spec=PrismaClient) + database.tx.side_effect = database_transaction + repository: Final = AccountingDeliveryRepository(database) + writer: Final = DBSpendUpdateWriter(accounting_clock=lambda: 0.0) + logger: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) + + async def apply(tx: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None: + await apply_deliveries(tx, deliveries, prisma_client=database, writer=writer, proxy_logging_obj=logger) + + first: Final = await repository.apply_pending(apply) + if fail_acknowledgement: + assert first.failed == 1 + assert ledger.daily == {} + assert ledger.entities == {} + retried: Final = await repository.apply_pending(apply) + assert retried.applied == 1 + else: + assert first.applied == 1 + await repository.apply_pending(apply) + await repository.apply_pending(apply) + assert ledger.daily["spend"] == 0.75 + assert ledger.daily["prompt_tokens"] == 10 + assert ledger.daily["completion_tokens"] == 5 + assert ledger.daily["api_requests"] == 1 + assert ledger.entities == {"organization": 0.75, "team": 0.75, "member": 0.75} + assert ledger.pending is False diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index 6e5461c3b60..40674758371 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_customer_usage\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/proxy/common_utils/test_customer_usage.py b/tests/unit/proxy/common_utils/test_customer_usage.py new file mode 100644 index 00000000000..79b16cafa82 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_customer_usage.py @@ -0,0 +1,225 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Final, cast # noqa: TID251 # the fake exposes only the Prisma lookup methods exercised here + +import pytest + +from litellm.proxy.common_utils.customer_usage import ( + CustomerUsageAttribution, + CustomerUsageDatabase, + CustomerUsageError, + CustomerUsageFailure, + resolve_customer_usage, +) + + +@dataclass(frozen=True, slots=True) +class Table: + identifier: str + rows: Mapping[str, Mapping[str, object]] + + async def find_unique(self, where: Mapping[str, object]) -> Mapping[str, object] | None: + return self.rows.get(str(where[self.identifier])) + + +@dataclass(frozen=True, slots=True) +class Database: + litellm_customerorganizationtable: Table + litellm_customerteamtable: Table + litellm_customermembertable: Table + + +DATABASE: Final = cast( # cast-ok: the external Prisma fake implements the lookup subset used by attribution + CustomerUsageDatabase, + Database( + Table("customer_org_id", {"org-1": {}, "org-2": {}}), + Table("customer_team_id", {"team-1": {"parent_org_id": "org-1"}, "team-2": {"parent_org_id": "org-2"}}), + Table("customer_member_id", {"member-1": {"parent_team_id": "team-1"}}), + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "expected"), + ( + ( + {"customer_member_id": "member-1"}, + CustomerUsageAttribution(customer_org_id="org-1", customer_team_id="team-1", customer_member_id="member-1"), + ), + ({"customer_team_id": "team-1"}, CustomerUsageAttribution(customer_org_id="org-1", customer_team_id="team-1")), + ({"customer_org_id": "org-1"}, CustomerUsageAttribution(customer_org_id="org-1")), + ), +) +async def test_resolves_ancestors_without_requiring_redundant_request_ids( + metadata: Mapping[str, object], expected: CustomerUsageAttribution +) -> None: + actual: Final = await resolve_customer_usage( + metadata=metadata, key_metadata={}, administrator=True, database=DATABASE, ledger_enabled=True + ) + assert actual == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "key_metadata", "administrator", "expected"), + ( + ({"customer_member_id": "member-1", "customer_team_id": "team-2"}, {}, True, CustomerUsageError.MISMATCH), + ({"customer_member_id": "member-1"}, {"customer_org_id": "org-2"}, False, CustomerUsageError.FORBIDDEN), + ({"customer_team_id": "team-2", "customer_org_id": "org-1"}, {}, True, CustomerUsageError.MISMATCH), + ({"customer_org_id": "org-2"}, {"customer_org_id": "org-1"}, False, CustomerUsageError.FORBIDDEN), + ({"customer_org_id": "org-1"}, {}, False, CustomerUsageError.UNBOUND), + ({"customer_member_id": "missing"}, {}, True, CustomerUsageError.NOT_FOUND), + ({"customer_team_id": "missing"}, {}, True, CustomerUsageError.NOT_FOUND), + ({"customer_org_id": "missing"}, {}, True, CustomerUsageError.NOT_FOUND), + ({"customer_org_id": 123}, {}, True, CustomerUsageError.INVALID), + ), +) +async def test_rejects_cross_organization_attribution_and_invalid_identifiers( + metadata: Mapping[str, object], + key_metadata: Mapping[str, object], + administrator: bool, + expected: CustomerUsageError, +) -> None: + actual: Final = await resolve_customer_usage( + metadata=metadata, + key_metadata=key_metadata, + administrator=administrator, + database=DATABASE, + ledger_enabled=True, + ) + assert actual == CustomerUsageFailure(expected) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "key_metadata"), + ( + ({"customer_member_id": "missing"}, {"customer_org_id": "org-1"}), + ({"customer_member_id": "member-1"}, {"customer_org_id": "org-2"}), + ( + {"customer_member_id": "member-1"}, + {"customer_org_id": "org-2", "customer_team_id": "team-2"}, + ), + ({"customer_team_id": "missing"}, {"customer_org_id": "org-1"}), + ({"customer_team_id": "team-2"}, {"customer_org_id": "org-1"}), + ({}, {"customer_org_id": "missing"}), + ), +) +async def test_non_admin_cannot_distinguish_missing_and_other_organization_identifiers( + metadata: Mapping[str, object], key_metadata: Mapping[str, object] +) -> None: + actual: Final = await resolve_customer_usage( + metadata=metadata, + key_metadata=key_metadata, + administrator=False, + database=DATABASE, + ledger_enabled=True, + ) + assert actual == CustomerUsageFailure(CustomerUsageError.FORBIDDEN) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("administrator", (False, True)) +async def test_invalid_identifier_types_keep_validation_errors(administrator: bool) -> None: + actual: Final = await resolve_customer_usage( + metadata={"customer_member_id": 123}, + key_metadata={"customer_org_id": "org-1"}, + administrator=administrator, + database=DATABASE, + ledger_enabled=True, + ) + assert actual == CustomerUsageFailure(CustomerUsageError.INVALID) + + +@pytest.mark.asyncio +async def test_key_binding_tracks_requests_with_no_repeated_customer_metadata() -> None: + actual: Final = await resolve_customer_usage( + metadata={}, + key_metadata={"customer_org_id": "org-1"}, + administrator=False, + database=DATABASE, + ledger_enabled=True, + ) + assert actual == CustomerUsageAttribution(customer_org_id="org-1") + + +@pytest.mark.asyncio +async def test_requires_accounting_before_accepting_a_tracked_request() -> None: + actual: Final = await resolve_customer_usage( + metadata={"customer_org_id": "org-1"}, + key_metadata={}, + administrator=True, + database=DATABASE, + ledger_enabled=False, + ) + assert actual == CustomerUsageFailure(CustomerUsageError.UNAVAILABLE) + + +@pytest.mark.asyncio +async def test_untracked_requests_do_not_require_a_database_or_ledger() -> None: + actual: Final = await resolve_customer_usage( + metadata={}, key_metadata={}, administrator=False, database=None, ledger_enabled=False + ) + assert actual is None + + +@pytest.mark.asyncio +async def test_untracked_request_enrichment_does_not_touch_customer_database_tables( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from fastapi import Request + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=object())) + request: Final = Request( + {"type": "http", "path": "/v1/chat/completions", "method": "POST", "headers": (), "query_string": b""} + ) + enriched: Final = await add_litellm_data_to_request( + data={"model": "usage-model", "metadata": {"application": "support"}}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + assert enriched["metadata"]["application"] == "support" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/chat/completions", "/v1/responses"]) +async def test_request_enrichment_replaces_forged_stamps_and_preserves_valid_attribution( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from fastapi import Request + from pydantic import TypeAdapter + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_utils.customer_usage import CUSTOMER_USAGE_METADATA_KEY + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request, get_metadata_variable_name + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=DATABASE)) + request: Final = Request({"type": "http", "path": path, "method": "POST", "headers": (), "query_string": b""}) + metadata_name: Final = get_metadata_variable_name(request) + forged: Final = CustomerUsageAttribution(customer_org_id="org-2").model_dump_json() + metadata: Final = {"customer_member_id": "member-1", CUSTOMER_USAGE_METADATA_KEY: forged} + enriched: Final = await add_litellm_data_to_request( + data={ + "model": "usage-model", + "metadata": {CUSTOMER_USAGE_METADATA_KEY: forged}, + "litellm_metadata": {CUSTOMER_USAGE_METADATA_KEY: forged}, + metadata_name: metadata, + }, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", metadata={"customer_org_id": "org-1"}), + proxy_config=ProxyConfig(), + general_settings={}, + ) + stamp: Final = TypeAdapter(str).validate_python(enriched[metadata_name][CUSTOMER_USAGE_METADATA_KEY]) + assert CustomerUsageAttribution.model_validate_json(stamp) == CustomerUsageAttribution( + customer_org_id="org-1", customer_team_id="team-1", customer_member_id="member-1" + ) diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index 554a8bb3215..a54912c44ba 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -3488,7 +3488,7 @@ async def test_a_failed_request_settles_the_team_share_gauges_too(monkeypatch: p registry = _prometheus_sink(monkeypatch) await _admit_then_settle(handler, _team_key(), tokens=90) claim_request_stash_for_data({"litellm_call_id": "failed-call"}) - await _check(handler, _team_key(), saturation=0.0, data=_prompt(words=40), priority=None) + await _check(handler, _team_key(), saturation=0.5) await handler.async_log_failure_event( kwargs={ "litellm_params": {"metadata": {"model_group": "shared-model"}}, @@ -3504,10 +3504,10 @@ async def test_a_failed_request_settles_the_team_share_gauges_too(monkeypatch: p @pytest.mark.asyncio @pytest.mark.parametrize( - "auth, burns, saturation_threshold, reason", + "auth, burns, saturation_threshold, reason, enforced_share_tokens", [ - (_team_key(), (60, 70), 0.1, "model_per_team_saturated"), - (_capped_team_key(2), (150, 60), 0.5, "model_per_team_max_borrow_multiple"), + (_team_key(), (60, 70), 0.0, "model_per_team_saturated", 100), + (_capped_team_key(2), (150, 60), 0.5, "model_per_team_max_borrow_multiple", 200), ], ids=["saturated", "borrow_cap"], ) @@ -3517,6 +3517,7 @@ async def test_a_refused_request_is_counted_under_the_reason_its_429_header_carr burns: tuple[int, ...], saturation_threshold: float, reason: str, + enforced_share_tokens: int, ): handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=saturation_threshold) registry = _prometheus_sink(monkeypatch) @@ -3532,7 +3533,10 @@ async def test_a_refused_request_is_counted_under_the_reason_its_429_header_carr ) == 1 ) - assert _sample(registry, "litellm_dynamic_limiter_borrowed_tokens", **TEAM_LABELS) == sum(burns) - 100 + assert _sample(registry, "litellm_dynamic_limiter_share_tokens", **TEAM_LABELS) == enforced_share_tokens + assert ( + _sample(registry, "litellm_dynamic_limiter_borrowed_tokens", **TEAM_LABELS) == sum(burns) - enforced_share_tokens + ) @pytest.mark.asyncio @@ -3596,7 +3600,7 @@ async def test_an_alternates_refusal_is_not_counted_while_the_preferred_model_ad @pytest.mark.asyncio async def test_share_gauges_and_throttles_carry_the_teams_priority_so_the_series_join(monkeypatch: pytest.MonkeyPatch): - handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.1) + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) registry = _prometheus_sink(monkeypatch) team: Final = UserAPIKeyAuth( api_key="sk-interactive", @@ -3619,7 +3623,7 @@ async def test_share_gauges_and_throttles_carry_the_teams_priority_so_the_series async def test_a_fallback_served_request_publishes_the_teams_share_on_the_fallback_model( monkeypatch: pytest.MonkeyPatch, ): - handler = _borrowing_handler(monkeypatch, borrows=True) + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) registry = _prometheus_sink(monkeypatch) team: Final = _team_key(team_limits={"model_tpm_limit": {"other-model": 40, "shared-model": 100}}) assert await handler.async_pre_call_hook(team, DualCache(), {"model": "other-model"}, "completion") is None @@ -3642,7 +3646,7 @@ async def test_a_failing_metrics_sink_never_changes_admission(monkeypatch: pytes def increment_dynamic_limiter_throttled(self, *args: object, **kwargs: object) -> None: raise RuntimeError("metrics backend down") - handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.1) + handler = _borrowing_handler(monkeypatch, borrows=True, saturation_threshold=0.0) _prometheus_sink(monkeypatch, BrokenSink) await _admit_at_the_hook(handler, _team_key(), tokens=60) await _admit_at_the_hook(handler, _team_key(), tokens=70) diff --git a/tests/unit/proxy/management_endpoints/management_v1/test_customer_usage.py b/tests/unit/proxy/management_endpoints/management_v1/test_customer_usage.py new file mode 100644 index 00000000000..af16d11373b --- /dev/null +++ b/tests/unit/proxy/management_endpoints/management_v1/test_customer_usage.py @@ -0,0 +1,278 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from functools import reduce +from typing import Final, cast + +import pytest +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient +from pydantic import BaseModel, ConfigDict + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.list_api.common import ManagementProblem, problem_response +from litellm.proxy.management_endpoints.management_v1.customer_usage import get_customer_usage_prisma, router +from litellm.proxy.utils import PrismaClient +from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics + + +class Record(BaseModel): + model_config = ConfigDict(frozen=True, extra="allow") + + +class DailyRecord(SpendMetrics): + model_config = ConfigDict(frozen=True) + + id: str + date: str + customer_org_id: str + customer_team_id: str + customer_member_id: str + api_key: str + model: str | None = None + model_group: str | None = None + custom_llm_provider: str | None = None + mcp_namespaced_tool_name: str | None = None + endpoint: str | None = None + + +def matches(values: Mapping[str, object], where: Mapping[str, object]) -> bool: + for key, condition in where.items(): + if key == "OR": + if not any(matches(values, clause) for clause in condition): + return False + elif key == "AND": + if not all(matches(values, clause) for clause in condition): + return False + elif isinstance(condition, Mapping): + if "in" in condition and values.get(key) not in condition["in"]: + return False + if "not" in condition and matches(values, {key: condition["not"]}): + return False + if "gte" in condition and str(values.get(key)) < str(condition["gte"]): + return False + if "lte" in condition and str(values.get(key)) > str(condition["lte"]): + return False + elif values.get(key) != condition: + return False + return True + + +@dataclass(frozen=True, slots=True) +class Table: + rows: tuple[Record | DailyRecord, ...] = () + + async def find_many( + self, + where: Mapping[str, object] | None = None, + take: int | None = None, + skip: int = 0, + order: Mapping[str, object] | Sequence[Mapping[str, object]] | None = None, + ) -> tuple[Record | DailyRecord, ...]: + selected: Final = tuple(row for row in self.rows if matches(row.model_dump(), where or {})) + orders: Final = (order,) if isinstance(order, Mapping) else tuple(order or ()) + ordered: Final = reduce( + lambda rows, item: tuple( + sorted( + rows, + key=lambda row: str(row.model_dump()[next(iter(item))]), + reverse=next(iter(item.values())) == "desc", + ) + ), + reversed(orders), + selected, + ) + return ordered[skip:] if take is None else ordered[skip : skip + take] + + async def count(self, where: Mapping[str, object] | None = None) -> int: + return len(await self.find_many(where=where)) + + +@dataclass(frozen=True, slots=True) +class Database: + litellm_dailycustomermemberspend: Table + litellm_customerorganizationtable: Table = Table( + ( + Record(customer_org_id="org-a", alias="Organization A"), + Record(customer_org_id="org-b", alias="Organization B"), + ) + ) + litellm_customerteamtable: Table = Table( + (Record(customer_team_id="team-a", alias="Product"), Record(customer_team_id="team-b", alias="Operations")) + ) + litellm_customermembertable: Table = Table( + ( + Record(customer_member_id="member-a", alias="Member A"), + Record(customer_member_id="member-b", alias="Member B"), + ) + ) + litellm_verificationtoken: Table = Table( + ( + Record(token="hash-a", key_alias="Support", team_id=None, user_id=None, metadata=None), + Record(token="hash-b", key_alias="Search", team_id=None, user_id=None, metadata=None), + ) + ) + litellm_deletedverificationtoken: Table = Table() + litellm_usertable: Table = Table() + + +@dataclass(frozen=True, slots=True) +class Client: + db: Database + + +def usage_row( + *, + id: str = "usage-a", + date: str = "2026-01-01", + customer_org_id: str = "org-a", + customer_team_id: str = "team-a", + customer_member_id: str = "member-a", + api_key: str = "hash-a", + model: str = "model-a", + model_group: str = "alias-a", + provider: str = "provider-a", + spend: float = 2.0, +) -> DailyRecord: + return DailyRecord( + id=id, + date=date, + customer_org_id=customer_org_id, + customer_team_id=customer_team_id, + customer_member_id=customer_member_id, + api_key=api_key, + model=model, + model_group=model_group, + custom_llm_provider=provider, + spend=spend, + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + api_requests=3, + successful_requests=2, + failed_requests=1, + ) + + +def client(rows: tuple[DailyRecord, ...] = (), role: LitellmUserRoles = LitellmUserRoles.PROXY_ADMIN) -> TestClient: + app: Final = FastAPI() + app.include_router(router) + + @app.exception_handler(ManagementProblem) + async def problem(request: Request, exc: ManagementProblem): + return problem_response(exc.problem) + + auth: Final = UserAPIKeyAuth(user_id="admin", user_role=role) + prisma: Final = cast(PrismaClient, Client(Database(Table(rows)))) # cast-ok: in-memory Prisma boundary + app.dependency_overrides[user_api_key_auth] = lambda: auth + app.dependency_overrides[get_customer_usage_prisma] = lambda: prisma + return TestClient(app) + + +DATES: Final = {"start_date": "2026-01-01", "end_date": "2026-01-03"} +SCOPES: Final = ( + ("customer_organizations", "customer_org_ids", "exclude_customer_org_ids", "org-a", "org-b", "Organization A"), + ("customer_teams", "customer_team_ids", "exclude_customer_team_ids", "team-a", "team-b", "Product"), + ("customer_members", "customer_member_ids", "exclude_customer_member_ids", "member-a", "member-b", "Member A"), +) + + +@pytest.mark.parametrize("resource,parameter,excluded,first,second,label", SCOPES) +def test_standard_daily_activity_filters_entities_and_preserves_aliases_and_dimensions( + resource: str, parameter: str, excluded: str, first: str, second: str, label: str +) -> None: + rows: Final = ( + usage_row(), + usage_row(id="usage-c", date="2026-01-03", model_group="alias-b", provider="provider-b", spend=4), + usage_row( + id="usage-b", customer_org_id="org-b", customer_team_id="team-b", customer_member_id="member-b", spend=10 + ), + usage_row(id="outside-range", date="2025-12-31", spend=50), + ) + response: Final = client(rows).get(f"/management/v1/{resource}/daily/activity", params={**DATES, parameter: first}) + assert response.status_code == 200, response.text + body: Final = response.json() + assert set(body) == {"results", "metadata"} + assert body["metadata"]["total_spend"] == 6 + assert body["metadata"]["total_tokens"] == 30 + assert body["metadata"]["total_api_requests"] == 6 + assert body["metadata"]["has_more"] is False + day: Final = body["results"][0] + assert day["breakdown"]["entities"][first]["metadata"]["alias"] == label + assert day["breakdown"]["models"]["model-a"]["metrics"]["spend"] == 4 + assert day["breakdown"]["model_groups"]["alias-b"]["metrics"]["spend"] == 4 + assert day["breakdown"]["providers"]["provider-b"]["metrics"]["spend"] == 4 + assert day["breakdown"]["api_keys"]["hash-a"]["metadata"]["key_alias"] == "Support" + + +@pytest.mark.parametrize("resource,parameter,excluded,first,second,label", SCOPES) +def test_inclusion_and_exclusion_lists_are_comma_separated( + resource: str, parameter: str, excluded: str, first: str, second: str, label: str +) -> None: + rows: Final = ( + usage_row(), + usage_row( + id="usage-b", customer_org_id="org-b", customer_team_id="team-b", customer_member_id="member-b", spend=10 + ), + ) + response: Final = client(rows).get( + f"/management/v1/{resource}/daily/activity", params={**DATES, parameter: f"{first},{second}", excluded: second} + ) + assert response.status_code == 200, response.text + assert response.json()["metadata"]["total_spend"] == 2 + + +@pytest.mark.parametrize("resource,parameter,excluded,first,second,label", SCOPES) +def test_standard_model_key_and_page_filters_use_the_shared_response_contract( + resource: str, parameter: str, excluded: str, first: str, second: str, label: str +) -> None: + rows: Final = ( + usage_row(id="page-a"), + usage_row(id="page-b", date="2026-01-02", spend=4), + usage_row(id="other-model", model="model-b", spend=20), + usage_row(id="other-key", api_key="hash-b", spend=30), + ) + response: Final = client(rows).get( + f"/management/v1/{resource}/daily/activity", + params={**DATES, parameter: first, "model": "model-a", "api_key": "hash-a", "page": 1, "page_size": 1}, + ) + assert response.status_code == 200, response.text + body: Final = response.json() + assert body["metadata"]["page"] == 1 + assert body["metadata"]["total_pages"] == 2 + assert body["metadata"]["has_more"] is True + assert body["metadata"]["total_spend"] == 4 + assert body["results"][0]["date"] == "2026-01-02" + next_page: Final = client(rows).get( + f"/management/v1/{resource}/daily/activity", + params={**DATES, parameter: first, "model": "model-a", "api_key": "hash-a", "page": 2, "page_size": 1}, + ) + assert next_page.status_code == 200, next_page.text + assert next_page.json()["metadata"]["total_spend"] == 2 + assert next_page.json()["metadata"]["has_more"] is False + + +@pytest.mark.parametrize("resource,parameter,excluded,first,second,label", SCOPES) +@pytest.mark.parametrize("role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)) +def test_all_hierarchy_daily_activity_routes_are_admin_only( + resource: str, parameter: str, excluded: str, first: str, second: str, label: str, role: LitellmUserRoles +) -> None: + response: Final = client(role=role).get(f"/management/v1/{resource}/daily/activity", params=DATES) + assert response.status_code == 403, response.text + + +@pytest.mark.parametrize("params", ({}, {"start_date": "invalid", "end_date": "2026-01-03"}, {**DATES, "page": 0})) +def test_shared_helper_rejects_missing_dates_invalid_dates_and_nonpositive_pagination( + params: Mapping[str, object], +) -> None: + response: Final = client().get("/management/v1/customer_members/daily/activity", params=params) + assert response.status_code == 400, response.text + + +def test_unmatched_entities_return_the_standard_empty_response() -> None: + response: Final = client((usage_row(),)).get( + "/management/v1/customer_members/daily/activity", params={**DATES, "customer_member_ids": "missing"} + ) + assert response.status_code == 200, response.text + assert response.json()["results"] == [] + assert response.json()["metadata"]["total_spend"] == 0 diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index ea36ffad044..e7bc5675e06 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -41,6 +41,7 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import ( ) from litellm.proxy.proxy_server import app from litellm.types.proxy.auth.auth_checks import UserNotFoundError +from litellm.types.proxy.common_proxy_errors import CommonProxyErrors from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains from tests.unit.proxy.management_endpoints.jwt_key_mapping_doubles import ( CascadingJWTMappingTable, @@ -50,6 +51,30 @@ from tests.unit.proxy.management_endpoints.jwt_key_mapping_doubles import ( client = TestClient(app) +@pytest.mark.asyncio +@pytest.mark.parametrize("binding", ["customer_org_id", "customer_team_id", "customer_member_id"]) +async def test_new_user_cannot_mint_customer_bound_keys_without_proxy_admin(binding: str) -> None: + with pytest.raises(HTTPException) as caught: + await new_user( + data=NewUserRequest(metadata={binding: "another-customer"}), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER), + ) + assert caught.value.status_code == 403 + assert caught.value.detail == "Only proxy admins can change a key's customer organization, team, or member binding." + + +@pytest.mark.asyncio +async def test_new_user_reports_missing_database_after_customer_binding_validation(mocker: MockerFixture) -> None: + mocker.patch("litellm.proxy.proxy_server.prisma_client", None) + with pytest.raises(ProxyException) as caught: + await new_user( + data=NewUserRequest(metadata={"customer_org_id": "org-1"}), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert str(caught.value.code) == "400" + assert caught.value.message == CommonProxyErrors.db_not_connected_error.value + + @pytest.mark.parametrize("user_id", ["", " \t\n", "user\x00id", 123, True, [], {}]) def test_new_user_rejects_invalid_custom_id(user_id: object) -> None: with pytest.raises(ValidationError) as exc: diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index c67cb974583..9e0e123436a 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from contextlib import ExitStack from datetime import datetime, timedelta, timezone from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, call, patch from uuid import UUID, uuid4 @@ -24618,3 +24618,89 @@ def test_user_new_without_a_key_does_not_check_guardrail_names(spend_permission_ assert response.status_code == 200, response.text find_many.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ("customer_org_id", "customer_team_id", "customer_member_id")) +async def test_customer_binding_requires_admin_on_key_creation(field: str) -> None: + with pytest.raises(HTTPException) as rejected: + await _common_key_generation_helper( + data=GenerateKeyRequest(metadata={field: "customer-id"}), + user_api_key_dict=UserAPIKeyAuth(user_id="member", user_role=LitellmUserRoles.INTERNAL_USER), + litellm_changed_by=None, + team_table=None, + ) + assert rejected.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_type", ("key", "user")) +async def test_direct_key_mint_cannot_bypass_customer_binding_authorization(request_type: Literal["key", "user"]) -> None: + with pytest.raises(HTTPException) as rejected: + await generate_key_helper_fn(request_type=request_type, metadata={"customer_org_id": "another-customer"}) + assert rejected.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ("update", "regenerate")) +@pytest.mark.parametrize("field", ("customer_org_id", "customer_team_id", "customer_member_id")) +@pytest.mark.parametrize("requested", ("another-customer-id", None)) +async def test_customer_binding_cannot_be_reassigned_or_cleared_by_key_holder( + operation: Literal["update", "regenerate"], field: str, requested: str | None +) -> None: + existing: Final = LiteLLM_VerificationToken(token="hashed-key", metadata={field: "customer-id"}) + data: Final = ( + UpdateKeyRequest(key="hashed-key", metadata={field: requested}) + if operation == "update" + else RegenerateKeyRequest(metadata={field: requested}) + ) + with pytest.raises(HTTPException) as rejected: + await prepare_key_update_data( + data=data, + existing_key_row=existing, + user_api_key_dict=UserAPIKeyAuth(user_id="member", user_role=LitellmUserRoles.INTERNAL_USER), + ) + assert rejected.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ("update", "regenerate")) +@pytest.mark.parametrize("metadata", ({"application": "support"}, {}, None)) +async def test_unrelated_key_metadata_changes_preserve_existing_customer_binding( + operation: Literal["update", "regenerate"], metadata: dict[str, object] | None +) -> None: + bindings: Final = {"customer_org_id": "org-id", "customer_team_id": "team-id", "customer_member_id": "member-id"} + existing: Final = LiteLLM_VerificationToken(token="hashed-key", metadata={**bindings, "old": "discard"}) + data: Final = ( + UpdateKeyRequest(key="hashed-key", metadata=metadata) + if operation == "update" + else RegenerateKeyRequest(metadata=metadata) + ) + actual: Final = await prepare_key_update_data( + data=data, + existing_key_row=existing, + user_api_key_dict=UserAPIKeyAuth(user_id="member", user_role=LitellmUserRoles.INTERNAL_USER), + ) + assert actual["metadata"] == {**(metadata or {}), **bindings} + assert existing.metadata == {**bindings, "old": "discard"} + assert data.metadata == metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ("update", "regenerate")) +@pytest.mark.parametrize("metadata", ({"customer_org_id": "another-org"}, {}, None)) +async def test_proxy_admin_can_replace_or_clear_customer_key_binding( + operation: Literal["update", "regenerate"], metadata: dict[str, object] | None +) -> None: + existing: Final = LiteLLM_VerificationToken(token="hashed-key", metadata={"customer_org_id": "org-id"}) + data: Final = ( + UpdateKeyRequest(key="hashed-key", metadata=metadata) + if operation == "update" + else RegenerateKeyRequest(metadata=metadata) + ) + actual: Final = await prepare_key_update_data( + data=data, + existing_key_row=existing, + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert actual["metadata"] == metadata diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 337aa2939dd..f4138260e28 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -4097,7 +4097,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"user_api_key_customer_usage": null, "user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30,