mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
Merge pull request #2236 from BerriAI/litellm_faster_usage_logs_pg_db
fix(proxy_server.py): introduces a beta endpoint for admin to view global spend
This commit is contained in:
Krish Dholakia
GitHub
commit
0633eecdb8
37 changed files with 1352 additions and 1005 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-a697f24d60c9c262.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a85b2c176012d8e5.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e1b183dda365ec86.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>🚅 LiteLLM</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-a697f24d60c9c262.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/6920a121699cde9c.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[24143,[\"303\",\"static/chunks/303-d80f23087a9e6aec.js\",\"931\",\"static/chunks/app/page-d4fe4a48cbd3572c.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/6920a121699cde9c.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"eSwVwl_InIrhYtCAqDMKF\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"🚅 LiteLLM\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-a697f24d60c9c262.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a85b2c176012d8e5.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e1b183dda365ec86.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>🚅 LiteLLM</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-a697f24d60c9c262.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/6920a121699cde9c.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[24143,[\"303\",\"static/chunks/303-d80f23087a9e6aec.js\",\"931\",\"static/chunks/app/page-cc9d300e3b13fc1b.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/6920a121699cde9c.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"h6IXdBMiZG7ES547qg1M-\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"🚅 LiteLLM\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[77831,[],""]
|
||||
3:I[24143,["303","static/chunks/303-d80f23087a9e6aec.js","931","static/chunks/app/page-d4fe4a48cbd3572c.js"],""]
|
||||
3:I[24143,["303","static/chunks/303-d80f23087a9e6aec.js","931","static/chunks/app/page-cc9d300e3b13fc1b.js"],""]
|
||||
4:I[5613,[],""]
|
||||
5:I[31778,[],""]
|
||||
0:["eSwVwl_InIrhYtCAqDMKF",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/6920a121699cde9c.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
0:["h6IXdBMiZG7ES547qg1M-",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/6920a121699cde9c.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"🚅 LiteLLM"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -2315,7 +2315,6 @@ async def startup_event():
|
|||
### CHECK IF VIEW EXISTS ###
|
||||
if prisma_client is not None:
|
||||
create_view_response = await prisma_client.check_view_exists()
|
||||
print(f"create_view_response: {create_view_response}") # noqa
|
||||
|
||||
### START BUDGET SCHEDULER ###
|
||||
if prisma_client is not None:
|
||||
|
|
@ -3788,7 +3787,7 @@ async def view_spend_tags(
|
|||
|
||||
@router.get(
|
||||
"/spend/logs",
|
||||
tags=["budget & spend Tracking"],
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
responses={
|
||||
200: {"model": List[LiteLLM_SpendLogs]},
|
||||
|
|
@ -4048,6 +4047,83 @@ async def view_spend_logs(
|
|||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/global/spend/logs",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def global_spend_logs():
|
||||
"""
|
||||
[BETA] This is a beta endpoint. It will change.
|
||||
|
||||
Use this to get global spend (spend per day for last 30d). Admin-only endpoint
|
||||
|
||||
More efficient implementation of /spend/logs, by creating a view over the spend logs table.
|
||||
"""
|
||||
global prisma_client
|
||||
|
||||
sql_query = """SELECT * FROM "MonthlyGlobalSpend";"""
|
||||
|
||||
response = await prisma_client.db.query_raw(query=sql_query)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@router.get(
|
||||
"/global/spend/keys",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def global_spend_keys(
|
||||
limit: int = fastapi.Query(
|
||||
default=None,
|
||||
description="Number of keys to get. Will return Top 'n' keys.",
|
||||
)
|
||||
):
|
||||
"""
|
||||
[BETA] This is a beta endpoint. It will change.
|
||||
|
||||
Use this to get the top 'n' keys with the highest spend, ordered by spend.
|
||||
"""
|
||||
global prisma_client
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(status_code=500, detail={"error": "No db connected"})
|
||||
sql_query = f"""SELECT * FROM "Last30dKeysBySpend" LIMIT {limit};"""
|
||||
|
||||
response = await prisma_client.db.query_raw(query=sql_query)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@router.get(
|
||||
"/global/spend/models",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def global_spend_models(
|
||||
limit: int = fastapi.Query(
|
||||
default=None,
|
||||
description="Number of models to get. Will return Top 'n' models.",
|
||||
)
|
||||
):
|
||||
"""
|
||||
[BETA] This is a beta endpoint. It will change.
|
||||
|
||||
Use this to get the top 'n' keys with the highest spend, ordered by spend.
|
||||
"""
|
||||
global prisma_client
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(status_code=500, detail={"error": "No db connected"})
|
||||
|
||||
sql_query = f"""SELECT * FROM "Last30dModelsBySpend" LIMIT {limit};"""
|
||||
|
||||
response = await prisma_client.db.query_raw(query=sql_query)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@router.get(
|
||||
"/daily_metrics",
|
||||
summary="Get daily spend metrics",
|
||||
|
|
@ -4064,7 +4140,11 @@ async def view_daily_metrics(
|
|||
description="Time till which to view key spend",
|
||||
),
|
||||
):
|
||||
""" """
|
||||
"""
|
||||
[BETA] This is a beta endpoint. It might change without notice.
|
||||
|
||||
Please give feedback - https://github.com/BerriAI/litellm/issues
|
||||
"""
|
||||
try:
|
||||
if os.getenv("CLICKHOUSE_HOST") is not None:
|
||||
# gettting spend logs from clickhouse
|
||||
|
|
|
|||
|
|
@ -489,18 +489,20 @@ class PrismaClient:
|
|||
)
|
||||
async def check_view_exists(self):
|
||||
"""
|
||||
Checks if the LiteLLM_VerificationTokenView exists in the user's db.
|
||||
Checks if the LiteLLM_VerificationTokenView and MonthlyGlobalSpend exists in the user's db.
|
||||
|
||||
This is used for getting the token + team data in user_api_key_auth
|
||||
LiteLLM_VerificationTokenView: This view is used for getting the token + team data in user_api_key_auth
|
||||
|
||||
MonthlyGlobalSpend: This view is used for the admin view to see global spend for this month
|
||||
|
||||
If the view doesn't exist, one will be created.
|
||||
"""
|
||||
try:
|
||||
# Try to select one row from the view
|
||||
await self.db.execute_raw(
|
||||
await self.db.query_raw(
|
||||
"""SELECT 1 FROM "LiteLLM_VerificationTokenView" LIMIT 1"""
|
||||
)
|
||||
return "LiteLLM_VerificationTokenView Exists!"
|
||||
print("LiteLLM_VerificationTokenView Exists!") # noqa
|
||||
except Exception as e:
|
||||
# If an error occurs, the view does not exist, so create it
|
||||
value = await self.health_check()
|
||||
|
|
@ -518,7 +520,29 @@ class PrismaClient:
|
|||
"""
|
||||
)
|
||||
|
||||
return "LiteLLM_VerificationTokenView Created!"
|
||||
print("LiteLLM_VerificationTokenView Created!") # noqa
|
||||
|
||||
try:
|
||||
await self.db.query_raw("""SELECT 1 FROM "MonthlyGlobalSpend" LIMIT 1""")
|
||||
print("MonthlyGlobalSpend Exists!") # noqa
|
||||
except Exception as e:
|
||||
sql_query = """
|
||||
CREATE OR REPLACE VIEW "MonthlyGlobalSpend" AS
|
||||
SELECT
|
||||
DATE("startTime") AS date,
|
||||
SUM("spend") AS spend
|
||||
FROM
|
||||
"LiteLLM_SpendLogs"
|
||||
WHERE
|
||||
"startTime" >= (CURRENT_DATE - INTERVAL '30 days')
|
||||
GROUP BY
|
||||
DATE("startTime");
|
||||
"""
|
||||
await self.db.execute_raw(query=sql_query)
|
||||
|
||||
print("MonthlyGlobalSpend Created!") # noqa
|
||||
|
||||
return
|
||||
|
||||
@backoff.on_exception(
|
||||
backoff.expo,
|
||||
|
|
|
|||
|
|
@ -1,253 +1,254 @@
|
|||
import sys
|
||||
import os
|
||||
import io, asyncio
|
||||
## @pytest.mark.skip(reason="AWS Suspended Account")
|
||||
# import sys
|
||||
# import os
|
||||
# import io, asyncio
|
||||
|
||||
# import logging
|
||||
# logging.basicConfig(level=logging.DEBUG)
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
# # import logging
|
||||
# # logging.basicConfig(level=logging.DEBUG)
|
||||
# sys.path.insert(0, os.path.abspath("../.."))
|
||||
|
||||
from litellm import completion
|
||||
import litellm
|
||||
# from litellm import completion
|
||||
# import litellm
|
||||
|
||||
litellm.num_retries = 3
|
||||
# litellm.num_retries = 3
|
||||
|
||||
import time, random
|
||||
import pytest
|
||||
# import time, random
|
||||
# import pytest
|
||||
|
||||
|
||||
def test_s3_logging():
|
||||
# all s3 requests need to be in one test function
|
||||
# since we are modifying stdout, and pytests runs tests in parallel
|
||||
# on circle ci - we only test litellm.acompletion()
|
||||
try:
|
||||
# redirect stdout to log_file
|
||||
litellm.cache = litellm.Cache(
|
||||
type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2"
|
||||
)
|
||||
# def test_s3_logging():
|
||||
# # all s3 requests need to be in one test function
|
||||
# # since we are modifying stdout, and pytests runs tests in parallel
|
||||
# # on circle ci - we only test litellm.acompletion()
|
||||
# try:
|
||||
# # redirect stdout to log_file
|
||||
# litellm.cache = litellm.Cache(
|
||||
# type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2"
|
||||
# )
|
||||
|
||||
litellm.success_callback = ["s3"]
|
||||
litellm.s3_callback_params = {
|
||||
"s3_bucket_name": "litellm-logs",
|
||||
"s3_aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
"s3_aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID",
|
||||
}
|
||||
litellm.set_verbose = True
|
||||
# litellm.success_callback = ["s3"]
|
||||
# litellm.s3_callback_params = {
|
||||
# "s3_bucket_name": "litellm-logs",
|
||||
# "s3_aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
# "s3_aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID",
|
||||
# }
|
||||
# litellm.set_verbose = True
|
||||
|
||||
print("Testing async s3 logging")
|
||||
# print("Testing async s3 logging")
|
||||
|
||||
expected_keys = []
|
||||
# expected_keys = []
|
||||
|
||||
import time
|
||||
# import time
|
||||
|
||||
curr_time = str(time.time())
|
||||
# curr_time = str(time.time())
|
||||
|
||||
async def _test():
|
||||
return await litellm.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": f"This is a test {curr_time}"}],
|
||||
max_tokens=10,
|
||||
temperature=0.7,
|
||||
user="ishaan-2",
|
||||
)
|
||||
# async def _test():
|
||||
# return await litellm.acompletion(
|
||||
# model="gpt-3.5-turbo",
|
||||
# messages=[{"role": "user", "content": f"This is a test {curr_time}"}],
|
||||
# max_tokens=10,
|
||||
# temperature=0.7,
|
||||
# user="ishaan-2",
|
||||
# )
|
||||
|
||||
response = asyncio.run(_test())
|
||||
print(f"response: {response}")
|
||||
expected_keys.append(response.id)
|
||||
# response = asyncio.run(_test())
|
||||
# print(f"response: {response}")
|
||||
# expected_keys.append(response.id)
|
||||
|
||||
async def _test():
|
||||
return await litellm.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": f"This is a test {curr_time}"}],
|
||||
max_tokens=10,
|
||||
temperature=0.7,
|
||||
user="ishaan-2",
|
||||
)
|
||||
# async def _test():
|
||||
# return await litellm.acompletion(
|
||||
# model="gpt-3.5-turbo",
|
||||
# messages=[{"role": "user", "content": f"This is a test {curr_time}"}],
|
||||
# max_tokens=10,
|
||||
# temperature=0.7,
|
||||
# user="ishaan-2",
|
||||
# )
|
||||
|
||||
response = asyncio.run(_test())
|
||||
expected_keys.append(response.id)
|
||||
print(f"response: {response}")
|
||||
time.sleep(5) # wait 5s for logs to land
|
||||
# response = asyncio.run(_test())
|
||||
# expected_keys.append(response.id)
|
||||
# print(f"response: {response}")
|
||||
# time.sleep(5) # wait 5s for logs to land
|
||||
|
||||
import boto3
|
||||
# import boto3
|
||||
|
||||
s3 = boto3.client("s3")
|
||||
bucket_name = "litellm-logs"
|
||||
# List objects in the bucket
|
||||
response = s3.list_objects(Bucket=bucket_name)
|
||||
# s3 = boto3.client("s3")
|
||||
# bucket_name = "litellm-logs"
|
||||
# # List objects in the bucket
|
||||
# response = s3.list_objects(Bucket=bucket_name)
|
||||
|
||||
# Sort the objects based on the LastModified timestamp
|
||||
objects = sorted(
|
||||
response["Contents"], key=lambda x: x["LastModified"], reverse=True
|
||||
)
|
||||
# Get the keys of the most recent objects
|
||||
most_recent_keys = [obj["Key"] for obj in objects]
|
||||
print(most_recent_keys)
|
||||
# for each key, get the part before "-" as the key. Do it safely
|
||||
cleaned_keys = []
|
||||
for key in most_recent_keys:
|
||||
split_key = key.split("_")
|
||||
if len(split_key) < 2:
|
||||
continue
|
||||
cleaned_keys.append(split_key[1])
|
||||
print("\n most recent keys", most_recent_keys)
|
||||
print("\n cleaned keys", cleaned_keys)
|
||||
print("\n Expected keys: ", expected_keys)
|
||||
matches = 0
|
||||
for key in expected_keys:
|
||||
key += ".json"
|
||||
assert key in cleaned_keys
|
||||
# # Sort the objects based on the LastModified timestamp
|
||||
# objects = sorted(
|
||||
# response["Contents"], key=lambda x: x["LastModified"], reverse=True
|
||||
# )
|
||||
# # Get the keys of the most recent objects
|
||||
# most_recent_keys = [obj["Key"] for obj in objects]
|
||||
# print(most_recent_keys)
|
||||
# # for each key, get the part before "-" as the key. Do it safely
|
||||
# cleaned_keys = []
|
||||
# for key in most_recent_keys:
|
||||
# split_key = key.split("_")
|
||||
# if len(split_key) < 2:
|
||||
# continue
|
||||
# cleaned_keys.append(split_key[1])
|
||||
# print("\n most recent keys", most_recent_keys)
|
||||
# print("\n cleaned keys", cleaned_keys)
|
||||
# print("\n Expected keys: ", expected_keys)
|
||||
# matches = 0
|
||||
# for key in expected_keys:
|
||||
# key += ".json"
|
||||
# assert key in cleaned_keys
|
||||
|
||||
if key in cleaned_keys:
|
||||
matches += 1
|
||||
# remove the match key
|
||||
cleaned_keys.remove(key)
|
||||
# this asserts we log, the first request + the 2nd cached request
|
||||
print("we had two matches ! passed ", matches)
|
||||
assert matches == 2
|
||||
try:
|
||||
# cleanup s3 bucket in test
|
||||
for key in most_recent_keys:
|
||||
s3.delete_object(Bucket=bucket_name, Key=key)
|
||||
except:
|
||||
# don't let cleanup fail a test
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"An exception occurred - {e}")
|
||||
finally:
|
||||
# post, close log file and verify
|
||||
# Reset stdout to the original value
|
||||
print("Passed! Testing async s3 logging")
|
||||
# if key in cleaned_keys:
|
||||
# matches += 1
|
||||
# # remove the match key
|
||||
# cleaned_keys.remove(key)
|
||||
# # this asserts we log, the first request + the 2nd cached request
|
||||
# print("we had two matches ! passed ", matches)
|
||||
# assert matches == 2
|
||||
# try:
|
||||
# # cleanup s3 bucket in test
|
||||
# for key in most_recent_keys:
|
||||
# s3.delete_object(Bucket=bucket_name, Key=key)
|
||||
# except:
|
||||
# # don't let cleanup fail a test
|
||||
# pass
|
||||
# except Exception as e:
|
||||
# pytest.fail(f"An exception occurred - {e}")
|
||||
# finally:
|
||||
# # post, close log file and verify
|
||||
# # Reset stdout to the original value
|
||||
# print("Passed! Testing async s3 logging")
|
||||
|
||||
|
||||
# test_s3_logging()
|
||||
# # test_s3_logging()
|
||||
|
||||
|
||||
def test_s3_logging_async():
|
||||
# this tests time added to make s3 logging calls, vs just acompletion calls
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
# Make 5 calls with an empty success_callback
|
||||
litellm.success_callback = []
|
||||
start_time_empty_callback = asyncio.run(make_async_calls())
|
||||
print("done with no callback test")
|
||||
# def test_s3_logging_async():
|
||||
# # this tests time added to make s3 logging calls, vs just acompletion calls
|
||||
# try:
|
||||
# litellm.set_verbose = True
|
||||
# # Make 5 calls with an empty success_callback
|
||||
# litellm.success_callback = []
|
||||
# start_time_empty_callback = asyncio.run(make_async_calls())
|
||||
# print("done with no callback test")
|
||||
|
||||
print("starting s3 logging load test")
|
||||
# Make 5 calls with success_callback set to "langfuse"
|
||||
litellm.success_callback = ["s3"]
|
||||
litellm.s3_callback_params = {
|
||||
"s3_bucket_name": "litellm-logs",
|
||||
"s3_aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
"s3_aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID",
|
||||
}
|
||||
start_time_s3 = asyncio.run(make_async_calls())
|
||||
print("done with s3 test")
|
||||
# print("starting s3 logging load test")
|
||||
# # Make 5 calls with success_callback set to "langfuse"
|
||||
# litellm.success_callback = ["s3"]
|
||||
# litellm.s3_callback_params = {
|
||||
# "s3_bucket_name": "litellm-logs",
|
||||
# "s3_aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
# "s3_aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID",
|
||||
# }
|
||||
# start_time_s3 = asyncio.run(make_async_calls())
|
||||
# print("done with s3 test")
|
||||
|
||||
# Compare the time for both scenarios
|
||||
print(f"Time taken with success_callback='s3': {start_time_s3}")
|
||||
print(f"Time taken with empty success_callback: {start_time_empty_callback}")
|
||||
# # Compare the time for both scenarios
|
||||
# print(f"Time taken with success_callback='s3': {start_time_s3}")
|
||||
# print(f"Time taken with empty success_callback: {start_time_empty_callback}")
|
||||
|
||||
# assert the diff is not more than 1 second
|
||||
assert abs(start_time_s3 - start_time_empty_callback) < 1
|
||||
# # assert the diff is not more than 1 second
|
||||
# assert abs(start_time_s3 - start_time_empty_callback) < 1
|
||||
|
||||
except litellm.Timeout as e:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"An exception occurred - {e}")
|
||||
# except litellm.Timeout as e:
|
||||
# pass
|
||||
# except Exception as e:
|
||||
# pytest.fail(f"An exception occurred - {e}")
|
||||
|
||||
|
||||
async def make_async_calls():
|
||||
tasks = []
|
||||
for _ in range(5):
|
||||
task = asyncio.create_task(
|
||||
litellm.acompletion(
|
||||
model="azure/chatgpt-v-2",
|
||||
messages=[{"role": "user", "content": "This is a test"}],
|
||||
max_tokens=5,
|
||||
temperature=0.7,
|
||||
timeout=5,
|
||||
user="langfuse_latency_test_user",
|
||||
mock_response="It's simple to use and easy to get started",
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
# async def make_async_calls():
|
||||
# tasks = []
|
||||
# for _ in range(5):
|
||||
# task = asyncio.create_task(
|
||||
# litellm.acompletion(
|
||||
# model="azure/chatgpt-v-2",
|
||||
# messages=[{"role": "user", "content": "This is a test"}],
|
||||
# max_tokens=5,
|
||||
# temperature=0.7,
|
||||
# timeout=5,
|
||||
# user="langfuse_latency_test_user",
|
||||
# mock_response="It's simple to use and easy to get started",
|
||||
# )
|
||||
# )
|
||||
# tasks.append(task)
|
||||
|
||||
# Measure the start time before running the tasks
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
# # Measure the start time before running the tasks
|
||||
# start_time = asyncio.get_event_loop().time()
|
||||
|
||||
# Wait for all tasks to complete
|
||||
responses = await asyncio.gather(*tasks)
|
||||
# # Wait for all tasks to complete
|
||||
# responses = await asyncio.gather(*tasks)
|
||||
|
||||
# Print the responses when tasks return
|
||||
for idx, response in enumerate(responses):
|
||||
print(f"Response from Task {idx + 1}: {response}")
|
||||
# # Print the responses when tasks return
|
||||
# for idx, response in enumerate(responses):
|
||||
# print(f"Response from Task {idx + 1}: {response}")
|
||||
|
||||
# Calculate the total time taken
|
||||
total_time = asyncio.get_event_loop().time() - start_time
|
||||
# # Calculate the total time taken
|
||||
# total_time = asyncio.get_event_loop().time() - start_time
|
||||
|
||||
return total_time
|
||||
# return total_time
|
||||
|
||||
|
||||
def test_s3_logging_r2():
|
||||
# all s3 requests need to be in one test function
|
||||
# since we are modifying stdout, and pytests runs tests in parallel
|
||||
# on circle ci - we only test litellm.acompletion()
|
||||
try:
|
||||
# redirect stdout to log_file
|
||||
# litellm.cache = litellm.Cache(
|
||||
# type="s3", s3_bucket_name="litellm-r2-bucket", s3_region_name="us-west-2"
|
||||
# )
|
||||
litellm.set_verbose = True
|
||||
from litellm._logging import verbose_logger
|
||||
import logging
|
||||
# def test_s3_logging_r2():
|
||||
# # all s3 requests need to be in one test function
|
||||
# # since we are modifying stdout, and pytests runs tests in parallel
|
||||
# # on circle ci - we only test litellm.acompletion()
|
||||
# try:
|
||||
# # redirect stdout to log_file
|
||||
# # litellm.cache = litellm.Cache(
|
||||
# # type="s3", s3_bucket_name="litellm-r2-bucket", s3_region_name="us-west-2"
|
||||
# # )
|
||||
# litellm.set_verbose = True
|
||||
# from litellm._logging import verbose_logger
|
||||
# import logging
|
||||
|
||||
verbose_logger.setLevel(level=logging.DEBUG)
|
||||
# verbose_logger.setLevel(level=logging.DEBUG)
|
||||
|
||||
litellm.success_callback = ["s3"]
|
||||
litellm.s3_callback_params = {
|
||||
"s3_bucket_name": "litellm-r2-bucket",
|
||||
"s3_aws_secret_access_key": "os.environ/R2_S3_ACCESS_KEY",
|
||||
"s3_aws_access_key_id": "os.environ/R2_S3_ACCESS_ID",
|
||||
"s3_endpoint_url": "os.environ/R2_S3_URL",
|
||||
"s3_region_name": "os.environ/R2_S3_REGION_NAME",
|
||||
}
|
||||
print("Testing async s3 logging")
|
||||
# litellm.success_callback = ["s3"]
|
||||
# litellm.s3_callback_params = {
|
||||
# "s3_bucket_name": "litellm-r2-bucket",
|
||||
# "s3_aws_secret_access_key": "os.environ/R2_S3_ACCESS_KEY",
|
||||
# "s3_aws_access_key_id": "os.environ/R2_S3_ACCESS_ID",
|
||||
# "s3_endpoint_url": "os.environ/R2_S3_URL",
|
||||
# "s3_region_name": "os.environ/R2_S3_REGION_NAME",
|
||||
# }
|
||||
# print("Testing async s3 logging")
|
||||
|
||||
expected_keys = []
|
||||
# expected_keys = []
|
||||
|
||||
import time
|
||||
# import time
|
||||
|
||||
curr_time = str(time.time())
|
||||
# curr_time = str(time.time())
|
||||
|
||||
async def _test():
|
||||
return await litellm.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": f"This is a test {curr_time}"}],
|
||||
max_tokens=10,
|
||||
temperature=0.7,
|
||||
user="ishaan-2",
|
||||
)
|
||||
# async def _test():
|
||||
# return await litellm.acompletion(
|
||||
# model="gpt-3.5-turbo",
|
||||
# messages=[{"role": "user", "content": f"This is a test {curr_time}"}],
|
||||
# max_tokens=10,
|
||||
# temperature=0.7,
|
||||
# user="ishaan-2",
|
||||
# )
|
||||
|
||||
response = asyncio.run(_test())
|
||||
print(f"response: {response}")
|
||||
expected_keys.append(response.id)
|
||||
# response = asyncio.run(_test())
|
||||
# print(f"response: {response}")
|
||||
# expected_keys.append(response.id)
|
||||
|
||||
import boto3
|
||||
# import boto3
|
||||
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.getenv("R2_S3_URL"),
|
||||
region_name=os.getenv("R2_S3_REGION_NAME"),
|
||||
aws_access_key_id=os.getenv("R2_S3_ACCESS_ID"),
|
||||
aws_secret_access_key=os.getenv("R2_S3_ACCESS_KEY"),
|
||||
)
|
||||
# s3 = boto3.client(
|
||||
# "s3",
|
||||
# endpoint_url=os.getenv("R2_S3_URL"),
|
||||
# region_name=os.getenv("R2_S3_REGION_NAME"),
|
||||
# aws_access_key_id=os.getenv("R2_S3_ACCESS_ID"),
|
||||
# aws_secret_access_key=os.getenv("R2_S3_ACCESS_KEY"),
|
||||
# )
|
||||
|
||||
bucket_name = "litellm-r2-bucket"
|
||||
# List objects in the bucket
|
||||
response = s3.list_objects(Bucket=bucket_name)
|
||||
# bucket_name = "litellm-r2-bucket"
|
||||
# # List objects in the bucket
|
||||
# response = s3.list_objects(Bucket=bucket_name)
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail(f"An exception occurred - {e}")
|
||||
finally:
|
||||
# post, close log file and verify
|
||||
# Reset stdout to the original value
|
||||
print("Passed! Testing async s3 logging")
|
||||
# except Exception as e:
|
||||
# pytest.fail(f"An exception occurred - {e}")
|
||||
# finally:
|
||||
# # post, close log file and verify
|
||||
# # Reset stdout to the original value
|
||||
# print("Passed! Testing async s3 logging")
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ async def test_hf_completion_tgi():
|
|||
# test_get_cloudflare_response_streaming()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_sagemaker():
|
||||
# litellm.set_verbose=True
|
||||
|
|
|
|||
|
|
@ -1,257 +1,259 @@
|
|||
import sys, os
|
||||
import traceback
|
||||
from dotenv import load_dotenv
|
||||
# @pytest.mark.skip(reason="AWS Suspended Account")
|
||||
# import sys, os
|
||||
# import traceback
|
||||
# from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
import os, io
|
||||
# load_dotenv()
|
||||
# import os, io
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
import pytest
|
||||
import litellm
|
||||
from litellm import embedding, completion, completion_cost, Timeout
|
||||
from litellm import RateLimitError
|
||||
# sys.path.insert(
|
||||
# 0, os.path.abspath("../..")
|
||||
# ) # Adds the parent directory to the system path
|
||||
# import pytest
|
||||
# import litellm
|
||||
# from litellm import embedding, completion, completion_cost, Timeout
|
||||
# from litellm import RateLimitError
|
||||
|
||||
# litellm.num_retries = 3
|
||||
litellm.cache = None
|
||||
litellm.success_callback = []
|
||||
user_message = "Write a short poem about the sky"
|
||||
messages = [{"content": user_message, "role": "user"}]
|
||||
# # litellm.num_retries = 3
|
||||
# litellm.cache = None
|
||||
# litellm.success_callback = []
|
||||
# user_message = "Write a short poem about the sky"
|
||||
# messages = [{"content": user_message, "role": "user"}]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_callbacks():
|
||||
print("\npytest fixture - resetting callbacks")
|
||||
litellm.success_callback = []
|
||||
litellm._async_success_callback = []
|
||||
litellm.failure_callback = []
|
||||
litellm.callbacks = []
|
||||
# @pytest.fixture(autouse=True)
|
||||
# def reset_callbacks():
|
||||
# print("\npytest fixture - resetting callbacks")
|
||||
# litellm.success_callback = []
|
||||
# litellm._async_success_callback = []
|
||||
# litellm.failure_callback = []
|
||||
# litellm.callbacks = []
|
||||
|
||||
|
||||
def test_completion_bedrock_claude_completion_auth():
|
||||
print("calling bedrock claude completion params auth")
|
||||
import os
|
||||
# def test_completion_bedrock_claude_completion_auth():
|
||||
# print("calling bedrock claude completion params auth")
|
||||
# import os
|
||||
|
||||
aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
|
||||
aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
|
||||
aws_region_name = os.environ["AWS_REGION_NAME"]
|
||||
# aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
|
||||
# aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
|
||||
# aws_region_name = os.environ["AWS_REGION_NAME"]
|
||||
|
||||
os.environ.pop("AWS_ACCESS_KEY_ID", None)
|
||||
os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
|
||||
os.environ.pop("AWS_REGION_NAME", None)
|
||||
# os.environ.pop("AWS_ACCESS_KEY_ID", None)
|
||||
# os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
|
||||
# os.environ.pop("AWS_REGION_NAME", None)
|
||||
|
||||
try:
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.1,
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_region_name=aws_region_name,
|
||||
)
|
||||
# Add any assertions here to check the response
|
||||
print(response)
|
||||
# try:
|
||||
# response = completion(
|
||||
# model="bedrock/anthropic.claude-instant-v1",
|
||||
# messages=messages,
|
||||
# max_tokens=10,
|
||||
# temperature=0.1,
|
||||
# aws_access_key_id=aws_access_key_id,
|
||||
# aws_secret_access_key=aws_secret_access_key,
|
||||
# aws_region_name=aws_region_name,
|
||||
# )
|
||||
# # Add any assertions here to check the response
|
||||
# print(response)
|
||||
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id
|
||||
os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key
|
||||
os.environ["AWS_REGION_NAME"] = aws_region_name
|
||||
except RateLimitError:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
# os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id
|
||||
# os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key
|
||||
# os.environ["AWS_REGION_NAME"] = aws_region_name
|
||||
# except RateLimitError:
|
||||
# pass
|
||||
# except Exception as e:
|
||||
# pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
# test_completion_bedrock_claude_completion_auth()
|
||||
# # test_completion_bedrock_claude_completion_auth()
|
||||
|
||||
|
||||
def test_completion_bedrock_claude_2_1_completion_auth():
|
||||
print("calling bedrock claude 2.1 completion params auth")
|
||||
import os
|
||||
# def test_completion_bedrock_claude_2_1_completion_auth():
|
||||
# print("calling bedrock claude 2.1 completion params auth")
|
||||
# import os
|
||||
|
||||
aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
|
||||
aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
|
||||
aws_region_name = os.environ["AWS_REGION_NAME"]
|
||||
# aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
|
||||
# aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
|
||||
# aws_region_name = os.environ["AWS_REGION_NAME"]
|
||||
|
||||
os.environ.pop("AWS_ACCESS_KEY_ID", None)
|
||||
os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
|
||||
os.environ.pop("AWS_REGION_NAME", None)
|
||||
try:
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-v2:1",
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.1,
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_region_name=aws_region_name,
|
||||
)
|
||||
# Add any assertions here to check the response
|
||||
print(response)
|
||||
# os.environ.pop("AWS_ACCESS_KEY_ID", None)
|
||||
# os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
|
||||
# os.environ.pop("AWS_REGION_NAME", None)
|
||||
# try:
|
||||
# response = completion(
|
||||
# model="bedrock/anthropic.claude-v2:1",
|
||||
# messages=messages,
|
||||
# max_tokens=10,
|
||||
# temperature=0.1,
|
||||
# aws_access_key_id=aws_access_key_id,
|
||||
# aws_secret_access_key=aws_secret_access_key,
|
||||
# aws_region_name=aws_region_name,
|
||||
# )
|
||||
# # Add any assertions here to check the response
|
||||
# print(response)
|
||||
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id
|
||||
os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key
|
||||
os.environ["AWS_REGION_NAME"] = aws_region_name
|
||||
except RateLimitError:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
# os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id
|
||||
# os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key
|
||||
# os.environ["AWS_REGION_NAME"] = aws_region_name
|
||||
# except RateLimitError:
|
||||
# pass
|
||||
# except Exception as e:
|
||||
# pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
# test_completion_bedrock_claude_2_1_completion_auth()
|
||||
# # test_completion_bedrock_claude_2_1_completion_auth()
|
||||
|
||||
|
||||
def test_completion_bedrock_claude_external_client_auth():
|
||||
print("\ncalling bedrock claude external client auth")
|
||||
import os
|
||||
# def test_completion_bedrock_claude_external_client_auth():
|
||||
# print("\ncalling bedrock claude external client auth")
|
||||
# import os
|
||||
|
||||
aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
|
||||
aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
|
||||
aws_region_name = os.environ["AWS_REGION_NAME"]
|
||||
# aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
|
||||
# aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
|
||||
# aws_region_name = os.environ["AWS_REGION_NAME"]
|
||||
|
||||
os.environ.pop("AWS_ACCESS_KEY_ID", None)
|
||||
os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
|
||||
os.environ.pop("AWS_REGION_NAME", None)
|
||||
# os.environ.pop("AWS_ACCESS_KEY_ID", None)
|
||||
# os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
|
||||
# os.environ.pop("AWS_REGION_NAME", None)
|
||||
|
||||
try:
|
||||
import boto3
|
||||
# try:
|
||||
# import boto3
|
||||
|
||||
litellm.set_verbose = True
|
||||
# litellm.set_verbose = True
|
||||
|
||||
bedrock = boto3.client(
|
||||
service_name="bedrock-runtime",
|
||||
region_name=aws_region_name,
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
endpoint_url=f"https://bedrock-runtime.{aws_region_name}.amazonaws.com",
|
||||
)
|
||||
# bedrock = boto3.client(
|
||||
# service_name="bedrock-runtime",
|
||||
# region_name=aws_region_name,
|
||||
# aws_access_key_id=aws_access_key_id,
|
||||
# aws_secret_access_key=aws_secret_access_key,
|
||||
# endpoint_url=f"https://bedrock-runtime.{aws_region_name}.amazonaws.com",
|
||||
# )
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.1,
|
||||
aws_bedrock_client=bedrock,
|
||||
)
|
||||
# Add any assertions here to check the response
|
||||
print(response)
|
||||
# response = completion(
|
||||
# model="bedrock/anthropic.claude-instant-v1",
|
||||
# messages=messages,
|
||||
# max_tokens=10,
|
||||
# temperature=0.1,
|
||||
# aws_bedrock_client=bedrock,
|
||||
# )
|
||||
# # Add any assertions here to check the response
|
||||
# print(response)
|
||||
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id
|
||||
os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key
|
||||
os.environ["AWS_REGION_NAME"] = aws_region_name
|
||||
except RateLimitError:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
# os.environ["AWS_ACCESS_KEY_ID"] = aws_access_key_id
|
||||
# os.environ["AWS_SECRET_ACCESS_KEY"] = aws_secret_access_key
|
||||
# os.environ["AWS_REGION_NAME"] = aws_region_name
|
||||
# except RateLimitError:
|
||||
# pass
|
||||
# except Exception as e:
|
||||
# pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
# test_completion_bedrock_claude_external_client_auth()
|
||||
# # test_completion_bedrock_claude_external_client_auth()
|
||||
|
||||
|
||||
def test_completion_bedrock_claude_sts_client_auth():
|
||||
print("\ncalling bedrock claude external client auth")
|
||||
import os
|
||||
# @pytest.mark.skip(reason="Expired token, need to renew")
|
||||
# def test_completion_bedrock_claude_sts_client_auth():
|
||||
# print("\ncalling bedrock claude external client auth")
|
||||
# import os
|
||||
|
||||
aws_access_key_id = os.environ["AWS_TEMP_ACCESS_KEY_ID"]
|
||||
aws_secret_access_key = os.environ["AWS_TEMP_SECRET_ACCESS_KEY"]
|
||||
aws_region_name = os.environ["AWS_REGION_NAME"]
|
||||
aws_role_name = os.environ["AWS_TEMP_ROLE_NAME"]
|
||||
# aws_access_key_id = os.environ["AWS_TEMP_ACCESS_KEY_ID"]
|
||||
# aws_secret_access_key = os.environ["AWS_TEMP_SECRET_ACCESS_KEY"]
|
||||
# aws_region_name = os.environ["AWS_REGION_NAME"]
|
||||
# aws_role_name = os.environ["AWS_TEMP_ROLE_NAME"]
|
||||
|
||||
try:
|
||||
import boto3
|
||||
# try:
|
||||
# import boto3
|
||||
|
||||
litellm.set_verbose = True
|
||||
# litellm.set_verbose = True
|
||||
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.1,
|
||||
aws_region_name=aws_region_name,
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_role_name=aws_role_name,
|
||||
aws_session_name="my-test-session",
|
||||
)
|
||||
# response = completion(
|
||||
# model="bedrock/anthropic.claude-instant-v1",
|
||||
# messages=messages,
|
||||
# max_tokens=10,
|
||||
# temperature=0.1,
|
||||
# aws_region_name=aws_region_name,
|
||||
# aws_access_key_id=aws_access_key_id,
|
||||
# aws_secret_access_key=aws_secret_access_key,
|
||||
# aws_role_name=aws_role_name,
|
||||
# aws_session_name="my-test-session",
|
||||
# )
|
||||
|
||||
response = embedding(
|
||||
model="cohere.embed-multilingual-v3",
|
||||
input=["hello world"],
|
||||
aws_region_name="us-east-1",
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_role_name=aws_role_name,
|
||||
aws_session_name="my-test-session",
|
||||
)
|
||||
# response = embedding(
|
||||
# model="cohere.embed-multilingual-v3",
|
||||
# input=["hello world"],
|
||||
# aws_region_name="us-east-1",
|
||||
# aws_access_key_id=aws_access_key_id,
|
||||
# aws_secret_access_key=aws_secret_access_key,
|
||||
# aws_role_name=aws_role_name,
|
||||
# aws_session_name="my-test-session",
|
||||
# )
|
||||
|
||||
response = completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=messages,
|
||||
aws_region_name="us-east-1",
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_role_name=aws_role_name,
|
||||
aws_session_name="my-test-session",
|
||||
)
|
||||
# Add any assertions here to check the response
|
||||
print(response)
|
||||
except RateLimitError:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
# response = completion(
|
||||
# model="gpt-3.5-turbo",
|
||||
# messages=messages,
|
||||
# aws_region_name="us-east-1",
|
||||
# aws_access_key_id=aws_access_key_id,
|
||||
# aws_secret_access_key=aws_secret_access_key,
|
||||
# aws_role_name=aws_role_name,
|
||||
# aws_session_name="my-test-session",
|
||||
# )
|
||||
# # Add any assertions here to check the response
|
||||
# print(response)
|
||||
# except RateLimitError:
|
||||
# pass
|
||||
# except Exception as e:
|
||||
# pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
test_completion_bedrock_claude_sts_client_auth()
|
||||
# # test_completion_bedrock_claude_sts_client_auth()
|
||||
|
||||
|
||||
def test_provisioned_throughput():
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
import botocore, json, io
|
||||
import botocore.session
|
||||
from botocore.stub import Stubber
|
||||
# def test_provisioned_throughput():
|
||||
# try:
|
||||
# litellm.set_verbose = True
|
||||
# import botocore, json, io
|
||||
# import botocore.session
|
||||
# from botocore.stub import Stubber
|
||||
|
||||
bedrock_client = botocore.session.get_session().create_client(
|
||||
"bedrock-runtime", region_name="us-east-1"
|
||||
)
|
||||
# bedrock_client = botocore.session.get_session().create_client(
|
||||
# "bedrock-runtime", region_name="us-east-1"
|
||||
# )
|
||||
|
||||
expected_params = {
|
||||
"accept": "application/json",
|
||||
"body": '{"prompt": "\\n\\nHuman: Hello, how are you?\\n\\nAssistant: ", '
|
||||
'"max_tokens_to_sample": 256}',
|
||||
"contentType": "application/json",
|
||||
"modelId": "provisioned-model-arn",
|
||||
}
|
||||
response_from_bedrock = {
|
||||
"body": io.StringIO(
|
||||
json.dumps(
|
||||
{
|
||||
"completion": " Here is a short poem about the sky:",
|
||||
"stop_reason": "max_tokens",
|
||||
"stop": None,
|
||||
}
|
||||
)
|
||||
),
|
||||
"contentType": "contentType",
|
||||
"ResponseMetadata": {"HTTPStatusCode": 200},
|
||||
}
|
||||
# expected_params = {
|
||||
# "accept": "application/json",
|
||||
# "body": '{"prompt": "\\n\\nHuman: Hello, how are you?\\n\\nAssistant: ", '
|
||||
# '"max_tokens_to_sample": 256}',
|
||||
# "contentType": "application/json",
|
||||
# "modelId": "provisioned-model-arn",
|
||||
# }
|
||||
# response_from_bedrock = {
|
||||
# "body": io.StringIO(
|
||||
# json.dumps(
|
||||
# {
|
||||
# "completion": " Here is a short poem about the sky:",
|
||||
# "stop_reason": "max_tokens",
|
||||
# "stop": None,
|
||||
# }
|
||||
# )
|
||||
# ),
|
||||
# "contentType": "contentType",
|
||||
# "ResponseMetadata": {"HTTPStatusCode": 200},
|
||||
# }
|
||||
|
||||
with Stubber(bedrock_client) as stubber:
|
||||
stubber.add_response(
|
||||
"invoke_model",
|
||||
service_response=response_from_bedrock,
|
||||
expected_params=expected_params,
|
||||
)
|
||||
response = litellm.completion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
model_id="provisioned-model-arn",
|
||||
messages=[{"content": "Hello, how are you?", "role": "user"}],
|
||||
aws_bedrock_client=bedrock_client,
|
||||
)
|
||||
print("response stubbed", response)
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
# with Stubber(bedrock_client) as stubber:
|
||||
# stubber.add_response(
|
||||
# "invoke_model",
|
||||
# service_response=response_from_bedrock,
|
||||
# expected_params=expected_params,
|
||||
# )
|
||||
# response = litellm.completion(
|
||||
# model="bedrock/anthropic.claude-instant-v1",
|
||||
# model_id="provisioned-model-arn",
|
||||
# messages=[{"content": "Hello, how are you?", "role": "user"}],
|
||||
# aws_bedrock_client=bedrock_client,
|
||||
# )
|
||||
# print("response stubbed", response)
|
||||
# except Exception as e:
|
||||
# pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
# test_provisioned_throughput()
|
||||
# # test_provisioned_throughput()
|
||||
|
|
|
|||
|
|
@ -546,6 +546,7 @@ def test_redis_cache_acompletion_stream():
|
|||
# test_redis_cache_acompletion_stream()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_redis_cache_acompletion_stream_bedrock():
|
||||
import asyncio
|
||||
|
||||
|
|
|
|||
|
|
@ -1404,6 +1404,7 @@ def test_customprompt_together_ai():
|
|||
# test_customprompt_together_ai()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_sagemaker():
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
|
|
@ -1429,6 +1430,7 @@ def test_completion_sagemaker():
|
|||
# test_completion_sagemaker()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_sagemaker_stream():
|
||||
try:
|
||||
litellm.set_verbose = False
|
||||
|
|
@ -1459,6 +1461,7 @@ def test_completion_sagemaker_stream():
|
|||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_chat_sagemaker():
|
||||
try:
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
|
|
@ -1483,6 +1486,7 @@ def test_completion_chat_sagemaker():
|
|||
# test_completion_chat_sagemaker()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_chat_sagemaker_mistral():
|
||||
try:
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
|
|
@ -1501,6 +1505,7 @@ def test_completion_chat_sagemaker_mistral():
|
|||
# test_completion_chat_sagemaker_mistral()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_bedrock_titan_null_response():
|
||||
try:
|
||||
response = completion(
|
||||
|
|
@ -1526,6 +1531,7 @@ def test_completion_bedrock_titan_null_response():
|
|||
pytest.fail(f"An error occurred - {str(e)}")
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_bedrock_titan():
|
||||
try:
|
||||
response = completion(
|
||||
|
|
@ -1547,6 +1553,7 @@ def test_completion_bedrock_titan():
|
|||
# test_completion_bedrock_titan()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_bedrock_claude():
|
||||
print("calling claude")
|
||||
try:
|
||||
|
|
@ -1568,6 +1575,7 @@ def test_completion_bedrock_claude():
|
|||
# test_completion_bedrock_claude()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_bedrock_cohere():
|
||||
print("calling bedrock cohere")
|
||||
litellm.set_verbose = True
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ def test_cost_openai_image_gen():
|
|||
assert cost == 0.019922944
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_cost_bedrock_pricing():
|
||||
"""
|
||||
- get pricing specific to region for a model
|
||||
|
|
|
|||
|
|
@ -478,6 +478,7 @@ async def test_async_chat_azure_stream():
|
|||
|
||||
|
||||
## Test Bedrock + sync
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_chat_bedrock_stream():
|
||||
try:
|
||||
customHandler = CompletionCustomHandler()
|
||||
|
|
@ -518,6 +519,7 @@ def test_chat_bedrock_stream():
|
|||
|
||||
|
||||
## Test Bedrock + Async
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_chat_bedrock_stream():
|
||||
try:
|
||||
|
|
@ -561,6 +563,7 @@ async def test_async_chat_bedrock_stream():
|
|||
|
||||
|
||||
## Test Sagemaker + Async
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_chat_sagemaker_stream():
|
||||
try:
|
||||
|
|
@ -793,6 +796,7 @@ async def test_async_embedding_azure():
|
|||
|
||||
|
||||
## Test Bedrock + Async
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_embedding_bedrock():
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -388,6 +388,7 @@ async def test_async_custom_handler_embedding_optional_param():
|
|||
# asyncio.run(test_async_custom_handler_embedding_optional_param())
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Account suspended. Pending their approval")
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_custom_handler_embedding_optional_param_bedrock():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -256,6 +256,7 @@ async def test_vertexai_aembedding():
|
|||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_bedrock_embedding_titan():
|
||||
try:
|
||||
# this tests if we support str input for bedrock embedding
|
||||
|
|
@ -301,6 +302,7 @@ def test_bedrock_embedding_titan():
|
|||
# test_bedrock_embedding_titan()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_bedrock_embedding_cohere():
|
||||
try:
|
||||
litellm.set_verbose = False
|
||||
|
|
@ -422,6 +424,7 @@ def test_aembedding_azure():
|
|||
# test_aembedding_azure()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_sagemaker_embeddings():
|
||||
try:
|
||||
response = litellm.embedding(
|
||||
|
|
@ -438,6 +441,7 @@ def test_sagemaker_embeddings():
|
|||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_aembeddings():
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ exception_models = [
|
|||
|
||||
|
||||
# Test 1: Context Window Errors
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.parametrize("model", exception_models)
|
||||
def test_context_window(model):
|
||||
print("Testing context window error")
|
||||
|
|
@ -120,9 +121,9 @@ def invalid_auth(model): # set the model key to an invalid key, depending on th
|
|||
os.environ["AI21_API_KEY"] = "bad-key"
|
||||
elif "togethercomputer" in model:
|
||||
temporary_key = os.environ["TOGETHERAI_API_KEY"]
|
||||
os.environ[
|
||||
"TOGETHERAI_API_KEY"
|
||||
] = "84060c79880fc49df126d3e87b53f8a463ff6e1c6d27fe64207cde25cdfcd1f24a"
|
||||
os.environ["TOGETHERAI_API_KEY"] = (
|
||||
"84060c79880fc49df126d3e87b53f8a463ff6e1c6d27fe64207cde25cdfcd1f24a"
|
||||
)
|
||||
elif model in litellm.openrouter_models:
|
||||
temporary_key = os.environ["OPENROUTER_API_KEY"]
|
||||
os.environ["OPENROUTER_API_KEY"] = "bad-key"
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ async def test_azure_img_gen_health_check():
|
|||
# asyncio.run(test_azure_img_gen_health_check())
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_embedding_health_check():
|
||||
response = await litellm.ahealth_check(
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ async def test_async_image_generation_azure():
|
|||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_image_generation_bedrock():
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
|
|
@ -141,6 +142,7 @@ def test_image_generation_bedrock():
|
|||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_aimage_generation_bedrock_with_optional_params():
|
||||
try:
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -80,6 +80,14 @@ request_data = {
|
|||
|
||||
@pytest.fixture
|
||||
def prisma_client():
|
||||
from litellm.proxy.proxy_cli import append_query_params
|
||||
|
||||
### add connection pool + pool timeout args
|
||||
params = {"connection_limit": 100, "pool_timeout": 60}
|
||||
database_url = os.getenv("DATABASE_URL")
|
||||
modified_url = append_query_params(database_url, params)
|
||||
os.environ["DATABASE_URL"] = modified_url
|
||||
|
||||
# Assuming DBClient is a class that needs to be instantiated
|
||||
prisma_client = PrismaClient(
|
||||
database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj
|
||||
|
|
@ -1633,3 +1641,99 @@ async def test_key_with_no_permissions(prisma_client):
|
|||
except Exception as e:
|
||||
print("Got Exception", e)
|
||||
print(e.message)
|
||||
|
||||
|
||||
async def track_cost_callback_helper_fn(generated_key: str, user_id: str):
|
||||
from litellm import ModelResponse, Choices, Message, Usage
|
||||
from litellm.proxy.proxy_server import (
|
||||
_PROXY_track_cost_callback as track_cost_callback,
|
||||
)
|
||||
|
||||
import uuid
|
||||
|
||||
request_id = f"chatcmpl-e41836bb-bb8b-4df2-8e70-8f3e160155ac{uuid.uuid4()}"
|
||||
resp = ModelResponse(
|
||||
id=request_id,
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason=None,
|
||||
index=0,
|
||||
message=Message(
|
||||
content=" Sure! Here is a short poem about the sky:\n\nA canvas of blue, a",
|
||||
role="assistant",
|
||||
),
|
||||
)
|
||||
],
|
||||
model="gpt-35-turbo", # azure always has model written like this
|
||||
usage=Usage(prompt_tokens=210, completion_tokens=200, total_tokens=410),
|
||||
)
|
||||
await track_cost_callback(
|
||||
kwargs={
|
||||
"call_type": "acompletion",
|
||||
"model": "sagemaker-chatgpt-v-2",
|
||||
"stream": True,
|
||||
"complete_streaming_response": resp,
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"user_api_key": hash_token(generated_key),
|
||||
"user_api_key_user_id": user_id,
|
||||
}
|
||||
},
|
||||
"response_cost": 0.00005,
|
||||
},
|
||||
completion_response=resp,
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
# @pytest.mark.skip(reason="High traffic load test for spend tracking")
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_load_test_db(prisma_client):
|
||||
"""
|
||||
Run 1500 req./s against track_cost_callback function
|
||||
"""
|
||||
setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
|
||||
setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
import logging, time
|
||||
|
||||
litellm.set_verbose = True
|
||||
verbose_proxy_logger.setLevel(logging.DEBUG)
|
||||
try:
|
||||
start_time = time.time()
|
||||
await litellm.proxy.proxy_server.prisma_client.connect()
|
||||
request = GenerateKeyRequest(max_budget=0.00001)
|
||||
key = await generate_key_fn(request)
|
||||
print(key)
|
||||
|
||||
generated_key = key.key
|
||||
user_id = key.user_id
|
||||
bearer_token = "Bearer " + generated_key
|
||||
|
||||
request = Request(scope={"type": "http"})
|
||||
request._url = URL(url="/chat/completions")
|
||||
|
||||
# use generated key to auth in
|
||||
result = await user_api_key_auth(request=request, api_key=bearer_token)
|
||||
print("result from user auth with new key", result)
|
||||
# update spend using track_cost callback, make 2nd request, it should fail
|
||||
n = 5000
|
||||
tasks = [
|
||||
track_cost_callback_helper_fn(generated_key=generated_key, user_id=user_id)
|
||||
for _ in range(n)
|
||||
]
|
||||
completions = await asyncio.gather(*tasks)
|
||||
await asyncio.sleep(120)
|
||||
try:
|
||||
# call spend logs
|
||||
spend_logs = await view_spend_logs(api_key=generated_key)
|
||||
|
||||
print(f"len responses: {len(spend_logs)}")
|
||||
assert len(spend_logs) == n
|
||||
print(n, time.time() - start_time, len(spend_logs))
|
||||
except:
|
||||
print(n, time.time() - start_time, 0)
|
||||
raise Exception(f"it worked! key={key.key}")
|
||||
except Exception as e:
|
||||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import litellm
|
|||
from litellm import completion
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_sagemaker():
|
||||
litellm.set_verbose = True
|
||||
litellm.drop_params = True
|
||||
|
|
|
|||
|
|
@ -473,6 +473,7 @@ def aleph_alpha_test_completion():
|
|||
# Sagemaker
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def sagemaker_test_completion():
|
||||
litellm.SagemakerConfig(max_new_tokens=10)
|
||||
# litellm.set_verbose=True
|
||||
|
|
@ -514,6 +515,7 @@ def sagemaker_test_completion():
|
|||
# Bedrock
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def bedrock_test_completion():
|
||||
litellm.AmazonCohereConfig(max_tokens=10)
|
||||
# litellm.set_verbose=True
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ def test_embedding(client_no_auth):
|
|||
pytest.fail(f"LiteLLM Proxy test failed. Exception - {str(e)}")
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_bedrock_embedding(client_no_auth):
|
||||
global headers
|
||||
from litellm.proxy.proxy_server import user_custom_auth
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ def generate_random_word(length=4):
|
|||
return "".join(random.choice(letters) for _ in range(length))
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_chat_completion(client_no_auth):
|
||||
global headers
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -166,14 +166,6 @@ def test_call_one_endpoint():
|
|||
"tpm": 240000,
|
||||
"rpm": 1800,
|
||||
},
|
||||
{
|
||||
"model_name": "claude-v1",
|
||||
"litellm_params": {
|
||||
"model": "bedrock/anthropic.claude-instant-v1",
|
||||
},
|
||||
"tpm": 100000,
|
||||
"rpm": 10000,
|
||||
},
|
||||
{
|
||||
"model_name": "text-embedding-ada-002",
|
||||
"litellm_params": {
|
||||
|
|
@ -202,15 +194,6 @@ def test_call_one_endpoint():
|
|||
)
|
||||
print("\n response", response)
|
||||
|
||||
async def call_bedrock_claude():
|
||||
response = await router.acompletion(
|
||||
model="bedrock/anthropic.claude-instant-v1",
|
||||
messages=[{"role": "user", "content": "hello this request will pass"}],
|
||||
specific_deployment=True,
|
||||
)
|
||||
|
||||
print("\n response", response)
|
||||
|
||||
async def call_azure_embedding():
|
||||
response = await router.aembedding(
|
||||
model="azure/azure-embedding-model",
|
||||
|
|
@ -221,7 +204,6 @@ def test_call_one_endpoint():
|
|||
print("\n response", response)
|
||||
|
||||
asyncio.run(call_azure_completion())
|
||||
asyncio.run(call_bedrock_claude())
|
||||
asyncio.run(call_azure_embedding())
|
||||
|
||||
os.environ["AZURE_API_BASE"] = old_api_base
|
||||
|
|
@ -593,6 +575,7 @@ def test_azure_embedding_on_router():
|
|||
# test_azure_embedding_on_router()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_bedrock_on_router():
|
||||
litellm.set_verbose = True
|
||||
print("\n Testing bedrock on router\n")
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ def test_router_timeouts():
|
|||
print("********** TOKENS USED SO FAR = ", total_tokens_used)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_timeouts_bedrock():
|
||||
import openai
|
||||
|
|
|
|||
|
|
@ -764,6 +764,7 @@ def test_completion_replicate_stream_bad_key():
|
|||
# test_completion_replicate_stream_bad_key()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_bedrock_claude_stream():
|
||||
try:
|
||||
litellm.set_verbose = False
|
||||
|
|
@ -810,6 +811,7 @@ def test_completion_bedrock_claude_stream():
|
|||
# test_completion_bedrock_claude_stream()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_bedrock_ai21_stream():
|
||||
try:
|
||||
litellm.set_verbose = False
|
||||
|
|
@ -911,6 +913,7 @@ def test_sagemaker_weird_response():
|
|||
# test_sagemaker_weird_response()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_sagemaker_streaming_async():
|
||||
try:
|
||||
|
|
@ -949,6 +952,7 @@ async def test_sagemaker_streaming_async():
|
|||
# asyncio.run(test_sagemaker_streaming_async())
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
def test_completion_sagemaker_stream():
|
||||
try:
|
||||
response = completion(
|
||||
|
|
|
|||
|
|
@ -6963,7 +6963,7 @@ def exception_type(
|
|||
if "500 An internal error has occurred." in error_str:
|
||||
exception_mapping_worked = True
|
||||
raise APIError(
|
||||
status_code=original_exception.status_code,
|
||||
status_code=getattr(original_exception, "status_code", 500),
|
||||
message=f"PalmException - {original_exception.message}",
|
||||
llm_provider="palm",
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -490,6 +490,7 @@ async def test_key_crossing_budget():
|
|||
assert "ExceededTokenBudget: Current spend for token:" in str(e)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="AWS Suspended Account")
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_info_spend_values_sagemaker():
|
||||
"""
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-a697f24d60c9c262.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a85b2c176012d8e5.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e1b183dda365ec86.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>🚅 LiteLLM</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-a697f24d60c9c262.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/6920a121699cde9c.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[24143,[\"303\",\"static/chunks/303-d80f23087a9e6aec.js\",\"931\",\"static/chunks/app/page-d4fe4a48cbd3572c.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/6920a121699cde9c.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"eSwVwl_InIrhYtCAqDMKF\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"🚅 LiteLLM\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-a697f24d60c9c262.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a85b2c176012d8e5.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e1b183dda365ec86.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>🚅 LiteLLM</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-a697f24d60c9c262.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/6920a121699cde9c.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[24143,[\"303\",\"static/chunks/303-d80f23087a9e6aec.js\",\"931\",\"static/chunks/app/page-cc9d300e3b13fc1b.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/6920a121699cde9c.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"h6IXdBMiZG7ES547qg1M-\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"🚅 LiteLLM\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[77831,[],""]
|
||||
3:I[24143,["303","static/chunks/303-d80f23087a9e6aec.js","931","static/chunks/app/page-d4fe4a48cbd3572c.js"],""]
|
||||
3:I[24143,["303","static/chunks/303-d80f23087a9e6aec.js","931","static/chunks/app/page-cc9d300e3b13fc1b.js"],""]
|
||||
4:I[5613,[],""]
|
||||
5:I[31778,[],""]
|
||||
0:["eSwVwl_InIrhYtCAqDMKF",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/6920a121699cde9c.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
0:["h6IXdBMiZG7ES547qg1M-",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/6920a121699cde9c.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"🚅 LiteLLM"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -313,6 +313,7 @@ export const userSpendLogsCall = async (
|
|||
endTime: String
|
||||
) => {
|
||||
try {
|
||||
console.log(`user role in spend logs call: ${userRole}`);
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/spend/logs` : `/spend/logs`;
|
||||
if (userRole == "App Owner") {
|
||||
url = `${url}/?user_id=${userID}&start_date=${startTime}&end_date=${endTime}`;
|
||||
|
|
@ -343,6 +344,96 @@ export const userSpendLogsCall = async (
|
|||
}
|
||||
};
|
||||
|
||||
export const adminSpendLogsCall = async (accessToken: String) => {
|
||||
try {
|
||||
let url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/global/spend/logs`
|
||||
: `/global/spend/logs`;
|
||||
|
||||
message.info("Making spend logs request");
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
console.log(data);
|
||||
message.success("Spend Logs received");
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to create key:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const adminTopKeysCall = async (accessToken: String) => {
|
||||
try {
|
||||
let url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/global/spend/keys?limit=5`
|
||||
: `/global/spend/keys?limit=5`;
|
||||
|
||||
message.info("Making spend keys request");
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
console.log(data);
|
||||
message.success("Spend Logs received");
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to create key:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const adminTopModelsCall = async (accessToken: String) => {
|
||||
try {
|
||||
let url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/global/spend/models?limit=5`
|
||||
: `/global/spend/models?limit=5`;
|
||||
|
||||
message.info("Making spend models request");
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
console.log(data);
|
||||
message.success("Spend Logs received");
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to create key:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const keyInfoCall = async (accessToken: String, keys: String[]) => {
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/v2/key/info` : `/v2/key/info`;
|
||||
|
|
|
|||
|
|
@ -2,7 +2,13 @@ import { BarChart, Card, Title } from "@tremor/react";
|
|||
|
||||
import React, { useState, useEffect } from "react";
|
||||
import { Grid, Col, Text, LineChart } from "@tremor/react";
|
||||
import { userSpendLogsCall, keyInfoCall } from "./networking";
|
||||
import {
|
||||
userSpendLogsCall,
|
||||
keyInfoCall,
|
||||
adminSpendLogsCall,
|
||||
adminTopKeysCall,
|
||||
adminTopModelsCall,
|
||||
} from "./networking";
|
||||
import { start } from "repl";
|
||||
|
||||
interface UsagePageProps {
|
||||
|
|
@ -164,41 +170,61 @@ const UsagePage: React.FC<UsagePageProps> = ({
|
|||
if (accessToken && token && userRole && userID) {
|
||||
const fetchData = async () => {
|
||||
try {
|
||||
await userSpendLogsCall(
|
||||
accessToken,
|
||||
token,
|
||||
userRole,
|
||||
userID,
|
||||
startTime,
|
||||
endTime
|
||||
).then(async (response) => {
|
||||
console.log("result from spend logs call", response);
|
||||
if ("daily_spend" in response) {
|
||||
// this is from clickhouse analytics
|
||||
//
|
||||
let daily_spend = response["daily_spend"];
|
||||
console.log("daily spend", daily_spend);
|
||||
setKeySpendData(daily_spend);
|
||||
let topApiKeys = response.top_api_keys;
|
||||
setTopKeys(topApiKeys);
|
||||
}
|
||||
else {
|
||||
const topKeysResponse = await keyInfoCall(
|
||||
accessToken,
|
||||
getTopKeys(response)
|
||||
);
|
||||
const filtered_keys = topKeysResponse["info"].map((k: any) => ({
|
||||
key: (k["key_name"] || k["key_alias"] || k["token"]).substring(
|
||||
0,
|
||||
7
|
||||
),
|
||||
spend: k["spend"],
|
||||
}));
|
||||
setTopKeys(filtered_keys);
|
||||
setTopUsers(getTopUsers(response));
|
||||
setKeySpendData(response);
|
||||
}
|
||||
});
|
||||
/**
|
||||
* If user is Admin - query the global views endpoints
|
||||
* If user is App Owner - use the normal spend logs call
|
||||
*/
|
||||
console.log(`user role: ${userRole}`);
|
||||
if (userRole == "Admin") {
|
||||
const overall_spend = await adminSpendLogsCall(accessToken);
|
||||
setKeySpendData(overall_spend);
|
||||
const top_keys = await adminTopKeysCall(accessToken);
|
||||
const filtered_keys = top_keys.map((k: any) => ({
|
||||
key: (k["key_name"] || k["key_alias"] || k["api_key"]).substring(
|
||||
0,
|
||||
7
|
||||
),
|
||||
spend: k["total_spend"],
|
||||
}));
|
||||
setTopKeys(filtered_keys);
|
||||
const top_models = await adminTopModelsCall(accessToken);
|
||||
} else if (userRole == "App Owner") {
|
||||
await userSpendLogsCall(
|
||||
accessToken,
|
||||
token,
|
||||
userRole,
|
||||
userID,
|
||||
startTime,
|
||||
endTime
|
||||
).then(async (response) => {
|
||||
console.log("result from spend logs call", response);
|
||||
if ("daily_spend" in response) {
|
||||
// this is from clickhouse analytics
|
||||
//
|
||||
let daily_spend = response["daily_spend"];
|
||||
console.log("daily spend", daily_spend);
|
||||
setKeySpendData(daily_spend);
|
||||
let topApiKeys = response.top_api_keys;
|
||||
setTopKeys(topApiKeys);
|
||||
} else {
|
||||
const topKeysResponse = await keyInfoCall(
|
||||
accessToken,
|
||||
getTopKeys(response)
|
||||
);
|
||||
const filtered_keys = topKeysResponse["info"].map((k: any) => ({
|
||||
key: (
|
||||
k["key_name"] ||
|
||||
k["key_alias"] ||
|
||||
k["token"]
|
||||
).substring(0, 7),
|
||||
spend: k["spend"],
|
||||
}));
|
||||
setTopKeys(filtered_keys);
|
||||
setTopUsers(getTopUsers(response));
|
||||
setKeySpendData(response);
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("There was an error fetching the data", error);
|
||||
// Optionally, update your UI to reflect the error state here as well
|
||||
|
|
@ -222,7 +248,7 @@ const UsagePage: React.FC<UsagePageProps> = ({
|
|||
valueFormatter={valueFormatter}
|
||||
yAxisWidth={100}
|
||||
tickGap={5}
|
||||
customTooltip={customTooltip}
|
||||
// customTooltip={customTooltip}
|
||||
/>
|
||||
</Card>
|
||||
</Col>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue