From 138f5ceb1e7e6cde0340c27f9292eee71cadba1e Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 16 Feb 2024 10:16:48 -0800 Subject: [PATCH 01/14] (feat) view spend per tag --- litellm/proxy/enterprise/LICENSE.md | 37 +++++++++ litellm/proxy/enterprise/README.md | 12 +++ .../callbacks/example_logging_api.py | 31 ++++++++ litellm/proxy/enterprise/utils.py | 16 ++++ litellm/proxy/proxy_server.py | 75 +++++++++++++++++++ 5 files changed, 171 insertions(+) create mode 100644 litellm/proxy/enterprise/LICENSE.md create mode 100644 litellm/proxy/enterprise/README.md create mode 100644 litellm/proxy/enterprise/callbacks/example_logging_api.py create mode 100644 litellm/proxy/enterprise/utils.py diff --git a/litellm/proxy/enterprise/LICENSE.md b/litellm/proxy/enterprise/LICENSE.md new file mode 100644 index 00000000000..5cd298ce658 --- /dev/null +++ b/litellm/proxy/enterprise/LICENSE.md @@ -0,0 +1,37 @@ + +The BerriAI Enterprise license (the "Enterprise License") +Copyright (c) 2024 - present Berrie AI Inc. + +With regard to the BerriAI Software: + +This software and associated documentation files (the "Software") may only be +used in production, if you (and any entity that you represent) have agreed to, +and are in compliance with, the BerriAI Subscription Terms of Service, available +via [call](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) or email (info@berri.ai) (the "Enterprise Terms"), or other +agreement governing the use of the Software, as agreed by you and BerriAI, +and otherwise have a valid BerriAI Enterprise license for the +correct number of user seats. Subject to the foregoing sentence, you are free to +modify this Software and publish patches to the Software. You agree that BerriAI +and/or its licensors (as applicable) retain all right, title and interest in and +to all such modifications and/or patches, and all such modifications and/or +patches may only be used, copied, modified, displayed, distributed, or otherwise +exploited with a valid BerriAI Enterprise license for the correct +number of user seats. Notwithstanding the foregoing, you may copy and modify +the Software for development and testing purposes, without requiring a +subscription. You agree that BerriAI and/or its licensors (as applicable) retain +all right, title and interest in and to all such modifications. You are not +granted any other rights beyond what is expressly stated herein. Subject to the +foregoing, it is forbidden to copy, merge, publish, distribute, sublicense, +and/or sell the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +For all third party components incorporated into the BerriAI Software, those +components are licensed under the original license provided by the owner of the +applicable component. \ No newline at end of file diff --git a/litellm/proxy/enterprise/README.md b/litellm/proxy/enterprise/README.md new file mode 100644 index 00000000000..fd7e68fd2b2 --- /dev/null +++ b/litellm/proxy/enterprise/README.md @@ -0,0 +1,12 @@ +## LiteLLM Enterprise + +Code in this folder is licensed under a commercial license. Please review the [LICENSE](./LICENSE.md) file within the /enterprise folder + +**These features are covered under the LiteLLM Enterprise contract** + +👉 **Using in an Enterprise / Need specific features ?** Meet with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat?month=2024-02) + +## Features: +- Custom API / microservice callbacks +- Google Text Moderation API + diff --git a/litellm/proxy/enterprise/callbacks/example_logging_api.py b/litellm/proxy/enterprise/callbacks/example_logging_api.py new file mode 100644 index 00000000000..a8c5b54293e --- /dev/null +++ b/litellm/proxy/enterprise/callbacks/example_logging_api.py @@ -0,0 +1,31 @@ +# this is an example endpoint to receive data from litellm +from fastapi import FastAPI, HTTPException, Request + +app = FastAPI() + + +@app.post("/log-event") +async def log_event(request: Request): + try: + print("Received /log-event request") # noqa + # Assuming the incoming request has JSON data + data = await request.json() + print("Received request data:") # noqa + print(data) # noqa + + # Your additional logic can go here + # For now, just printing the received data + + return {"message": "Request received successfully"} + except Exception as e: + print(f"Error processing request: {str(e)}") # noqa + import traceback + + traceback.print_exc() + raise HTTPException(status_code=500, detail="Internal Server Error") + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/litellm/proxy/enterprise/utils.py b/litellm/proxy/enterprise/utils.py new file mode 100644 index 00000000000..61a621f8669 --- /dev/null +++ b/litellm/proxy/enterprise/utils.py @@ -0,0 +1,16 @@ +# Enterprise Proxy Util Endpoints + + +async def get_spend_by_tags(start_date=None, end_date=None, prisma_client=None): + response = await prisma_client.db.query_raw( + """ + SELECT + jsonb_array_elements_text(request_tags) AS individual_request_tag, + COUNT(*) AS log_count, + SUM(spend) AS total_spend + FROM "LiteLLM_SpendLogs" + GROUP BY individual_request_tag; + """ + ) + + return response diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 761b9db8ec5..c8dd2bbe9b4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3482,6 +3482,81 @@ async def spend_user_fn( ) +@router.get( + "/spend/tags", + tags=["budget & spend Tracking"], + dependencies=[Depends(user_api_key_auth)], + responses={ + 200: {"model": List[LiteLLM_SpendLogs]}, + }, +) +async def view_spend_tags( + start_date: Optional[str] = fastapi.Query( + default=None, + description="Time from which to start viewing key spend", + ), + end_date: Optional[str] = fastapi.Query( + default=None, + description="Time till which to view key spend", + ), +): + """ + LiteLLM Enterprise - View Spend Per Request Tag + + Example Request: + ``` + curl -X GET "http://0.0.0.0:8000/spend/tags" \ +-H "Authorization: Bearer sk-1234" + ``` + + Spend with Start Date and End Date + ``` + curl -X GET "http://0.0.0.0:8000/spend/tags?start_date=2022-01-01&end_date=2022-02-01" \ +-H "Authorization: Bearer sk-1234" + ``` + """ + + from litellm.proxy.enterprise.utils import get_spend_by_tags + + global prisma_client + try: + if prisma_client is None: + raise Exception( + f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" + ) + + # run the following SQL query on prisma + """ + SELECT + jsonb_array_elements_text(request_tags) AS individual_request_tag, + COUNT(*) AS log_count, + SUM(spend) AS total_spend + FROM "LiteLLM_SpendLogs" + GROUP BY individual_request_tag; + """ + response = await get_spend_by_tags( + start_date=start_date, end_date=end_date, prisma_client=prisma_client + ) + + return response + except Exception as e: + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"/spend/tags Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="/spend/tags Error" + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + @router.get( "/spend/logs", tags=["budget & spend Tracking"], From e9b27d98110cbe5c0eb365065898cbd48e0ff9cb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 08:49:37 -0800 Subject: [PATCH 02/14] fix(proxy_server.py): re-add /team/info endpoint (fixing merge issue) --- litellm/proxy/proxy_server.py | 69 ++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c8dd2bbe9b4..66669e70a2c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4076,8 +4076,75 @@ async def team_info( ): """ get info on team + related keys + + ``` + curl --location 'http://localhost:4000/team/info' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "teams": ["",..] + }' + ``` """ - pass + global prisma_client + try: + if prisma_client is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "error": f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" + }, + ) + if team_id is None: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={"message": "Malformed request. No team id passed in."}, + ) + + team_info = await prisma_client.get_data( + team_id=team_id, table_name="team", query_type="find_unique" + ) + ## GET ALL KEYS ## + keys = await prisma_client.get_data( + team_id=team_id, + table_name="key", + query_type="find_all", + expires=datetime.now(), + ) + + if team_info is None: + ## make sure we still return a total spend ## + spend = 0 + for k in keys: + spend += getattr(k, "spend", 0) + team_info = {"spend": spend} + + ## REMOVE HASHED TOKEN INFO before returning ## + for key in keys: + try: + key = key.model_dump() # noqa + except: + # if using pydantic v1 + key = key.dict() + key.pop("token", None) + return {"team_id": team_id, "team_info": team_info, "keys": keys} + + except Exception as e: + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Authentication Error({str(e)})"), + type="auth_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Authentication Error, " + str(e), + type="auth_error", + param=getattr(e, "param", "None"), + code=status.HTTP_400_BAD_REQUEST, + ) #### MODEL MANAGEMENT #### From 08ce7a08d30b67888a21baae63c55e98b82d8227 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 08:55:21 -0800 Subject: [PATCH 03/14] test(test_team.py): adding testing for team endpoints --- tests/test_team.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/test_team.py diff --git a/tests/test_team.py b/tests/test_team.py new file mode 100644 index 00000000000..b7da1bf9e42 --- /dev/null +++ b/tests/test_team.py @@ -0,0 +1,72 @@ +# What this tests ? +## Tests /team endpoints. +import pytest +import asyncio +import aiohttp +import time +from openai import AsyncOpenAI + + +async def new_team( + session, + i, +): + url = "http://0.0.0.0:4000/team/new" + headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} + data = { + "team_alias": "my-new-team", + "admins": ["user-1234"], + "members": ["user-1234"], + } + async with session.post(url, headers=headers, json=data) as response: + status = response.status + response_text = await response.text() + + print(f"Response {i} (Status code: {status}):") + print(response_text) + print() + + if status != 200: + raise Exception(f"Request {i} did not return a 200 status code: {status}") + + return await response.json() + + +@pytest.mark.asyncio +async def test_team_new(): + """ + Make 20 parallel calls to /user/new. Assert all worked. + """ + async with aiohttp.ClientSession() as session: + tasks = [new_team(session, i) for i in range(1, 11)] + await asyncio.gather(*tasks) + + +async def get_team_info(session, get_team, call_key): + url = f"http://0.0.0.0:4000/team/info?team_id={get_team}" + headers = { + "Authorization": f"Bearer {call_key}", + "Content-Type": "application/json", + } + + async with session.get(url, headers=headers) as response: + status = response.status + response_text = await response.text() + print(response_text) + print() + + if status != 200: + raise Exception(f"Request did not return a 200 status code: {status}") + return await response.json() + + +@pytest.mark.asyncio +async def test_team_info(): + async with aiohttp.ClientSession() as session: + new_team_data = await new_team( + session, + 0, + ) + team_id = new_team_data["team_id"] + ## as admin ## + await get_team_info(session=session, get_team=team_id, call_key="sk-1234") From 9267361683efd8c7c16ba8f7f41aa2f19eafa546 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 08:56:08 -0800 Subject: [PATCH 04/14] test(test_team.py): trigger new build --- tests/test_team.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_team.py b/tests/test_team.py index b7da1bf9e42..27284feaeb0 100644 --- a/tests/test_team.py +++ b/tests/test_team.py @@ -3,6 +3,7 @@ import pytest import asyncio import aiohttp + import time from openai import AsyncOpenAI From 21d493d3f2d4c9ce2a1a20b1adbdd19349de65e8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 08:56:13 -0800 Subject: [PATCH 05/14] =?UTF-8?q?bump:=20version=201.24.3=20=E2=86=92=201.?= =?UTF-8?q?24.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 04667a88feb..e6d14c5904c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.24.3" +version = "1.24.4" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -69,7 +69,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.24.3" +version = "1.24.4" version_files = [ "pyproject.toml:^version" ] From 51dc3c64df82bc23f7ed445f612eb950cfb117aa Mon Sep 17 00:00:00 2001 From: Toni Engelhardt Date: Fri, 16 Feb 2024 17:42:19 +0000 Subject: [PATCH 06/14] Update NLP Cloude model pricing --- model_prices_and_context_window.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index fa50b06f1f7..8778aa36089 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -965,15 +965,15 @@ }, "dolphin": { "max_tokens": 4096, - "input_cost_per_token": 0.00002, - "output_cost_per_token": 0.00002, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000005, "litellm_provider": "nlp_cloud", "mode": "completion" }, "chatdolphin": { "max_tokens": 4096, - "input_cost_per_token": 0.00002, - "output_cost_per_token": 0.00002, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000005, "litellm_provider": "nlp_cloud", "mode": "chat" }, From 3fffe96f9704239cb7eaf57b68573a6a593736d7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 09:07:02 -0800 Subject: [PATCH 07/14] refactor(test_team.py): trigger new devrelease --- tests/test_team.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_team.py b/tests/test_team.py index 27284feaeb0..b7da1bf9e42 100644 --- a/tests/test_team.py +++ b/tests/test_team.py @@ -3,7 +3,6 @@ import pytest import asyncio import aiohttp - import time from openai import AsyncOpenAI From b3d48da64076ddc69f1292cd61cc6c8e36fbf806 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 09:56:59 -0800 Subject: [PATCH 08/14] fix(main.py): map list input to ollama prompt input format --- litellm/exceptions.py | 11 ++++++++++- litellm/main.py | 20 +++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 09b3758112b..a7bf394f6d4 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -24,6 +24,7 @@ from openai import ( PermissionDeniedError, ) import httpx +from typing import Optional class AuthenticationError(AuthenticationError): # type: ignore @@ -50,11 +51,19 @@ class NotFoundError(NotFoundError): # type: ignore class BadRequestError(BadRequestError): # type: ignore - def __init__(self, message, model, llm_provider, response: httpx.Response): + def __init__( + self, message, model, llm_provider, response: Optional[httpx.Response] = None + ): self.status_code = 400 self.message = message self.model = model self.llm_provider = llm_provider + response = response or httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="GET", url="https://litellm.ai" + ), # mock request object + ) super().__init__( self.message, response=response, body=None ) # Call the base class constructor with the parameters it needs diff --git a/litellm/main.py b/litellm/main.py index 93ea3c6441a..2539039cd70 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2590,10 +2590,28 @@ def embedding( model_response=EmbeddingResponse(), ) elif custom_llm_provider == "ollama": + ollama_input = None + if isinstance(input, list) and len(input) > 1: + raise litellm.BadRequestError( + message=f"Ollama Embeddings don't support batch embeddings", + model=model, # type: ignore + llm_provider="ollama", # type: ignore + ) + if isinstance(input, list) and len(input) == 1: + ollama_input = "".join(input[0]) + elif isinstance(input, str): + ollama_input = input + else: + raise litellm.BadRequestError( + message=f"Invalid input for ollama embeddings. input={input}", + model=model, # type: ignore + llm_provider="ollama", # type: ignore + ) + if aembedding == True: response = ollama.ollama_aembeddings( model=model, - prompt=input, + prompt=ollama_input, encoding=encoding, logging_obj=logging, optional_params=optional_params, From 4067082f4788228cd7969cd3c8110755272ed515 Mon Sep 17 00:00:00 2001 From: Toni Engelhardt Date: Fri, 16 Feb 2024 18:04:37 +0000 Subject: [PATCH 09/14] Update NLP Cloud max_tokens --- model_prices_and_context_window.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8778aa36089..75d0ba55f33 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -964,14 +964,14 @@ "mode": "completion" }, "dolphin": { - "max_tokens": 4096, + "max_tokens": 16384, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "nlp_cloud", "mode": "completion" }, "chatdolphin": { - "max_tokens": 4096, + "max_tokens": 16384, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "nlp_cloud", From 02a5c105ac9c28141b23e525b8fbf93476b9565a Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 16 Feb 2024 10:47:33 -0800 Subject: [PATCH 10/14] (docs) view spend per tag --- docs/my-website/docs/proxy/spend.md | 147 ++++++++++++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 148 insertions(+) create mode 100644 docs/my-website/docs/proxy/spend.md diff --git a/docs/my-website/docs/proxy/spend.md b/docs/my-website/docs/proxy/spend.md new file mode 100644 index 00000000000..c76bbc7e98f --- /dev/null +++ b/docs/my-website/docs/proxy/spend.md @@ -0,0 +1,147 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 💸 Spend Tracking + +:::info + +This is an Enterprise only feature [Get Started with Enterprise here](https://github.com/BerriAI/litellm/tree/main/enterprise) + +::: + +Requirements: + +- Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys) + + +## Tracking Spend per Request Tag + +### Usage - /chat/completions requests with request tags + + + + + + + +Set `extra_body={"metadata": { }}` to `metadata` you want to pass + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:8000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"] + } + } +) + +print(response) +``` + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:8000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "metadata": {"tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"]} +}' +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:8000", + model = "gpt-3.5-turbo", + temperature=0.1, + extra_body={ + "metadata": { + "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"] + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + +### Viewing Spend per tag + +#### `/spend/tags` Request Format +```shell +curl -X GET "http://0.0.0.0:4000/spend/tags" \ +-H "Authorization: Bearer sk-1234" +``` + +#### `/spend/tags`Response Format +```shell +[ + { + "individual_request_tag": "model-anthropic-claude-v2.1", + "log_count": 6, + "total_spend": 0.000672 + }, + { + "individual_request_tag": "app-ishaan-local", + "log_count": 4, + "total_spend": 0.000448 + }, + { + "individual_request_tag": "app-ishaan-prod", + "log_count": 2, + "total_spend": 0.000224 + } +] + +``` + + + \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 111d8923269..b564befba5c 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -112,6 +112,7 @@ const sidebars = { "proxy/user_keys", "proxy/virtual_keys", "proxy/users", + "proxy/spend", "proxy/ui", "proxy/model_management", "proxy/health", From 936e460a30cdd526d44e9208c2a941ec34647306 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 16 Feb 2024 10:53:33 -0800 Subject: [PATCH 11/14] (chore) fix spend tracking request tags --- litellm/proxy/proxy_server.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 66669e70a2c..54554042217 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -820,7 +820,6 @@ async def _PROXY_track_cost_callback( "user_api_key_user_id", None ) team_id = kwargs["litellm_params"]["metadata"].get("user_api_key_team_id", None) - request_tags = kwargs["litellm_params"]["metadata"].get("tags", None) if kwargs.get("response_cost", None) is not None: response_cost = kwargs["response_cost"] user_api_key = kwargs["litellm_params"]["metadata"].get( @@ -845,7 +844,6 @@ async def _PROXY_track_cost_callback( response_cost=response_cost, user_id=user_id, team_id=team_id, - request_tags=request_tags, kwargs=kwargs, completion_response=completion_response, start_time=start_time, @@ -884,7 +882,6 @@ async def update_database( response_cost, user_id=None, team_id=None, - request_tags=None, kwargs=None, completion_response=None, start_time=None, @@ -892,7 +889,7 @@ async def update_database( ): try: verbose_proxy_logger.info( - f"Enters prisma db call, response_cost: {response_cost}, token: {token}; user_id: {user_id}; team_id: {team_id} request_tags: {request_tags}" + f"Enters prisma db call, response_cost: {response_cost}, token: {token}; user_id: {user_id}; team_id: {team_id}" ) ### [TODO] STEP 1: GET KEY + USER SPEND ### (key, user) @@ -906,12 +903,6 @@ async def update_database( - Update litellm-proxy-budget row (global proxy spend) """ user_ids = [user_id, litellm_proxy_budget_name] - if request_tags is not None: - # add prefix: litellm_request_tag_ to request_tag - new_request_tags = [ - f"litellm_request_tag_{tag}" for tag in request_tags - ] - user_ids += new_request_tags data_list = [] try: for id in user_ids: @@ -926,7 +917,7 @@ async def update_database( key=id, table_name="user" ) verbose_proxy_logger.debug( - f"Updating existing_spend_obj: {existing_spend_obj}, user_id: {id}" + f"Updating existing_spend_obj: {existing_spend_obj}" ) if existing_spend_obj is None: # if user does not exist in LiteLLM_UserTable, create a new user From 01077b20b9cdb2d3818d2ba396eb5b85fe2d1f2b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 11:49:59 -0800 Subject: [PATCH 12/14] fix(proxy_server.py): restrict model access for /v1/completions endpoint --- litellm/model_prices_and_context_window_backup.json | 8 ++++---- litellm/proxy/proxy_server.py | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index fa50b06f1f7..8778aa36089 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -965,15 +965,15 @@ }, "dolphin": { "max_tokens": 4096, - "input_cost_per_token": 0.00002, - "output_cost_per_token": 0.00002, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000005, "litellm_provider": "nlp_cloud", "mode": "completion" }, "chatdolphin": { "max_tokens": 4096, - "input_cost_per_token": 0.00002, - "output_cost_per_token": 0.00002, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000005, "litellm_provider": "nlp_cloud", "mode": "chat" }, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 54554042217..c563396387e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2259,8 +2259,13 @@ async def completion( response = await llm_router.atext_completion( **data, specific_deployment=True ) - else: # router is not set + elif user_model is not None: # `litellm --model ` response = await litellm.atext_completion(**data) + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "Invalid model name passed in"}, + ) if hasattr(response, "_hidden_params"): model_id = response._hidden_params.get("model_id", None) or "" From 0d548871493e2bc1e1aec57fe7bba362fef24510 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 11:51:41 -0800 Subject: [PATCH 13/14] test(test_amazing_vertex_completion.py): handle rate limit errors --- litellm/model_prices_and_context_window_backup.json | 4 ++-- litellm/tests/test_amazing_vertex_completion.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8778aa36089..75d0ba55f33 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -964,14 +964,14 @@ "mode": "completion" }, "dolphin": { - "max_tokens": 4096, + "max_tokens": 16384, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "nlp_cloud", "mode": "completion" }, "chatdolphin": { - "max_tokens": 4096, + "max_tokens": 16384, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "nlp_cloud", diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 5e48d7941b9..9b7473ea275 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -266,6 +266,8 @@ async def test_async_vertexai_streaming_response(): complete_response += chunk.choices[0].delta.content print(f"complete_response: {complete_response}") assert len(complete_response) > 0 + except litellm.RateLimitError as e: + pass except litellm.Timeout as e: pass except Exception as e: From cab8a3c2f539486edaf576859c3ca8d88d0f94ce Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 16 Feb 2024 11:52:59 -0800 Subject: [PATCH 14/14] =?UTF-8?q?bump:=20version=201.24.4=20=E2=86=92=201.?= =?UTF-8?q?24.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e6d14c5904c..3833fd0ae3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.24.4" +version = "1.24.5" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -69,7 +69,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.24.4" +version = "1.24.5" version_files = [ "pyproject.toml:^version" ]