From a4acece4c13b3e426aa84d4bc3cf32f010698d1f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 26 Jan 2024 19:30:06 -0800 Subject: [PATCH 01/27] (docstring) /key/info --- litellm/proxy/proxy_server.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4a854ec7619..13f0d55a970 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2510,6 +2510,26 @@ async def info_key_fn( ), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): + """ + Retrieve information about a key. + Parameters: + key: Optional[str] = Query parameter representing the key in the request + user_api_key_dict: UserAPIKeyAuth = Dependency representing the user's API key + Returns: + Dict containing the key and its associated information + + Example Curl: + ``` + curl -X GET "http://0.0.0.0:8000/key/info?key=sk-02Wr4IAlN3NvPXvL5JVvDA" \ +-H "Authorization: Bearer sk-1234" + ``` + + Example - if no key is passed, it will use the Key Passed in Authorization Header + ``` + curl -X GET "http://0.0.0.0:8000/key/info" \ +-H "Authorization: Bearer sk-02Wr4IAlN3NvPXvL5JVvDA" + ``` + """ global prisma_client try: if prisma_client is None: From 18624f8490bf6bc314f9b921eda9e2f51c84b02b Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 26 Jan 2024 19:30:35 -0800 Subject: [PATCH 02/27] (docstring) /key/info --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 13f0d55a970..d9b8fcce2d9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2524,7 +2524,7 @@ async def info_key_fn( -H "Authorization: Bearer sk-1234" ``` - Example - if no key is passed, it will use the Key Passed in Authorization Header + Example Curl - if no key is passed, it will use the Key Passed in Authorization Header ``` curl -X GET "http://0.0.0.0:8000/key/info" \ -H "Authorization: Bearer sk-02Wr4IAlN3NvPXvL5JVvDA" From 950c753429d41c2dc05086085e5e6950471f039c Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 08:31:50 -0800 Subject: [PATCH 03/27] (docs) on callbacks tracking api_key, base etc --- docs/my-website/docs/routing.md | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 3b796c87ffe..151065d76a1 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -605,6 +605,49 @@ response = router.completion(model="gpt-3.5-turbo", messages=messages) print(f"response: {response}") ``` +## Custom Callbacks - Track API Key, API Endpoint, Model Used + +If you need to track the api_key, api endpoint, model, custom_llm_provider used for each completion call, you can setup a [custom callback](https://docs.litellm.ai/docs/observability/custom_callback) + +### Usage + +```python +import litellm +from litellm.integrations.custom_logger import CustomLogger + +class MyCustomHandler(CustomLogger): + def log_success_event(self, kwargs, response_obj, start_time, end_time): + print(f"On Success") + print("kwargs=", kwargs) + litellm_params= kwargs.get("litellm_params") + api_key = litellm_params.get("api_key") + api_base = litellm_params.get("api_base") + custom_llm_provider= litellm_params.get("custom_llm_provider") + response_cost = kwargs.get("response_cost") + + # print the values + print("api_key=", api_key) + print("api_base=", api_base) + print("custom_llm_provider=", custom_llm_provider) + print("response_cost=", response_cost) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + print(f"On Failure") + print("kwargs=") + +customHandler = MyCustomHandler() + +litellm.callbacks = [customHandler] + +# Init Router +router = Router(model_list=model_list, routing_strategy="simple-shuffle") + +# router completion call +response = router.completion( + model="gpt-3.5-turbo", + messages=[{ "role": "user", "content": "Hi who are you"}] +) +``` ## Deploy Router From 4ba809b8357e113b8387991c4abd2758fbb2fdf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Campion?= Date: Sat, 27 Jan 2024 19:16:53 +0100 Subject: [PATCH 04/27] Allow optional usage of the tls encryption for SMTPA For local dev, a local SMTP server like mailhog is useful and allow to manually manage user creation --- litellm/proxy/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 713f117cabb..754b8a053a4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1022,7 +1022,8 @@ async def send_email(sender_name, sender_email, receiver_email, subject, html): print_verbose(f"SMTP Connection Init") # Establish a secure connection with the SMTP server with smtplib.SMTP(smtp_host, smtp_port) as server: - server.starttls() + if os.getenv("SMTP_TLS", 'True') != "False": + server.starttls() # Login to your email account server.login(smtp_username, smtp_password) From 1a8dee372ae986cb7e0d3721b3847e909f4be31a Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 13:57:04 -0800 Subject: [PATCH 05/27] (feat) add google login to litellm proxy --- litellm/proxy/proxy_server.py | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d9b8fcce2d9..ac1dbcc2415 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -104,6 +104,7 @@ from fastapi.responses import ( ORJSONResponse, JSONResponse, ) +from fastapi.responses import RedirectResponse from fastapi.middleware.cors import CORSMiddleware from fastapi.security.api_key import APIKeyHeader import json @@ -2856,6 +2857,70 @@ async def user_auth(request: Request): return "Email sent!" +@app.get("/google-login", tags=["experimental"]) +async def google_login(): + GOOGLE_REDIRECT_URI = "http://localhost:4000/google-callback" + GOOGLE_CLIENT_ID = ( + "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" + ) + google_auth_url = f"https://accounts.google.com/o/oauth2/auth?client_id={GOOGLE_CLIENT_ID}&redirect_uri={GOOGLE_REDIRECT_URI}&response_type=code&scope=openid%20profile%20email" + return RedirectResponse(url=google_auth_url) + + +@app.get("/google-callback", tags=["experimental"]) +async def google_callback(code: str): + import httpx + + GOOGLE_REDIRECT_URI = "http://localhost:4000/google-callback" + GOOGLE_CLIENT_ID = ( + "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" + ) + # Exchange code for access token + async with httpx.AsyncClient() as client: + token_url = f"https://oauth2.googleapis.com/token" + data = { + "code": code, + "client_id": GOOGLE_CLIENT_ID, + "client_secret": "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", + "redirect_uri": GOOGLE_REDIRECT_URI, + "grant_type": "authorization_code", + } + response = await client.post(token_url, data=data) + + # Process the response, extract user info, etc. + if response.status_code == 200: + access_token = response.json()["access_token"] + + # Fetch user info using the access token + async with httpx.AsyncClient() as client: + user_info_url = "https://www.googleapis.com/oauth2/v1/userinfo" + headers = {"Authorization": f"Bearer {access_token}"} + user_info_response = await client.get(user_info_url, headers=headers) + + # Process user info response + if user_info_response.status_code == 200: + user_info = user_info_response.json() + user_email = user_info.get("email") + user_name = user_info.get("name") + + # we can use user_email on litellm proxy now + + # TODO: Handle user info as needed, for example, store it in a database, authenticate the user, etc. + return JSONResponse( + content={"user_email": user_email, "user_name": user_name}, + status_code=200, + ) + else: + # Handle user info retrieval error + raise HTTPException( + status_code=user_info_response.status_code, + detail=user_info_response.text, + ) + else: + # Handle the error from the token exchange + raise HTTPException(status_code=response.status_code, detail=response.text) + + @router.get( "/user/info", tags=["user management"], dependencies=[Depends(user_api_key_auth)] ) From 180a1a7f9c730a66fad297d9caaa780c6b1e4ec0 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 14:17:03 -0800 Subject: [PATCH 06/27] (fix) return dict response --- litellm/proxy/proxy_server.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ac1dbcc2415..749705157c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2867,7 +2867,7 @@ async def google_login(): return RedirectResponse(url=google_auth_url) -@app.get("/google-callback", tags=["experimental"]) +@app.get("/google-callback", tags=["experimental"], response_model=GenerateKeyResponse) async def google_callback(code: str): import httpx @@ -2906,10 +2906,16 @@ async def google_callback(code: str): # we can use user_email on litellm proxy now # TODO: Handle user info as needed, for example, store it in a database, authenticate the user, etc. - return JSONResponse( - content={"user_email": user_email, "user_name": user_name}, - status_code=200, + response = await generate_key_helper_fn( + **{"duration": "24hr", "models": [], "aliases": {}, "config": {}, "spend": 0, "user_id": user_email, "team_id": "litellm-dashboard"} # type: ignore ) + + key = response["token"] # type: ignore + user_id = response["user_id"] # type: ignore + { + "key": key, + "user_id": user_id, + } else: # Handle user info retrieval error raise HTTPException( From a7bbc0de0ff5bf4d3801ba73957600629146323d Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 14:19:11 -0800 Subject: [PATCH 07/27] (fix) google-login auth flow proxy --- litellm/proxy/proxy_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 749705157c0..61103e079cb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2912,10 +2912,10 @@ async def google_callback(code: str): key = response["token"] # type: ignore user_id = response["user_id"] # type: ignore - { - "key": key, - "user_id": user_id, - } + return JSONResponse( + content={"key": key, "user_id": user_id}, status_code=200 + ) + else: # Handle user info retrieval error raise HTTPException( From 8d3e818fb15fe259593c60a49e4596bc57075d31 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Jan 2024 14:36:30 -0800 Subject: [PATCH 08/27] Update Dockerfile --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e18d5d97952..5c0c4ad36dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,9 +47,13 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels +# Generate prisma client +RUN prisma generate RUN chmod +x entrypoint.sh EXPOSE 4000/tcp +# # Set your entrypoint and command + ENTRYPOINT ["litellm"] -CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--detailed_debug"] \ No newline at end of file +CMD ["--port", "4000"] From ad6607b530fd698e479e737d5e09517da6d6a81e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Jan 2024 14:37:38 -0800 Subject: [PATCH 09/27] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5c0c4ad36dc..8bc6e57838f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,5 +55,5 @@ EXPOSE 4000/tcp # # Set your entrypoint and command -ENTRYPOINT ["litellm"] +ENTRYPOINT ["python3 litellm/proxy/proxy_cli.py"] CMD ["--port", "4000"] From 4a98104af343cd0f529ce157ab140ec203ab01f5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Jan 2024 14:43:50 -0800 Subject: [PATCH 10/27] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8bc6e57838f..897736bb59e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,5 +55,5 @@ EXPOSE 4000/tcp # # Set your entrypoint and command -ENTRYPOINT ["python3 litellm/proxy/proxy_cli.py"] +ENTRYPOINT ["python3 litellm/litellm/proxy/proxy_cli.py"] CMD ["--port", "4000"] From 18906e6aff2ad04ead94f522794450536998bbf8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Jan 2024 14:57:41 -0800 Subject: [PATCH 11/27] Update Dockerfile --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 897736bb59e..ac7b21e5c10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,10 @@ RUN chmod +x entrypoint.sh EXPOSE 4000/tcp # # Set your entrypoint and command +RUN echo "app contents" + +# List contents of /app +RUN ls -la /app ENTRYPOINT ["python3 litellm/litellm/proxy/proxy_cli.py"] CMD ["--port", "4000"] From 7f2e4036d6e16faaa21b6d2d93b28b4d265a7d56 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Jan 2024 14:59:36 -0800 Subject: [PATCH 12/27] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ac7b21e5c10..de47b60195f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,5 +59,5 @@ RUN echo "app contents" # List contents of /app RUN ls -la /app -ENTRYPOINT ["python3 litellm/litellm/proxy/proxy_cli.py"] +ENTRYPOINT ["python3 litellm/proxy/proxy_cli.py"] CMD ["--port", "4000"] From 6b0024a7d3fa3ae775b817302fbee334b72fd4c4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Jan 2024 15:01:02 -0800 Subject: [PATCH 13/27] Update Dockerfile --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index de47b60195f..39d88d4ebf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,5 +59,7 @@ RUN echo "app contents" # List contents of /app RUN ls -la /app -ENTRYPOINT ["python3 litellm/proxy/proxy_cli.py"] + +ENTRYPOINT ["python3", "/app/litellm/proxy/proxy_cli.py"] + CMD ["--port", "4000"] From 28580306c0473a147dff51b36edbf84e37d9b93f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 15:02:50 -0800 Subject: [PATCH 14/27] =?UTF-8?q?bump:=20version=201.20.0=20=E2=86=92=201.?= =?UTF-8?q?20.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 744263c69aa..d46b85ea234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.20.0" +version = "1.20.1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -63,7 +63,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.20.0" +version = "1.20.1" version_files = [ "pyproject.toml:^version" ] From 020b1ec7568a02359fefa201e9361b963192785e Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 15:13:43 -0800 Subject: [PATCH 15/27] (fix) use dynamic redirect urls --- litellm/proxy/proxy_server.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 61103e079cb..6b5502b3c28 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2858,8 +2858,11 @@ async def user_auth(request: Request): @app.get("/google-login", tags=["experimental"]) -async def google_login(): - GOOGLE_REDIRECT_URI = "http://localhost:4000/google-callback" +async def google_login(request: Request): + scheme = request.url.scheme + host = request.url.hostname + port = request.url.port or 4000 + GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" ) @@ -2868,10 +2871,14 @@ async def google_login(): @app.get("/google-callback", tags=["experimental"], response_model=GenerateKeyResponse) -async def google_callback(code: str): +async def google_callback(code: str, request: Request): import httpx - GOOGLE_REDIRECT_URI = "http://localhost:4000/google-callback" + scheme = request.url.scheme + host = request.url.hostname + port = request.url.port or 4000 + + GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" ) From dc78d16d055b2a6895dace5c9c1ce70933312bfb Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 15:25:00 -0800 Subject: [PATCH 16/27] (fix) auth google --- litellm/proxy/proxy_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6b5502b3c28..3da2e40e3f5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2875,8 +2875,10 @@ async def google_callback(code: str, request: Request): import httpx scheme = request.url.scheme - host = request.url.hostname + host = request.url.hostname or "localhost" port = request.url.port or 4000 + if "localhost" not in host: + scheme = "https" GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" GOOGLE_CLIENT_ID = ( From b522edcdf1f097b1668fed3d7e4e4f0392165d2b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Jan 2024 15:36:52 -0800 Subject: [PATCH 17/27] Update proxy_server.py --- litellm/proxy/proxy_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3da2e40e3f5..10748042ca5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2862,6 +2862,8 @@ async def google_login(request: Request): scheme = request.url.scheme host = request.url.hostname port = request.url.port or 4000 + if "localhost" not in host: + scheme = "https" GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" From 3699df2b31fdab3c97e75ae9e4f966dea12acadc Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 15:38:09 -0800 Subject: [PATCH 18/27] (fix) google auth --- litellm/proxy/proxy_server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3da2e40e3f5..d54eb39900e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2859,9 +2859,11 @@ async def user_auth(request: Request): @app.get("/google-login", tags=["experimental"]) async def google_login(request: Request): - scheme = request.url.scheme - host = request.url.hostname + scheme = request.url.scheme or "https" + host = request.url.hostname or "localhost" port = request.url.port or 4000 + if "localhost" not in host: + scheme = "https" GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" @@ -2874,7 +2876,7 @@ async def google_login(request: Request): async def google_callback(code: str, request: Request): import httpx - scheme = request.url.scheme + scheme = request.url.scheme or "https" host = request.url.hostname or "localhost" port = request.url.port or 4000 if "localhost" not in host: From d8690e50ea3f823904258686faa2e7a31bc23729 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Jan 2024 16:26:12 -0800 Subject: [PATCH 19/27] Update proxy_server.py --- litellm/proxy/proxy_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d54eb39900e..1257015282c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2864,7 +2864,7 @@ async def google_login(request: Request): port = request.url.port or 4000 if "localhost" not in host: scheme = "https" - GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" + GOOGLE_REDIRECT_URI = f"https://litellm-production-7002.up.railway.app/google-callback" GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" ) @@ -2882,7 +2882,7 @@ async def google_callback(code: str, request: Request): if "localhost" not in host: scheme = "https" - GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" + GOOGLE_REDIRECT_URI = f"https://litellm-production-7002.up.railway.app/google-callback" GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" ) From 9bd4fa40ff8592d6df14ba3161bfad2aed1c8c7e Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 16:38:56 -0800 Subject: [PATCH 20/27] (fix) proxy use env as GOOGLE_REDIRECT_URI --- litellm/proxy/proxy_server.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d54eb39900e..4675ab0651c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2859,12 +2859,7 @@ async def user_auth(request: Request): @app.get("/google-login", tags=["experimental"]) async def google_login(request: Request): - scheme = request.url.scheme or "https" - host = request.url.hostname or "localhost" - port = request.url.port or 4000 - if "localhost" not in host: - scheme = "https" - GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" + GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI") GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" ) @@ -2876,13 +2871,7 @@ async def google_login(request: Request): async def google_callback(code: str, request: Request): import httpx - scheme = request.url.scheme or "https" - host = request.url.hostname or "localhost" - port = request.url.port or 4000 - if "localhost" not in host: - scheme = "https" - - GOOGLE_REDIRECT_URI = f"{scheme}://{host}:{port}/google-callback" + GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI") GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" ) From 9dd857e0a6d44a9b4c2f68395e086ad357ffb3bd Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 16:41:11 -0800 Subject: [PATCH 21/27] (fix) endpoint name --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4675ab0651c..1154cf816d4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2857,7 +2857,7 @@ async def user_auth(request: Request): return "Email sent!" -@app.get("/google-login", tags=["experimental"]) +@app.get("/google-login/key/generate", tags=["experimental"]) async def google_login(request: Request): GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI") GOOGLE_CLIENT_ID = ( From 308351458b3ba159d80bc55f61f5fbd17eb0a60c Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 16:47:10 -0800 Subject: [PATCH 22/27] Revert "Merge branch 'main' into main" This reverts commit a92461caa5352bd06ca40962e3bed7f36c7e669c, reversing changes made to 9dd857e0a6d44a9b4c2f68395e086ad357ffb3bd. --- Dockerfile | 14 ++------------ litellm/proxy/proxy_server.py | 2 +- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 39d88d4ebf5..e18d5d97952 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,19 +47,9 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels -# Generate prisma client -RUN prisma generate RUN chmod +x entrypoint.sh EXPOSE 4000/tcp -# # Set your entrypoint and command -RUN echo "app contents" - -# List contents of /app -RUN ls -la /app - - -ENTRYPOINT ["python3", "/app/litellm/proxy/proxy_cli.py"] - -CMD ["--port", "4000"] +ENTRYPOINT ["litellm"] +CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--detailed_debug"] \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 664dabd101a..1154cf816d4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2859,7 +2859,6 @@ async def user_auth(request: Request): @app.get("/google-login/key/generate", tags=["experimental"]) async def google_login(request: Request): - GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI") GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" @@ -2871,6 +2870,7 @@ async def google_login(request: Request): @app.get("/google-callback", tags=["experimental"], response_model=GenerateKeyResponse) async def google_callback(code: str, request: Request): import httpx + GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI") GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" From 9d5bfa45c13dbcfdcf6b1b043165ffc85f4c8b12 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 16:50:53 -0800 Subject: [PATCH 23/27] (fix) use GOOGLE_REDIRECT_URI. --- litellm/proxy/proxy_server.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1154cf816d4..72029b4fbd4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2860,6 +2860,17 @@ async def user_auth(request: Request): @app.get("/google-login/key/generate", tags=["experimental"]) async def google_login(request: Request): GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI") + if GOOGLE_REDIRECT_URI is None: + raise ProxyException( + message="GOOGLE_REDIRECT_URI not set. Set it in .env file", + type="auth_error", + param="GOOGLE_REDIRECT_URI", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + if GOOGLE_REDIRECT_URI.endswith("/"): + GOOGLE_REDIRECT_URI += "google-callback" + else: + GOOGLE_REDIRECT_URI += "/google-callback" GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" ) @@ -2872,6 +2883,18 @@ async def google_callback(code: str, request: Request): import httpx GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI") + if GOOGLE_REDIRECT_URI is None: + raise ProxyException( + message="GOOGLE_REDIRECT_URI not set. Set it in .env file", + type="auth_error", + param="GOOGLE_REDIRECT_URI", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + # Add "/google-callback"" to your callback URL + if GOOGLE_REDIRECT_URI.endswith("/"): + GOOGLE_REDIRECT_URI += "google-callback" + else: + GOOGLE_REDIRECT_URI += "/google-callback" GOOGLE_CLIENT_ID = ( "246483686424-clje5sggkjma26ilktj6qssakqhoon0m.apps.googleusercontent.com" ) From 4731f869e30f2686b62a26305e9dc038f29a25f4 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 16:56:17 -0800 Subject: [PATCH 24/27] (docstring) /google-login --- litellm/proxy/proxy_server.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 72029b4fbd4..b6651e022f0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2859,6 +2859,13 @@ async def user_auth(request: Request): @app.get("/google-login/key/generate", tags=["experimental"]) async def google_login(request: Request): + """ + Create Proxy API Keys using Google Workspace SSO. Requires setting GOOGLE_REDIRECT_URI in .env + + GOOGLE_REDIRECT_URI should be the your deployed proxy endpoint, e.g. GOOGLE_REDIRECT_URI="https://litellm-production-7002.up.railway.app" + Example: + + """ GOOGLE_REDIRECT_URI = os.getenv("GOOGLE_REDIRECT_URI") if GOOGLE_REDIRECT_URI is None: raise ProxyException( From f7e64cd528e21a95a6b7351b804d89c01ad31639 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 27 Jan 2024 18:17:58 -0800 Subject: [PATCH 25/27] (fix) google/login use redirect resp --- litellm/proxy/proxy_server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b6651e022f0..c017d7c13f8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2942,9 +2942,11 @@ async def google_callback(code: str, request: Request): key = response["token"] # type: ignore user_id = response["user_id"] # type: ignore - return JSONResponse( - content={"key": key, "user_id": user_id}, status_code=200 - ) + return RedirectResponse(url="chat.openai.com") + # return RedirectResponse(url=google_auth_url) + # return JSONResponse( + # content={"key": key, "user_id": user_id}, status_code=200 + # ) else: # Handle user info retrieval error From bcad7c58a8ac61a0feb03860eb8a34ca2c6cd88e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 25 Jan 2024 20:07:31 -0800 Subject: [PATCH 26/27] feat(main.py): support auto-infering mode if not set --- litellm/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/main.py b/litellm/main.py index 84b1b79d199..05b352fa6b1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3175,6 +3175,9 @@ async def ahealth_check( if model is None: raise Exception("model not set") + if model in litellm.model_cost and mode is None: + mode = litellm.model_cost[model]["mode"] + model, custom_llm_provider, _, _ = get_llm_provider(model=model) mode = mode or "chat" # default to chat completion calls From 0c9e56aff1f3cdf775d363d6fa5bb0ddc2074a7b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 27 Jan 2024 20:03:07 -0800 Subject: [PATCH 27/27] docs(virtual_keys.md): add key alias and key name to docs --- docs/my-website/docs/proxy/virtual_keys.md | 31 +++++++++------------- litellm/proxy/proxy_server.py | 3 ++- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/docs/my-website/docs/proxy/virtual_keys.md b/docs/my-website/docs/proxy/virtual_keys.md index e1c89bbc216..701af27e94b 100644 --- a/docs/my-website/docs/proxy/virtual_keys.md +++ b/docs/my-website/docs/proxy/virtual_keys.md @@ -81,15 +81,17 @@ curl 'http://0.0.0.0:8000/key/generate' \ Request Params: -- `models`: *list or null (optional)* - Specify the models a token has access too. If null, then token has access to all models on server. +- `duration`: *Optional[str]* - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). +- `key_alias`: *Optional[str]* - User defined key alias +- `team_id`: *Optional[str]* - The team id of the user +- `models`: *Optional[list]* - Model_name's a user is allowed to call. (if empty, key is allowed to call all models) +- `aliases`: *Optional[dict]* - Any alias mappings, on top of anything in the config.yaml model list. - https://docs.litellm.ai/docs/proxy/virtual_keys#managing-auth---upgradedowngrade-models +- `config`: *Optional[dict]* - any key-specific configs, overrides config in config.yaml +- `spend`: *Optional[int]* - Amount spent by key. Default is 0. Will be updated by proxy whenever key is used. https://docs.litellm.ai/docs/proxy/virtual_keys#managing-auth---tracking-spend +- `max_budget`: *Optional[float]* - Specify max budget for a given key. +- `max_parallel_requests`: *Optional[int]* - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. +- `metadata`: *Optional[dict]* - Metadata for key, store information for key. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } -- `duration`: *str or null (optional)* Specify the length of time the token is valid for. If null, default is set to 1 hour. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). - -- `metadata`: *dict or null (optional)* Pass metadata for the created token. If null defaults to {} - -- `team_id`: *str or null (optional)* Specify team_id for the associated key - -- `max_budget`: *float or null (optional)* Specify max budget (in Dollars $) for a given key. If no value is set, the key has no budget ### Response @@ -97,20 +99,11 @@ Request Params: { "key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token "expires": "2023-11-19T01:38:25.838000+00:00" # datetime object + "key_name": "sk-...7sFA" # abbreviated key string, ONLY stored in db if `allow_user_auth: true` set - [see](./ui.md) + ... } ``` -### Keys that don't expire - -Just set duration to None. - -```bash -curl --location 'http://0.0.0.0:8000/key/generate' \ ---header 'Authorization: Bearer ' \ ---header 'Content-Type: application/json' \ ---data '{"models": ["azure-models"], "aliases": {"mistral-7b": "gpt-3.5-turbo"}, "duration": null}' -``` - ### Upgrade/Downgrade Models If a user is expected to use a given model (i.e. gpt3-5), and you want to: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c017d7c13f8..c4487990a5c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2358,7 +2358,8 @@ async def generate_key_fn( Docs: https://docs.litellm.ai/docs/proxy/virtual_keys Parameters: - - duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). **(Default is set to 1 hour.)** + - duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). + - key_alias: Optional[str] - User defined key alias - team_id: Optional[str] - The team id of the user - models: Optional[list] - Model_name's a user is allowed to call. (if empty, key is allowed to call all models) - aliases: Optional[dict] - Any alias mappings, on top of anything in the config.yaml model list. - https://docs.litellm.ai/docs/proxy/virtual_keys#managing-auth---upgradedowngrade-models