From 3c4c78a71f8e5048fcf219c2bc27ae922e8a555d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 5 Aug 2024 11:18:59 -0700 Subject: [PATCH 01/17] feat(caching.py): enable caching on provider-specific optional params Closes https://github.com/BerriAI/litellm/issues/5049 --- litellm/__init__.py | 3 + litellm/caching.py | 23 +++++-- litellm/main.py | 69 +++----------------- litellm/tests/.litellm_cache/cache.db | Bin 0 -> 32768 bytes litellm/tests/test_caching.py | 87 +++++++++++++++++++++++--- litellm/types/utils.py | 62 ++++++++++++++++++ litellm/utils.py | 2 +- 7 files changed, 172 insertions(+), 74 deletions(-) create mode 100644 litellm/tests/.litellm_cache/cache.db diff --git a/litellm/__init__.py b/litellm/__init__.py index 6dc678b3e59..22255eb34a2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -146,6 +146,9 @@ return_response_headers: bool = ( ) ################## logging: bool = True +enable_caching_on_optional_params: bool = ( + False # feature-flag for caching on optional params - e.g. 'top_k' +) caching: bool = ( False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) diff --git a/litellm/caching.py b/litellm/caching.py index c23c1641b0e..ab62c344064 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -23,6 +23,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs from litellm.types.services import ServiceLoggerPayload, ServiceTypes +from litellm.types.utils import all_litellm_params def print_verbose(print_statement): @@ -1838,6 +1839,7 @@ class Cache: "seed", "tools", "tool_choice", + "stream", ] embedding_only_kwargs = [ "input", @@ -1851,9 +1853,9 @@ class Cache: combined_kwargs = ( completion_kwargs + embedding_only_kwargs + transcription_only_kwargs ) - for param in combined_kwargs: - # ignore litellm params here - if param in kwargs: + litellm_param_kwargs = all_litellm_params + for param in kwargs: + if param in combined_kwargs: # check if param == model and model_group is passed in, then override model with model_group if param == "model": model_group = None @@ -1897,6 +1899,17 @@ class Cache: continue # ignore None params param_value = kwargs[param] cache_key += f"{str(param)}: {str(param_value)}" + elif ( + param not in litellm_param_kwargs + ): # check if user passed in optional param - e.g. top_k + if ( + litellm.enable_caching_on_optional_params is True + ): # feature flagged for now + if kwargs[param] is None: + continue # ignore None params + param_value = kwargs[param] + cache_key += f"{str(param)}: {str(param_value)}" + print_verbose(f"\nCreated cache key: {cache_key}") # Use hashlib to create a sha256 hash of the cache key hash_object = hashlib.sha256(cache_key.encode()) @@ -2101,9 +2114,7 @@ class Cache: try: cache_list = [] for idx, i in enumerate(kwargs["input"]): - preset_cache_key = litellm.cache.get_cache_key( - *args, **{**kwargs, "input": i} - ) + preset_cache_key = self.get_cache_key(*args, **{**kwargs, "input": i}) kwargs["cache_key"] = preset_cache_key embedding_response = result.data[idx] cache_key, cached_data, kwargs = self._add_cache_logic( diff --git a/litellm/main.py b/litellm/main.py index f0eb00ecdd7..fd1adc15ba0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -125,7 +125,11 @@ from .llms.vertex_ai_partner import VertexAIPartnerModels from .llms.vertex_httpx import VertexLLM from .llms.watsonx import IBMWatsonXAI from .types.llms.openai import HttpxBinaryResponseContent -from .types.utils import AdapterCompletionStreamWrapper, ChatCompletionMessageToolCall +from .types.utils import ( + AdapterCompletionStreamWrapper, + ChatCompletionMessageToolCall, + all_litellm_params, +) encoding = tiktoken.get_encoding("cl100k_base") from litellm.utils import ( @@ -744,64 +748,9 @@ def completion( "top_logprobs", "extra_headers", ] - litellm_params = [ - "metadata", - "tags", - "acompletion", - "atext_completion", - "text_completion", - "caching", - "mock_response", - "api_key", - "api_version", - "api_base", - "force_timeout", - "logger_fn", - "verbose", - "custom_llm_provider", - "litellm_logging_obj", - "litellm_call_id", - "use_client", - "id", - "fallbacks", - "azure", - "headers", - "model_list", - "num_retries", - "context_window_fallback_dict", - "retry_policy", - "roles", - "final_prompt_value", - "bos_token", - "eos_token", - "request_timeout", - "complete_response", - "self", - "client", - "rpm", - "tpm", - "max_parallel_requests", - "input_cost_per_token", - "output_cost_per_token", - "input_cost_per_second", - "output_cost_per_second", - "hf_model_name", - "model_info", - "proxy_server_request", - "preset_cache_key", - "caching_groups", - "ttl", - "cache", - "no-log", - "base_model", - "stream_timeout", - "supports_system_message", - "region_name", - "allowed_model_region", - "model_config", - "fastest_response", - "cooldown_time", - ] + litellm_params = ( + all_litellm_params # use the external var., used in creating cache key as well. + ) default_params = openai_params + litellm_params non_default_params = { @@ -5205,7 +5154,7 @@ def stream_chunk_builder( response["choices"][0]["message"]["function_call"][ "arguments" ] = combined_arguments - + content_chunks = [ chunk for chunk in chunks diff --git a/litellm/tests/.litellm_cache/cache.db b/litellm/tests/.litellm_cache/cache.db new file mode 100644 index 0000000000000000000000000000000000000000..4099576493ee1e354d046b3474e8cae05abe98ff GIT binary patch literal 32768 zcmeI4U2NOd6@V#Iu_QYt^XDjT8iWxrG(km&|34`P)Nz>9m}|GTjHDPgg1o%6*-)Y? zQi-#`K++X^S>KjnuLA~b7_dDT$kWi5KBe8$fUUrWVJLBBJWk`!&Zvl9bu z0B1b}B$Ahhhv$6v=t%cmQp>N;2MOl}S+??IFd!#eTr7GRG$#PI!lZKK{?~m&W>}85SS_1b_e#00KY& z2mpaS5x8%RS5B+->f0BSUa-1~d&@XUg3fAe$HBL-?coq7c-BE{cy`H}X<1xrY3|xJ zYl+)Qj+?pKB8RS6^A?d@T;Q@3xvPsyoHaB13b(X)ox5yZn_J*kUb~ViE0l1{R*Spk zgni6?b+G2S#rZ4KsS(`uSF9zAyNU0RgN?MM@e*e(T)96xmYZ(vXnMcvIeRfZmm8Qq zuxo#&zsN0E*K^bN;Du3pTn7Z5826GVcR(&T@M3OZ*;;Dtvjc{_*sTMUYUQ*}V&^NL z?*J6_JIP*xXISuyjUI7(*kl(P#EnvUtl4`&o}2wNnD1_OYjm{E*GqBG@$h;U5O$nI zJ!~gIJO6}M2OrV7g)7#L{IfSCvK0>Jx6Zq8{m^LrV!d>GNGo3N2E%GAJE$F!8PYq{ zdSR(NT7RKl>JKRZ+rS80wEv(1b_e# z00KY&2mk>f00e*l5C8%|;D8gDV17p(Vcw&d_n2QXkC^w_zp;N{AF@ATf56^j*VtEC zg+0U4%%2Xp8(f00e*l5C8%|00{hl5~xkk^-?x(U#=aYr>gBBj&WRJ z>B;I^ki->+Y@~Dc#QU*YCU@$2S}l1&eADg*=w^uRZZAqA6orRQou)-PJJJq1b~!&6fww?Y7g+#M2q|sZ+Fkd~o2+sNd^2p`G3# zaN7w_*6E8QgF`62@gX~jqb5&oT_T;Zw{JP9-)}RM^tlr1hoKz??I0ODdV)SzP94hk zUnUoO;W+)Gj&B7h38IeOjluxk39%C=d=DesNy0mPc2k6R_!xb*N^WJavzq}Xj?(9< zapELFoCGKyV~>*Y|Hl+tVSm9I>{<4=%s-evGyh>fVt>khkG;$K>>~3q`zg5~EIf00a&q0fr2rw+6aL$LOh2UM4tFr6)^inSf0<(or>2E6NEI z#1m$#OmJ$KqQUcI*q&Dtw$%(yr_KLcFnI0+%@@s^1%s(NEf3{O%-Pw`bjS9PQ!Z|2uOvwnQcUoF<mfHvFIy#A|ks%xOd&qKnLCbcskdi_y3wG#kW-*jdGm zOLrS6>Lg^&gGja3f|%>{+b-^LWXrkj_rr7Kv|bbv$%YfhWa7giqRmDUMWKzHFeKOO zkhzv-!|!+U>6ElJZ`|1U1~H2mkznGGs7sPzh?-%L%afTMBGg#zCcHeY@=3qvMnrAo zzKKz4O6vSHiirC-oueU#;yZDI+hmTZL!Q^~_JY*n{L1~p6EZE;?k08;-NYSYmq`5F zQnA@Ak>(^y$ox_vC<_}K5Bk&ZF!c6lSQ8cB^nB9Kz9-49s;j=PN-{=r?a$m|{iO6jWLJg*VyE=rdOB8o zLF7eGHF<>?#k-my@E-Cs8Ifr5CG<1`;_0LZ$&?RldMIs=hBX&UBpgK7b2V4dk?%U9 zPg1f*(t_$^tox!QsFG@!NLE!vHy!dVVtRr}!UgMwo27?Y`@{7-8a`ZqQhFe=Ys1m7 zQ+m){&zA)O^VmcZuZY;>4MX>M$rVl4!;*sxf00ed+aP<(WkC*ma7BA;&_FGS-b2a;|!i`v| zQ?>n;Z?f?}`xlD+H~TO4Q?daI5C8%|00;m9AOHk_01yBIKmZ5;0U+>f6DZSEshqyi z>DvfRQ?-2j|A=D$&i;}84g2WXb`Y!w0zd!=00AHX1b_e#00KY&2mk>f@FgH{n68xa a$F%&_^489>Ocy)pb2}sBbfs22uJK>owEPtS literal 0 HcmV?d00001 diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index a4a70a535a2..b08f0039c48 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -207,11 +207,17 @@ async def test_caching_with_cache_controls(sync_flag): else: ## TTL = 0 response1 = await litellm.acompletion( - model="gpt-3.5-turbo", messages=messages, cache={"ttl": 0} + model="gpt-3.5-turbo", + messages=messages, + cache={"ttl": 0}, + mock_response="Hello world", ) await asyncio.sleep(10) response2 = await litellm.acompletion( - model="gpt-3.5-turbo", messages=messages, cache={"s-maxage": 10} + model="gpt-3.5-turbo", + messages=messages, + cache={"s-maxage": 10}, + mock_response="Hello world", ) assert response2["id"] != response1["id"] @@ -220,21 +226,33 @@ async def test_caching_with_cache_controls(sync_flag): ## TTL = 5 if sync_flag: response1 = completion( - model="gpt-3.5-turbo", messages=messages, cache={"ttl": 5} + model="gpt-3.5-turbo", + messages=messages, + cache={"ttl": 5}, + mock_response="Hello world", ) response2 = completion( - model="gpt-3.5-turbo", messages=messages, cache={"s-maxage": 5} + model="gpt-3.5-turbo", + messages=messages, + cache={"s-maxage": 5}, + mock_response="Hello world", ) print(f"response1: {response1}") print(f"response2: {response2}") assert response2["id"] == response1["id"] else: response1 = await litellm.acompletion( - model="gpt-3.5-turbo", messages=messages, cache={"ttl": 25} + model="gpt-3.5-turbo", + messages=messages, + cache={"ttl": 25}, + mock_response="Hello world", ) await asyncio.sleep(10) response2 = await litellm.acompletion( - model="gpt-3.5-turbo", messages=messages, cache={"s-maxage": 25} + model="gpt-3.5-turbo", + messages=messages, + cache={"s-maxage": 25}, + mock_response="Hello world", ) print(f"response1: {response1}") print(f"response2: {response2}") @@ -282,6 +300,61 @@ def test_caching_with_models_v2(): # test_caching_with_models_v2() + +def test_caching_with_optional_params(): + litellm.enable_caching_on_optional_params = True + messages = [ + {"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"} + ] + litellm.cache = Cache() + print("test2 for caching") + litellm.set_verbose = True + + response1 = completion( + model="gpt-3.5-turbo", + messages=messages, + top_k=10, + caching=True, + mock_response="Hello: {}".format(uuid.uuid4()), + ) + response2 = completion( + model="gpt-3.5-turbo", + messages=messages, + top_k=10, + caching=True, + mock_response="Hello: {}".format(uuid.uuid4()), + ) + response3 = completion( + model="gpt-3.5-turbo", + messages=messages, + top_k=9, + caching=True, + mock_response="Hello: {}".format(uuid.uuid4()), + ) + print(f"response1: {response1}") + print(f"response2: {response2}") + print(f"response3: {response3}") + litellm.cache = None + litellm.success_callback = [] + litellm._async_success_callback = [] + if ( + response3["choices"][0]["message"]["content"] + == response2["choices"][0]["message"]["content"] + ): + # if models are different, it should not return cached response + print(f"response2: {response2}") + print(f"response3: {response3}") + pytest.fail(f"Error occurred:") + if ( + response1["choices"][0]["message"]["content"] + != response2["choices"][0]["message"]["content"] + ): + print(f"response1: {response1}") + print(f"response2: {response2}") + pytest.fail(f"Error occurred:") + litellm.enable_caching_on_optional_params = False + + embedding_large_text = ( """ small text @@ -1347,7 +1420,7 @@ def test_get_cache_key(): "litellm_logging_obj": {}, } ) - cache_key_str = "model: gpt-3.5-turbomessages: [{'role': 'user', 'content': 'write a one sentence poem about: 7510'}]temperature: 0.2max_tokens: 40" + cache_key_str = "model: gpt-3.5-turbomessages: [{'role': 'user', 'content': 'write a one sentence poem about: 7510'}]max_tokens: 40temperature: 0.2stream: True" hash_object = hashlib.sha256(cache_key_str.encode()) # Hexadecimal representation of the hash hash_hex = hash_object.hexdigest() diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 481f762eefa..7f734482cf7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1052,6 +1052,68 @@ class ResponseFormatChunk(TypedDict, total=False): response_schema: dict +all_litellm_params = [ + "metadata", + "tags", + "acompletion", + "atext_completion", + "text_completion", + "caching", + "mock_response", + "api_key", + "api_version", + "api_base", + "force_timeout", + "logger_fn", + "verbose", + "custom_llm_provider", + "litellm_logging_obj", + "litellm_call_id", + "use_client", + "id", + "fallbacks", + "azure", + "headers", + "model_list", + "num_retries", + "context_window_fallback_dict", + "retry_policy", + "roles", + "final_prompt_value", + "bos_token", + "eos_token", + "request_timeout", + "complete_response", + "self", + "client", + "rpm", + "tpm", + "max_parallel_requests", + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_second", + "output_cost_per_second", + "hf_model_name", + "model_info", + "proxy_server_request", + "preset_cache_key", + "caching_groups", + "ttl", + "cache", + "no-log", + "base_model", + "stream_timeout", + "supports_system_message", + "region_name", + "allowed_model_region", + "model_config", + "fastest_response", + "cooldown_time", + "cache_key", + "max_retries", +] + + class LoggedLiteLLMParams(TypedDict, total=False): force_timeout: Optional[float] custom_llm_provider: Optional[str] diff --git a/litellm/utils.py b/litellm/utils.py index 825caf326d6..5948543bde2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1084,7 +1084,7 @@ def client(original_function): and str(original_function.__name__) in litellm.cache.supported_call_types ): - print_verbose(f"Checking Cache") + print_verbose("Checking Cache") if call_type == CallTypes.aembedding.value and isinstance( kwargs["input"], list ): From a9fdfb5a99fb23169070f547e2c36b93c6c3a826 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 5 Aug 2024 11:23:20 -0700 Subject: [PATCH 02/17] fix(init.py): rename feature_flag --- litellm/__init__.py | 2 +- litellm/caching.py | 2 +- litellm/tests/test_caching.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 22255eb34a2..bf084e20118 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -146,7 +146,7 @@ return_response_headers: bool = ( ) ################## logging: bool = True -enable_caching_on_optional_params: bool = ( +enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) caching: bool = ( diff --git a/litellm/caching.py b/litellm/caching.py index ab62c344064..6ed6eb9fdbb 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -1903,7 +1903,7 @@ class Cache: param not in litellm_param_kwargs ): # check if user passed in optional param - e.g. top_k if ( - litellm.enable_caching_on_optional_params is True + litellm.enable_caching_on_provider_specific_optional_params is True ): # feature flagged for now if kwargs[param] is None: continue # ignore None params diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index b08f0039c48..a5579560680 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -302,7 +302,7 @@ def test_caching_with_models_v2(): def test_caching_with_optional_params(): - litellm.enable_caching_on_optional_params = True + litellm.enable_caching_on_provider_specific_optional_params = True messages = [ {"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"} ] @@ -352,7 +352,7 @@ def test_caching_with_optional_params(): print(f"response1: {response1}") print(f"response2: {response2}") pytest.fail(f"Error occurred:") - litellm.enable_caching_on_optional_params = False + litellm.enable_caching_on_provider_specific_optional_params = False embedding_large_text = ( From aef25d5d0011d2e7721accb772c3fa7b94d33b26 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 5 Aug 2024 11:23:49 -0700 Subject: [PATCH 03/17] fix: cleanup test --- litellm/tests/test_caching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index a5579560680..ee8b0535401 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -301,7 +301,7 @@ def test_caching_with_models_v2(): # test_caching_with_models_v2() -def test_caching_with_optional_params(): +def c(): litellm.enable_caching_on_provider_specific_optional_params = True messages = [ {"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"} From 5c3afc3aba4e53a9811f387de599aa45dffd3207 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 10:12:34 -0700 Subject: [PATCH 04/17] add get_request_route --- litellm/proxy/auth/auth_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index f9be71c35da..aa48a6396ab 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -80,6 +80,19 @@ def is_llm_api_route(route: str) -> bool: return False +def get_request_route(request: Request) -> str: + """ + Helper to get the route from the request + + remove base url from path if set e.g. `/genai/chat/completions` -> `/chat/completions + """ + if request.url.path.startswith(request.base_url.path): + # remove base_url from path + return request.url.path[len(request.base_url.path) - 1 :] + else: + return request.url.path + + async def check_if_request_size_is_safe(request: Request) -> bool: """ Enterprise Only: From 35dde9d2a8011bbc851cf4932e333489b3a141f1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 10:13:47 -0700 Subject: [PATCH 05/17] use get_request_route --- litellm/proxy/auth/user_api_key_auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index b8be226050d..1192490422a 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_checks import ( ) from litellm.proxy.auth.auth_utils import ( check_if_request_size_is_safe, + get_request_route, is_llm_api_route, route_in_additonal_public_routes, ) @@ -115,7 +116,7 @@ async def user_api_key_auth( ) try: - route: str = request.url.path + route: str = get_request_route(request=request) ### LiteLLM Enterprise Security Checks # Check 1. Check if request size is under max_request_size_mb From 533ffd6020a8b8245d0698e6dd1c7b8be4ff31c0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 10:14:23 -0700 Subject: [PATCH 06/17] test proxy server routes --- litellm/tests/test_proxy_routes.py | 53 +++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_proxy_routes.py b/litellm/tests/test_proxy_routes.py index 03f112a5e22..90fda07a114 100644 --- a/litellm/tests/test_proxy_routes.py +++ b/litellm/tests/test_proxy_routes.py @@ -16,10 +16,12 @@ import asyncio import logging import pytest +from fastapi import Request +from starlette.datastructures import URL, Headers, QueryParams import litellm from litellm.proxy._types import LiteLLMRoutes -from litellm.proxy.auth.auth_utils import is_llm_api_route +from litellm.proxy.auth.auth_utils import get_request_route, is_llm_api_route from litellm.proxy.proxy_server import app # Configure logging @@ -98,3 +100,52 @@ def test_is_llm_api_route_similar_but_false(route: str): def test_anthropic_api_routes(): # allow non proxy admins to call anthropic api routes assert is_llm_api_route(route="/v1/messages") is True + + +def create_request(path: str, base_url: str = "http://testserver") -> Request: + return Request( + { + "type": "http", + "method": "GET", + "scheme": "http", + "server": ("testserver", 80), + "path": path, + "query_string": b"", + "headers": Headers().raw, + "client": ("testclient", 50000), + "root_path": URL(base_url).path, + } + ) + + +def test_get_request_route_with_base_url(): + request = create_request( + path="/genai/chat/completions", base_url="http://testserver/genai" + ) + result = get_request_route(request) + assert result == "/chat/completions" + + +def test_get_request_route_without_base_url(): + request = create_request("/chat/completions") + result = get_request_route(request) + assert result == "/chat/completions" + + +def test_get_request_route_with_nested_path(): + request = create_request(path="/embeddings", base_url="http://testserver/ishaan") + result = get_request_route(request) + assert result == "/embeddings" + + +def test_get_request_route_with_query_params(): + request = create_request(path="/genai/test", base_url="http://testserver/genai") + request.scope["query_string"] = b"param=value" + result = get_request_route(request) + assert result == "/test" + + +def test_get_request_route_with_base_url_not_at_start(): + request = create_request("/api/genai/test") + result = get_request_route(request) + assert result == "/api/genai/test" From 2eea29ecb3f14106e9d43c268928db028dbadea3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 10:33:40 -0700 Subject: [PATCH 07/17] fix get_request_route --- litellm/proxy/auth/auth_utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index aa48a6396ab..d1e1b170983 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -86,10 +86,16 @@ def get_request_route(request: Request) -> str: remove base url from path if set e.g. `/genai/chat/completions` -> `/chat/completions """ - if request.url.path.startswith(request.base_url.path): - # remove base_url from path - return request.url.path[len(request.base_url.path) - 1 :] - else: + try: + if request.url.path.startswith(request.base_url.path): + # remove base_url from path + return request.url.path[len(request.base_url.path) - 1 :] + else: + return request.url.path + except Exception as e: + verbose_proxy_logger.warning( + f"error on get_request_route: {str(e)}, defaulting to request.url.path" + ) return request.url.path From 28a37c0069b2f58736ffbcbf491c8438395d62ad Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 11:08:13 -0700 Subject: [PATCH 08/17] fix test fine tuning api azure --- litellm/tests/test_fine_tuning_api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index 412ffb497c1..436adedbde5 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -158,8 +158,7 @@ async def test_azure_create_fine_tune_jobs_async(): model="gpt-35-turbo-1106", training_file=file_id, custom_llm_provider="azure", - api_key=os.getenv("AZURE_SWEDEN_API_KEY"), - api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", + api_base="https://exampleopenaiendpoint-production.up.railway.app", ) print( @@ -167,15 +166,16 @@ async def test_azure_create_fine_tune_jobs_async(): ) assert create_fine_tuning_response.id is not None - assert create_fine_tuning_response.model == "gpt-35-turbo-1106" + + # response from Example/mocked endpoint + assert create_fine_tuning_response.model == "davinci-002" # list fine tuning jobs print("listing ft jobs") ft_jobs = await litellm.alist_fine_tuning_jobs( limit=2, custom_llm_provider="azure", - api_key=os.getenv("AZURE_SWEDEN_API_KEY"), - api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", + api_base="https://exampleopenaiendpoint-production.up.railway.app", ) print("response from litellm.list_fine_tuning_jobs=", ft_jobs) @@ -184,7 +184,7 @@ async def test_azure_create_fine_tune_jobs_async(): fine_tuning_job_id=create_fine_tuning_response.id, custom_llm_provider="azure", api_key=os.getenv("AZURE_SWEDEN_API_KEY"), - api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", + api_base="https://exampleopenaiendpoint-production.up.railway.app", ) print("response from litellm.cancel_fine_tuning_job=", response) From b02f8aa53fe25fb7c9f92274db297b4d1d1565bd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Jun 2024 18:45:07 -0700 Subject: [PATCH 09/17] fix allow setting UI _BASE path --- ui/litellm-dashboard/next.config.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/next.config.mjs b/ui/litellm-dashboard/next.config.mjs index e1f8aa083ee..6e2924677c8 100644 --- a/ui/litellm-dashboard/next.config.mjs +++ b/ui/litellm-dashboard/next.config.mjs @@ -1,7 +1,7 @@ /** @type {import('next').NextConfig} */ const nextConfig = { output: 'export', - basePath: '/ui', + basePath: process.env.UI_BASE_PATH || '/ui', }; nextConfig.experimental = { From 13c4f13c6abb7a2af25f58ebd526767236d117a8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 11 Jun 2024 18:59:42 -0700 Subject: [PATCH 10/17] fix edit docker file ui base path --- build_admin_ui.sh | 110 +++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 49 deletions(-) diff --git a/build_admin_ui.sh b/build_admin_ui.sh index 5373ad0e3d9..897449023d2 100755 --- a/build_admin_ui.sh +++ b/build_admin_ui.sh @@ -7,56 +7,68 @@ echo pwd +# if UI_BASE_PATH env is set +if [ -z "$UI_BASE_PATH" ]; then -# only run this step for litellm enterprise, we run this if enterprise/enterprise_ui/_enterprise.json exists -if [ ! -f "enterprise/enterprise_ui/enterprise_colors.json" ]; then + +# only run this step for litellm enterprise, we run this if enterprise/enterprise_ui/_enterprise.json exists or env var UI_BASE_PATH is set +if [ -f "enterprise/enterprise_ui/enterprise_colors.json" ] || [ -n "${UI_BASE_PATH:-}" ]; then + echo "Building Admin UI..." + + # Install dependencies + # Check if we are on macOS + if [[ "$(uname)" == "Darwin" ]]; then + # Install dependencies using Homebrew + if ! command -v brew &> /dev/null; then + echo "Error: Homebrew not found. Please install Homebrew and try again." + exit 1 + fi + brew update + brew install curl + else + # Assume Linux, try using apt-get + if command -v apt-get &> /dev/null; then + apt-get update + apt-get install -y curl + elif command -v apk &> /dev/null; then + # Try using apk if apt-get is not available + apk update + apk add curl + else + echo "Error: Unsupported package manager. Cannot install dependencies." + exit 1 + fi + fi + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash + source ~/.nvm/nvm.sh + nvm install v18.17.0 + nvm use v18.17.0 + npm install -g npm + + if [ -n "${UI_BASE_PATH:-}" ]; then + echo "Using UI_BASE_PATH: $UI_BASE_PATH" + + # make a file call .env in ui/litellm-dashboard and store the UI_BASE_PATH in it + echo "UI_BASE_PATH=$UI_BASE_PATH" > ui/litellm-dashboard/.env + + fi + + # copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json + cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json + + # cd in to /ui/litellm-dashboard + cd ui/litellm-dashboard + + # ensure have access to build_ui.sh + chmod +x ./build_ui.sh + + # run ./build_ui.sh + ./build_ui.sh + + # return to root directory + cd ../.. + +else echo "Admin UI - using default LiteLLM UI" exit 0 fi - -echo "Building Custom Admin UI..." - -# Install dependencies -# Check if we are on macOS -if [[ "$(uname)" == "Darwin" ]]; then - # Install dependencies using Homebrew - if ! command -v brew &> /dev/null; then - echo "Error: Homebrew not found. Please install Homebrew and try again." - exit 1 - fi - brew update - brew install curl -else - # Assume Linux, try using apt-get - if command -v apt-get &> /dev/null; then - apt-get update - apt-get install -y curl - elif command -v apk &> /dev/null; then - # Try using apk if apt-get is not available - apk update - apk add curl - else - echo "Error: Unsupported package manager. Cannot install dependencies." - exit 1 - fi -fi -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash -source ~/.nvm/nvm.sh -nvm install v18.17.0 -nvm use v18.17.0 -npm install -g npm - -# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json -cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json - -# cd in to /ui/litellm-dashboard -cd ui/litellm-dashboard - -# ensure have access to build_ui.sh -chmod +x ./build_ui.sh - -# run ./build_ui.sh -./build_ui.sh - -# return to root directory -cd ../.. \ No newline at end of file From 68f245c96482c94d9c6d384b291a2eb2f8602b07 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 10:12:34 -0700 Subject: [PATCH 11/17] add get_request_route --- litellm/proxy/auth/auth_utils.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index d1e1b170983..aa48a6396ab 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -86,16 +86,10 @@ def get_request_route(request: Request) -> str: remove base url from path if set e.g. `/genai/chat/completions` -> `/chat/completions """ - try: - if request.url.path.startswith(request.base_url.path): - # remove base_url from path - return request.url.path[len(request.base_url.path) - 1 :] - else: - return request.url.path - except Exception as e: - verbose_proxy_logger.warning( - f"error on get_request_route: {str(e)}, defaulting to request.url.path" - ) + if request.url.path.startswith(request.base_url.path): + # remove base_url from path + return request.url.path[len(request.base_url.path) - 1 :] + else: return request.url.path From 775ef8a786fe50a19107ee9eec21d7251ce89841 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 10:33:40 -0700 Subject: [PATCH 12/17] fix get_request_route --- litellm/proxy/auth/auth_utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index aa48a6396ab..d1e1b170983 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -86,10 +86,16 @@ def get_request_route(request: Request) -> str: remove base url from path if set e.g. `/genai/chat/completions` -> `/chat/completions """ - if request.url.path.startswith(request.base_url.path): - # remove base_url from path - return request.url.path[len(request.base_url.path) - 1 :] - else: + try: + if request.url.path.startswith(request.base_url.path): + # remove base_url from path + return request.url.path[len(request.base_url.path) - 1 :] + else: + return request.url.path + except Exception as e: + verbose_proxy_logger.warning( + f"error on get_request_route: {str(e)}, defaulting to request.url.path" + ) return request.url.path From 36ed40d555c33d51da7dbaf1e6c021e685056ac6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 15:22:03 -0700 Subject: [PATCH 13/17] Revert "[FIX] allow setting UI BASE path" --- build_admin_ui.sh | 110 ++++++++++++--------------- ui/litellm-dashboard/next.config.mjs | 2 +- 2 files changed, 50 insertions(+), 62 deletions(-) diff --git a/build_admin_ui.sh b/build_admin_ui.sh index 897449023d2..5373ad0e3d9 100755 --- a/build_admin_ui.sh +++ b/build_admin_ui.sh @@ -7,68 +7,56 @@ echo pwd -# if UI_BASE_PATH env is set -if [ -z "$UI_BASE_PATH" ]; then - -# only run this step for litellm enterprise, we run this if enterprise/enterprise_ui/_enterprise.json exists or env var UI_BASE_PATH is set -if [ -f "enterprise/enterprise_ui/enterprise_colors.json" ] || [ -n "${UI_BASE_PATH:-}" ]; then - echo "Building Admin UI..." - - # Install dependencies - # Check if we are on macOS - if [[ "$(uname)" == "Darwin" ]]; then - # Install dependencies using Homebrew - if ! command -v brew &> /dev/null; then - echo "Error: Homebrew not found. Please install Homebrew and try again." - exit 1 - fi - brew update - brew install curl - else - # Assume Linux, try using apt-get - if command -v apt-get &> /dev/null; then - apt-get update - apt-get install -y curl - elif command -v apk &> /dev/null; then - # Try using apk if apt-get is not available - apk update - apk add curl - else - echo "Error: Unsupported package manager. Cannot install dependencies." - exit 1 - fi - fi - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash - source ~/.nvm/nvm.sh - nvm install v18.17.0 - nvm use v18.17.0 - npm install -g npm - - if [ -n "${UI_BASE_PATH:-}" ]; then - echo "Using UI_BASE_PATH: $UI_BASE_PATH" - - # make a file call .env in ui/litellm-dashboard and store the UI_BASE_PATH in it - echo "UI_BASE_PATH=$UI_BASE_PATH" > ui/litellm-dashboard/.env - - fi - - # copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json - cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json - - # cd in to /ui/litellm-dashboard - cd ui/litellm-dashboard - - # ensure have access to build_ui.sh - chmod +x ./build_ui.sh - - # run ./build_ui.sh - ./build_ui.sh - - # return to root directory - cd ../.. - -else +# only run this step for litellm enterprise, we run this if enterprise/enterprise_ui/_enterprise.json exists +if [ ! -f "enterprise/enterprise_ui/enterprise_colors.json" ]; then echo "Admin UI - using default LiteLLM UI" exit 0 fi + +echo "Building Custom Admin UI..." + +# Install dependencies +# Check if we are on macOS +if [[ "$(uname)" == "Darwin" ]]; then + # Install dependencies using Homebrew + if ! command -v brew &> /dev/null; then + echo "Error: Homebrew not found. Please install Homebrew and try again." + exit 1 + fi + brew update + brew install curl +else + # Assume Linux, try using apt-get + if command -v apt-get &> /dev/null; then + apt-get update + apt-get install -y curl + elif command -v apk &> /dev/null; then + # Try using apk if apt-get is not available + apk update + apk add curl + else + echo "Error: Unsupported package manager. Cannot install dependencies." + exit 1 + fi +fi +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +source ~/.nvm/nvm.sh +nvm install v18.17.0 +nvm use v18.17.0 +npm install -g npm + +# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json +cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json + +# cd in to /ui/litellm-dashboard +cd ui/litellm-dashboard + +# ensure have access to build_ui.sh +chmod +x ./build_ui.sh + +# run ./build_ui.sh +./build_ui.sh + +# return to root directory +cd ../.. \ No newline at end of file diff --git a/ui/litellm-dashboard/next.config.mjs b/ui/litellm-dashboard/next.config.mjs index 6e2924677c8..e1f8aa083ee 100644 --- a/ui/litellm-dashboard/next.config.mjs +++ b/ui/litellm-dashboard/next.config.mjs @@ -1,7 +1,7 @@ /** @type {import('next').NextConfig} */ const nextConfig = { output: 'export', - basePath: process.env.UI_BASE_PATH || '/ui', + basePath: '/ui', }; nextConfig.experimental = { From b7826ed7c376e5acb6e5cca4b7e4b1bfae26ea1d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 15:38:13 -0700 Subject: [PATCH 14/17] working sh script --- ui/litellm-dashboard/build_ui_custom_path.sh | 59 ++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 ui/litellm-dashboard/build_ui_custom_path.sh diff --git a/ui/litellm-dashboard/build_ui_custom_path.sh b/ui/litellm-dashboard/build_ui_custom_path.sh new file mode 100644 index 00000000000..11b054b593c --- /dev/null +++ b/ui/litellm-dashboard/build_ui_custom_path.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Check if BASE_UI_PATH argument is provided +if [ -z "$1" ]; then + echo "Error: BASE_UI_PATH argument is required." + echo "Usage: $0 " + exit 1 +fi + +# Set BASE_UI_PATH from the first argument +BASE_UI_PATH="$1" + +# Check if nvm is not installed +if ! command -v nvm &> /dev/null; then + # Install nvm + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash + + # Source nvm script in the current session + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +fi + +# Use nvm to set the required Node.js version +nvm use v18.17.0 + +# Check if nvm use was successful +if [ $? -ne 0 ]; then + echo "Error: Failed to switch to Node.js v18.17.0. Deployment aborted." + exit 1 +fi + +# print contents of ui_colors.json +echo "Contents of ui_colors.json:" +cat ui_colors.json + +# Run npm build with the environment variable +BASE_UI_PATH=$BASE_UI_PATH npm run build + +# Check if the build was successful +if [ $? -eq 0 ]; then + echo "Build successful. Copying files..." + + # echo current dir + echo + pwd + + # Specify the destination directory + destination_dir="../../litellm/proxy/_experimental/out" + + # Remove existing files in the destination directory + rm -rf "$destination_dir"/* + + # Copy the contents of the output directory to the specified destination + cp -r ./out/* "$destination_dir" + + echo "Deployment completed." +else + echo "Build failed. Deployment aborted." +fi \ No newline at end of file From 3b8fc93091b2a3dbdf71a1887444f032dd97a364 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 15:48:44 -0700 Subject: [PATCH 15/17] set PROXY_BASE_URL when server root path set --- litellm/proxy/proxy_server.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a9b49138b9c..17b23a361f6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -281,9 +281,12 @@ except Exception as e: except Exception as e: pass +server_root_path = os.getenv("SERVER_ROOT_PATH", "") +if server_root_path != "" and os.getenv("PROXY_BASE_URL") is None: + os.environ["PROXY_BASE_URL"] = server_root_path _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -ui_link = f"/ui/" +ui_link = f"{server_root_path}/ui/" ui_message = ( f"👉 [```LiteLLM Admin Panel on /ui```]({ui_link}). Create, Edit Keys with SSO" ) @@ -303,14 +306,13 @@ _description = ( else f"Proxy Server to call 100+ LLMs in the OpenAI format. {custom_swagger_message}\n\n{ui_message}" ) + app = FastAPI( docs_url=_docs_url, title=_title, description=_description, version=version, - root_path=os.environ.get( - "SERVER_ROOT_PATH", "" - ), # check if user passed root path, FastAPI defaults this value to "" + root_path=server_root_path, # check if user passed root path, FastAPI defaults this value to "" ) From beefd221d3c30f5dc6d43d064818c139daeceee9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 15:59:50 -0700 Subject: [PATCH 16/17] use correct build paths --- ui/litellm-dashboard/build_ui_custom_path.sh | 12 ++++++------ ui/litellm-dashboard/next.config.mjs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) mode change 100644 => 100755 ui/litellm-dashboard/build_ui_custom_path.sh diff --git a/ui/litellm-dashboard/build_ui_custom_path.sh b/ui/litellm-dashboard/build_ui_custom_path.sh old mode 100644 new mode 100755 index 11b054b593c..ceb8c733d37 --- a/ui/litellm-dashboard/build_ui_custom_path.sh +++ b/ui/litellm-dashboard/build_ui_custom_path.sh @@ -1,14 +1,14 @@ #!/bin/bash -# Check if BASE_UI_PATH argument is provided +# Check if UI_BASE_PATH argument is provided if [ -z "$1" ]; then - echo "Error: BASE_UI_PATH argument is required." - echo "Usage: $0 " + echo "Error: UI_BASE_PATH argument is required." + echo "Usage: $0 " exit 1 fi -# Set BASE_UI_PATH from the first argument -BASE_UI_PATH="$1" +# Set UI_BASE_PATH from the first argument +UI_BASE_PATH="$1" # Check if nvm is not installed if ! command -v nvm &> /dev/null; then @@ -34,7 +34,7 @@ echo "Contents of ui_colors.json:" cat ui_colors.json # Run npm build with the environment variable -BASE_UI_PATH=$BASE_UI_PATH npm run build +UI_BASE_PATH=$UI_BASE_PATH npm run build # Check if the build was successful if [ $? -eq 0 ]; then diff --git a/ui/litellm-dashboard/next.config.mjs b/ui/litellm-dashboard/next.config.mjs index e1f8aa083ee..6e2924677c8 100644 --- a/ui/litellm-dashboard/next.config.mjs +++ b/ui/litellm-dashboard/next.config.mjs @@ -1,7 +1,7 @@ /** @type {import('next').NextConfig} */ const nextConfig = { output: 'export', - basePath: '/ui', + basePath: process.env.UI_BASE_PATH || '/ui', }; nextConfig.experimental = { From 2e8a2548e1fe637814122f890331427a4746b18f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 5 Aug 2024 16:34:37 -0700 Subject: [PATCH 17/17] build ui on custom path --- litellm/proxy/common_utils/admin_ui_utils.py | 65 ++++++++++++++++++++ litellm/proxy/proxy_server.py | 5 +- ui/litellm-dashboard/build_ui_custom_path.sh | 4 -- 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_utils/admin_ui_utils.py b/litellm/proxy/common_utils/admin_ui_utils.py index bb35ecd6914..3389f723d88 100644 --- a/litellm/proxy/common_utils/admin_ui_utils.py +++ b/litellm/proxy/common_utils/admin_ui_utils.py @@ -1,4 +1,5 @@ import os +import subprocess def show_missing_vars_in_env(): @@ -165,3 +166,67 @@ def missing_keys_form(missing_key_names: str): """ return missing_keys_html_form.format(missing_keys=missing_key_names) + + +def setup_admin_ui_on_server_root_path(): + """ + Helper util to setup Admin UI on Server root path + """ + from litellm._logging import verbose_proxy_logger + + server_root_path = os.getenv("SERVER_ROOT_PATH", "") + if server_root_path != "": + if os.getenv("PROXY_BASE_URL") is None: + os.environ["PROXY_BASE_URL"] = server_root_path + + # re-build admin UI on server root path + # Save the original directory + original_dir = os.getcwd() + + current_dir = ( + os.path.dirname(os.path.abspath(__file__)) + + "/../../../ui/litellm-dashboard/" + ) + build_ui_path = os.path.join(current_dir, "build_ui_custom_path.sh") + package_path = os.path.join(current_dir, "package.json") + + verbose_proxy_logger.debug( + f"Setting up Admin UI on {server_root_path}/ui ......." + ) # noqa + + try: + # Change the current working directory + os.chdir(current_dir) + + # Make the script executable + subprocess.run(["chmod", "+x", "build_ui_custom_path.sh"], check=True) + + # Run npm install + subprocess.run(["npm", "install"], check=True) + + # Run npm run build + subprocess.run(["npm", "run", "build"], check=True) + + # Run the custom build script with the argument + subprocess.run( + ["./build_ui_custom_path.sh", f"{server_root_path}/ui"], check=True + ) + + verbose_proxy_logger.debug("Admin UI setup completed successfully.") # noqa + + except subprocess.CalledProcessError as e: + verbose_proxy_logger.debug( + f"An error occurred during the Admin UI setup: {e}" + ) # noqa + + except Exception as e: + verbose_proxy_logger.debug(f"An unexpected error occurred: {e}") + + finally: + # Always return to the original directory, even if an error occurred + os.chdir(original_dir) + verbose_proxy_logger.debug( + f"Returned to original directory: {original_dir}" + ) # noqa + + pass diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 17b23a361f6..260c728b50c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -138,6 +138,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.caching_routes import router as caching_router from litellm.proxy.common_utils.admin_ui_utils import ( html_form, + setup_admin_ui_on_server_root_path, show_missing_vars_in_env, ) from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router @@ -282,8 +283,8 @@ except Exception as e: pass server_root_path = os.getenv("SERVER_ROOT_PATH", "") -if server_root_path != "" and os.getenv("PROXY_BASE_URL") is None: - os.environ["PROXY_BASE_URL"] = server_root_path +if server_root_path != "": + setup_admin_ui_on_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() ui_link = f"{server_root_path}/ui/" diff --git a/ui/litellm-dashboard/build_ui_custom_path.sh b/ui/litellm-dashboard/build_ui_custom_path.sh index ceb8c733d37..f947f87d3b7 100755 --- a/ui/litellm-dashboard/build_ui_custom_path.sh +++ b/ui/litellm-dashboard/build_ui_custom_path.sh @@ -29,10 +29,6 @@ if [ $? -ne 0 ]; then exit 1 fi -# print contents of ui_colors.json -echo "Contents of ui_colors.json:" -cat ui_colors.json - # Run npm build with the environment variable UI_BASE_PATH=$UI_BASE_PATH npm run build