scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-11 03:38:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

Revert "Litellm dev 10 29 2024 (#6502)"

Browse source 
This reverts commit 1e403a8447.
This commit is contained in:
Ishaan Jaff 2024-10-31 00:06:39 +05:30
parent 2c37aad1c4
commit ac24f87e87
14 changed files with 51 additions and 303 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -284,7 +284,9 @@ Output from script
:::info
Customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm)
Customer This is the value of `user_id` passed when calling [`/key/generate`](https://litellm-api.up.railway.app/#/key%20management/generate_key_fn_key_generate_post)
[this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm)
- [LiteLLM API key](virtual_keys.md)

8
litellm/caching/base_cache.py
View file

@ -23,12 +23,8 @@ class BaseCache:
self.default_ttl = default_ttl
def get_ttl(self, **kwargs) -> Optional[int]:
kwargs_ttl: Optional[int] = kwargs.get("ttl")
if kwargs_ttl is not None:
try:
return int(kwargs_ttl)
except ValueError:
return self.default_ttl
if kwargs.get("ttl") is not None:
return kwargs.get("ttl")
return self.default_ttl
def set_cache(self, key, value, **kwargs):

1
litellm/caching/redis_cache.py
View file

@ -301,7 +301,6 @@ class RedisCache(BaseCache):
print_verbose(
f"Set ASYNC Redis Cache: key: {key}\nValue {value}\nttl={ttl}"
)
try:
if not hasattr(redis_client, "set"):
raise Exception(

36
litellm/integrations/prometheus.py
View file

@ -849,13 +849,9 @@ class PrometheusLogger(CustomLogger):
):
try:
verbose_logger.debug("setting remaining tokens requests metric")
standard_logging_payload: Optional[StandardLoggingPayload] = (
request_kwargs.get("standard_logging_object")
standard_logging_payload: StandardLoggingPayload = request_kwargs.get(
"standard_logging_object", {}
)
if standard_logging_payload is None:
return
model_group = standard_logging_payload["model_group"]
api_base = standard_logging_payload["api_base"]
_response_headers = request_kwargs.get("response_headers")
@ -866,18 +862,22 @@ class PrometheusLogger(CustomLogger):
_model_info = _metadata.get("model_info") or {}
model_id = _model_info.get("id", None)
remaining_requests: Optional[int] = None
remaining_tokens: Optional[int] = None
if additional_headers := standard_logging_payload["hidden_params"][
"additional_headers"
]:
# OpenAI / OpenAI Compatible headers
remaining_requests = additional_headers.get(
"x_ratelimit_remaining_requests", None
)
remaining_tokens = additional_headers.get(
"x_ratelimit_remaining_tokens", None
)
remaining_requests = None
remaining_tokens = None
# OpenAI / OpenAI Compatible headers
if (
_response_headers
and "x-ratelimit-remaining-requests" in _response_headers
):
remaining_requests = _response_headers["x-ratelimit-remaining-requests"]
if (
_response_headers
and "x-ratelimit-remaining-tokens" in _response_headers
):
remaining_tokens = _response_headers["x-ratelimit-remaining-tokens"]
verbose_logger.debug(
f"remaining requests: {remaining_requests}, remaining tokens: {remaining_tokens}"
)
if remaining_requests:
"""

2
litellm/litellm_core_utils/core_helpers.py
View file

@ -80,7 +80,7 @@ def _get_parent_otel_span_from_kwargs(
) -> Union[Span, None]:
try:
if kwargs is None:
return None
raise ValueError("kwargs is None")
litellm_params = kwargs.get("litellm_params")
_metadata = kwargs.get("metadata") or {}
if "litellm_parent_otel_span" in _metadata:

67
litellm/litellm_core_utils/litellm_logging.py
View file

@ -42,7 +42,6 @@ from litellm.types.utils import (
ImageResponse,
ModelResponse,
StandardCallbackDynamicParams,
StandardLoggingAdditionalHeaders,
StandardLoggingHiddenParams,
StandardLoggingMetadata,
StandardLoggingModelCostFailureDebugInformation,
@ -2641,52 +2640,6 @@ class StandardLoggingPayloadSetup:
return final_response_obj
@staticmethod
def get_additional_headers(
additiona_headers: Optional[dict],
) -> Optional[StandardLoggingAdditionalHeaders]:
if additiona_headers is None:
return None
additional_logging_headers: StandardLoggingAdditionalHeaders = {}
for key in StandardLoggingAdditionalHeaders.__annotations__.keys():
_key = key.lower()
_key = _key.replace("_", "-")
if _key in additiona_headers:
try:
additional_logging_headers[key] = int(additiona_headers[_key]) # type: ignore
except (ValueError, TypeError):
verbose_logger.debug(
f"Could not convert {additiona_headers[_key]} to int for key {key}."
)
return additional_logging_headers
@staticmethod
def get_hidden_params(
hidden_params: Optional[dict],
) -> StandardLoggingHiddenParams:
clean_hidden_params = StandardLoggingHiddenParams(
model_id=None,
cache_key=None,
api_base=None,
response_cost=None,
additional_headers=None,
)
if hidden_params is not None:
for key in StandardLoggingHiddenParams.__annotations__.keys():
if key in hidden_params:
if key == "additional_headers":
clean_hidden_params["additional_headers"] = (
StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
)
)
else:
clean_hidden_params[key] = hidden_params[key] # type: ignore
return clean_hidden_params
def get_standard_logging_object_payload(
kwargs: Optional[dict],
@ -2718,9 +2671,7 @@ def get_standard_logging_object_payload(
if response_headers is not None:
hidden_params = dict(
StandardLoggingHiddenParams(
additional_headers=StandardLoggingPayloadSetup.get_additional_headers(
dict(response_headers)
),
additional_headers=dict(response_headers),
model_id=None,
cache_key=None,
api_base=None,
@ -2761,9 +2712,21 @@ def get_standard_logging_object_payload(
)
)
# clean up litellm hidden params
clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
hidden_params
clean_hidden_params = StandardLoggingHiddenParams(
model_id=None,
cache_key=None,
api_base=None,
response_cost=None,
additional_headers=None,
)
if hidden_params is not None:
clean_hidden_params = StandardLoggingHiddenParams(
**{ # type: ignore
key: hidden_params[key]
for key in StandardLoggingHiddenParams.__annotations__.keys()
if key in hidden_params
}
)
# clean up litellm metadata
clean_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(
metadata=metadata

8
litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -431,13 +431,9 @@ class VertexGeminiConfig:
elif openai_function_object is not None:
gtool_func_declaration = FunctionDeclaration(
name=openai_function_object["name"],
description=openai_function_object.get("description", ""),
parameters=openai_function_object.get("parameters", {}),
)
_description = openai_function_object.get("description", None)
_parameters = openai_function_object.get("parameters", None)
if _description is not None:
gtool_func_declaration["description"] = _description
if _parameters is not None:
gtool_func_declaration["parameters"] = _parameters
gtool_func_declarations.append(gtool_func_declaration)
else:
# assume it's a provider-specific param

2
litellm/proxy/_new_secret_config.yaml
View file

@ -17,7 +17,7 @@ model_list:
litellm_settings:
fallbacks: [{ "claude-3-5-sonnet-20240620": ["claude-3-5-sonnet-aihubmix"] }]
callbacks: ["otel", "prometheus"]
callbacks: ["otel"]
router_settings:
routing_strategy: latency-based-routing

9
litellm/types/utils.py
View file

@ -1436,19 +1436,12 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
requester_metadata: Optional[dict]
class StandardLoggingAdditionalHeaders(TypedDict, total=False):
x_ratelimit_limit_requests: int
x_ratelimit_limit_tokens: int
x_ratelimit_remaining_requests: int
x_ratelimit_remaining_tokens: int
class StandardLoggingHiddenParams(TypedDict):
model_id: Optional[str]
cache_key: Optional[str]
api_base: Optional[str]
response_cost: Optional[str]
additional_headers: Optional[StandardLoggingAdditionalHeaders]
additional_headers: Optional[dict]
class StandardLoggingModelInformation(TypedDict):

115
tests/llm_translation/test_vertex.py
View file

@ -12,9 +12,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import pytest
import litellm
from litellm import get_optional_params
def test_completion_pydantic_obj_2():
@ -118,115 +117,3 @@ def test_build_vertex_schema():
assert new_schema["type"] == schema["type"]
assert new_schema["properties"] == schema["properties"]
assert "required" in new_schema and new_schema["required"] == schema["required"]
@pytest.mark.parametrize(
"tools, key",
[
([{"googleSearchRetrieval": {}}], "googleSearchRetrieval"),
([{"code_execution": {}}], "code_execution"),
],
)
def test_vertex_tool_params(tools, key):
optional_params = get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="vertex_ai",
tools=tools,
)
print(optional_params)
assert optional_params["tools"][0][key] == {}
@pytest.mark.parametrize(
"tool, expect_parameters",
[
(
{
"name": "test_function",
"description": "test_function_description",
"parameters": {
"type": "object",
"properties": {"test_param": {"type": "string"}},
},
},
True,
),
(
{
"name": "test_function",
},
False,
),
],
)
def test_vertex_function_translation(tool, expect_parameters):
"""
If param not set, don't set it in the request
"""
tools = [tool]
optional_params = get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="vertex_ai",
tools=tools,
)
print(optional_params)
if expect_parameters:
assert "parameters" in optional_params["tools"][0]["function_declarations"][0]
else:
assert (
"parameters" not in optional_params["tools"][0]["function_declarations"][0]
)
def test_function_calling_with_gemini():
from litellm.llms.custom_httpx.http_handler import HTTPHandler
litellm.set_verbose = True
client = HTTPHandler()
with patch.object(client, "post", new=MagicMock()) as mock_post:
try:
litellm.completion(
model="gemini/gemini-1.5-pro-002",
messages=[
{
"content": [
{
"type": "text",
"text": "You are a helpful assistant that can interact with a computer to solve tasks.\n<IMPORTANT>\n* If user provides a path, you should NOT assume it's relative to the current working directory. Instead, you should explore the file system to find the file before working on it.\n</IMPORTANT>\n",
}
],
"role": "system",
},
{
"content": [{"type": "text", "text": "Hey, how's it going?"}],
"role": "user",
},
],
tools=[
{
"type": "function",
"function": {
"name": "finish",
"description": "Finish the interaction when the task is complete OR if the assistant cannot proceed further with the task.",
},
},
],
client=client,
)
except Exception as e:
print(e)
mock_post.assert_called_once()
print(mock_post.call_args.kwargs)
assert mock_post.call_args.kwargs["json"]["tools"] == [
{
"function_declarations": [
{
"name": "finish",
"description": "Finish the interaction when the task is complete OR if the assistant cannot proceed further with the task.",
}
]
}
]

4
tests/local_testing/test_caching.py
View file

@ -609,7 +609,7 @@ async def test_embedding_caching_redis_ttl():
type="redis",
host="dummy_host",
password="dummy_password",
default_in_redis_ttl=2,
default_in_redis_ttl=2.5,
)
inputs = [
@ -635,7 +635,7 @@ async def test_embedding_caching_redis_ttl():
print(f"redis pipeline set args: {args}")
print(f"redis pipeline set kwargs: {kwargs}")
assert kwargs.get("ex") == datetime.timedelta(
seconds=2
seconds=2.5
) # Check if TTL is set to 2.5 seconds

50
tests/local_testing/test_router_timeout.py
View file

@ -13,7 +13,7 @@ sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
from unittest.mock import patch, MagicMock, AsyncMock
import os
from dotenv import load_dotenv
@ -139,51 +139,3 @@ async def test_router_timeouts_bedrock():
pytest.fail(
f"Did not raise error `openai.APITimeoutError`. Instead raised error type: {type(e)}, Error: {e}"
)
@pytest.mark.parametrize(
"num_retries, expected_call_count",
[(0, 1), (1, 2), (2, 3), (3, 4)],
)
def test_router_timeout_with_retries_anthropic_model(num_retries, expected_call_count):
"""
If request hits custom timeout, ensure it's retried.
"""
litellm._turn_on_debug()
from litellm.llms.custom_httpx.http_handler import HTTPHandler
import time
litellm.num_retries = num_retries
litellm.request_timeout = 0.000001
router = Router(
model_list=[
{
"model_name": "claude-3-haiku",
"litellm_params": {
"model": "anthropic/claude-3-haiku-20240307",
},
}
],
)
custom_client = HTTPHandler()
with patch.object(custom_client, "post", new=MagicMock()) as mock_client:
try:
def delayed_response(*args, **kwargs):
time.sleep(0.01) # Exceeds the 0.000001 timeout
raise TimeoutError("Request timed out.")
mock_client.side_effect = delayed_response
router.completion(
model="claude-3-haiku",
messages=[{"role": "user", "content": "hello, who are u"}],
client=custom_client,
)
except litellm.Timeout:
pass
assert mock_client.call_count == expected_call_count

9
tests/logging_callback_tests/test_prometheus_unit_tests.py
View file

@ -549,14 +549,13 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
standard_logging_payload = create_standard_logging_payload()
standard_logging_payload["hidden_params"]["additional_headers"] = {
"x_ratelimit_remaining_requests": 123,
"x_ratelimit_remaining_tokens": 4321,
}
# Create test data
request_kwargs = {
"model": "gpt-3.5-turbo",
"response_headers": {
"x-ratelimit-remaining-requests": 123,
"x-ratelimit-remaining-tokens": 4321,
},
"litellm_params": {
"custom_llm_provider": "openai",
"metadata": {"model_info": {"id": "model-123"}},

39
tests/logging_callback_tests/test_standard_logging_payload.py
View file

@ -65,42 +65,3 @@ def test_get_usage(response_obj, expected_values):
assert usage.prompt_tokens == expected_values[0]
assert usage.completion_tokens == expected_values[1]
assert usage.total_tokens == expected_values[2]
def test_get_additional_headers():
additional_headers = {
"x-ratelimit-limit-requests": "2000",
"x-ratelimit-remaining-requests": "1999",
"x-ratelimit-limit-tokens": "160000",
"x-ratelimit-remaining-tokens": "160000",
"llm_provider-date": "Tue, 29 Oct 2024 23:57:37 GMT",
"llm_provider-content-type": "application/json",
"llm_provider-transfer-encoding": "chunked",
"llm_provider-connection": "keep-alive",
"llm_provider-anthropic-ratelimit-requests-limit": "2000",
"llm_provider-anthropic-ratelimit-requests-remaining": "1999",
"llm_provider-anthropic-ratelimit-requests-reset": "2024-10-29T23:57:40Z",
"llm_provider-anthropic-ratelimit-tokens-limit": "160000",
"llm_provider-anthropic-ratelimit-tokens-remaining": "160000",
"llm_provider-anthropic-ratelimit-tokens-reset": "2024-10-29T23:57:36Z",
"llm_provider-request-id": "req_01F6CycZZPSHKRCCctcS1Vto",
"llm_provider-via": "1.1 google",
"llm_provider-cf-cache-status": "DYNAMIC",
"llm_provider-x-robots-tag": "none",
"llm_provider-server": "cloudflare",
"llm_provider-cf-ray": "8da71bdbc9b57abb-SJC",
"llm_provider-content-encoding": "gzip",
"llm_provider-x-ratelimit-limit-requests": "2000",
"llm_provider-x-ratelimit-remaining-requests": "1999",
"llm_provider-x-ratelimit-limit-tokens": "160000",
"llm_provider-x-ratelimit-remaining-tokens": "160000",
}
additional_logging_headers = StandardLoggingPayloadSetup.get_additional_headers(
additional_headers
)
assert additional_logging_headers == {
"x_ratelimit_limit_requests": 2000,
"x_ratelimit_remaining_requests": 1999,
"x_ratelimit_limit_tokens": 160000,
"x_ratelimit_remaining_tokens": 160000,
}