From 9cf3d5f568664335e5895308d516665effd9423c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 6 Aug 2024 18:16:07 -0700 Subject: [PATCH 1/8] feat(utils.py): support passing response_format as pydantic model Related issue - https://github.com/BerriAI/litellm/issues/5074 --- litellm/main.py | 2 +- litellm/tests/test_completion.py | 37 ++++++++++++++++++++++++++++++++ litellm/utils.py | 37 ++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 1209306c8b0..01e3d2f9533 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -608,7 +608,7 @@ def completion( logit_bias: Optional[dict] = None, user: Optional[str] = None, # openai v1.0+ new params - response_format: Optional[dict] = None, + response_format: Optional[Union[dict, type[BaseModel]]] = None, seed: Optional[int] = None, tools: Optional[List] = None, tool_choice: Optional[Union[str, dict]] = None, diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index eec163f26a4..04b260c2e8f 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2123,6 +2123,43 @@ def test_completion_openai(): pytest.fail(f"Error occurred: {e}") +def test_completion_openai_pydantic(): + try: + litellm.set_verbose = True + from pydantic import BaseModel + + class CalendarEvent(BaseModel): + name: str + date: str + participants: list[str] + + print(f"api key: {os.environ['OPENAI_API_KEY']}") + litellm.api_key = os.environ["OPENAI_API_KEY"] + response = completion( + model="gpt-4o-2024-08-06", + messages=[{"role": "user", "content": "Hey"}], + max_tokens=10, + metadata={"hi": "bye"}, + response_format=CalendarEvent, + ) + print("This is the response object\n", response) + + response_str = response["choices"][0]["message"]["content"] + response_str_2 = response.choices[0].message.content + + cost = completion_cost(completion_response=response) + print("Cost for completion call with gpt-3.5-turbo: ", f"${float(cost):.10f}") + assert response_str == response_str_2 + assert type(response_str) == str + assert len(response_str) > 1 + + litellm.api_key = None + except Timeout as e: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + def test_completion_openai_organization(): try: litellm.set_verbose = True diff --git a/litellm/utils.py b/litellm/utils.py index 20beb47dc2f..ed155ab143a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -45,6 +45,8 @@ import requests import tiktoken from httpx import Proxy from httpx._utils import get_environment_proxies +from openai.lib import _parsing, _pydantic +from openai.types.chat.completion_create_params import ResponseFormat from pydantic import BaseModel from tokenizers import Tokenizer @@ -2806,6 +2808,11 @@ def get_optional_params( message=f"Function calling is not supported by {custom_llm_provider}.", ) + if "response_format" in non_default_params: + non_default_params["response_format"] = type_to_response_format_param( + response_format=non_default_params["response_format"] + ) + if "tools" in non_default_params and isinstance( non_default_params, list ): # fixes https://github.com/BerriAI/litellm/issues/4933 @@ -6112,6 +6119,36 @@ def _should_retry(status_code: int): return False +def type_to_response_format_param( + response_format: Optional[Union[type[BaseModel], dict]], +) -> Optional[dict]: + """ + Re-implementation of openai's 'type_to_response_format_param' function + + Used for converting pydantic object to api schema. + """ + if response_format is None: + return None + + if isinstance(response_format, dict): + return response_format + + # type checkers don't narrow the negation of a `TypeGuard` as it isn't + # a safe default behaviour but we know that at this point the `response_format` + # can only be a `type` + if not _parsing._completions.is_basemodel_type(response_format): + raise TypeError(f"Unsupported response_format type - {response_format}") + + return { + "type": "json_schema", + "json_schema": { + "schema": _pydantic.to_strict_json_schema(response_format), + "name": response_format.__name__, + "strict": True, + }, + } + + def _get_retry_after_from_exception_header( response_headers: Optional[httpx.Headers] = None, ): From 0c88cc41538a66d2189fde44e24f67140a41fdaf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 6 Aug 2024 18:27:06 -0700 Subject: [PATCH 2/8] docs(json_mode.md): add example of calling openai with pydantic model via litellm --- docs/my-website/docs/completion/json_mode.md | 39 +++----------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/docs/my-website/docs/completion/json_mode.md b/docs/my-website/docs/completion/json_mode.md index 92e135dff5c..3c3bca3adba 100644 --- a/docs/my-website/docs/completion/json_mode.md +++ b/docs/my-website/docs/completion/json_mode.md @@ -71,12 +71,6 @@ response_format: { "type": "json_schema", "json_schema": … , "strict": true } Works for OpenAI models -:::info - -Support for passing in a pydantic object to litellm sdk will be [coming soon](https://github.com/BerriAI/litellm/issues/5074#issuecomment-2272355842) - -::: - @@ -89,36 +83,15 @@ os.environ["OPENAI_API_KEY"] = "" messages = [{"role": "user", "content": "List 5 cookie recipes"}] +class CalendarEvent(BaseModel): + name: str + date: str + participants: list[str] + resp = completion( model="gpt-4o-2024-08-06", messages=messages, - response_format={ - "type": "json_schema", - "json_schema": { - "name": "math_reasoning", - "schema": { - "type": "object", - "properties": { - "steps": { - "type": "array", - "items": { - "type": "object", - "properties": { - "explanation": { "type": "string" }, - "output": { "type": "string" } - }, - "required": ["explanation", "output"], - "additionalProperties": False - } - }, - "final_answer": { "type": "string" } - }, - "required": ["steps", "final_answer"], - "additionalProperties": False - }, - "strict": True - }, - } + response_format=CalendarEvent ) print("Received={}".format(resp)) From 5dfde2ee0b3327edd4e71b2c81d811e289e3b886 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 6 Aug 2024 19:06:14 -0700 Subject: [PATCH 3/8] feat: Translate openai 'response_format' json_schema to 'response_schema' for vertex ai + google ai studio Closes https://github.com/BerriAI/litellm/issues/5074 --- litellm/llms/vertex_ai_anthropic.py | 9 +- litellm/llms/vertex_httpx.py | 19 ++- .../tests/test_amazing_vertex_completion.py | 144 ++++++++++++++++-- litellm/utils.py | 24 +++ poetry.lock | 82 +++++++++- pyproject.toml | 2 +- requirements.txt | 2 +- 7 files changed, 250 insertions(+), 32 deletions(-) diff --git a/litellm/llms/vertex_ai_anthropic.py b/litellm/llms/vertex_ai_anthropic.py index 900e7795f76..5887458527e 100644 --- a/litellm/llms/vertex_ai_anthropic.py +++ b/litellm/llms/vertex_ai_anthropic.py @@ -148,7 +148,12 @@ class VertexAIAnthropicConfig: optional_params["temperature"] = value if param == "top_p": optional_params["top_p"] = value - if param == "response_format" and "response_schema" in value: + if param == "response_format" and isinstance(value, dict): + json_schema: Optional[dict] = None + if "response_schema" in value: + json_schema = value["response_schema"] + elif "json_schema" in value: + json_schema = value["json_schema"]["schema"] """ When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode - You usually want to provide a single tool @@ -162,7 +167,7 @@ class VertexAIAnthropicConfig: name="json_tool_call", input_schema={ "type": "object", - "properties": {"values": value["response_schema"]}, # type: ignore + "properties": {"values": json_schema}, # type: ignore }, ) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index db61b129b31..fa6308bef7e 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -181,13 +181,17 @@ class GoogleAIStudioGeminiConfig: # key diff from VertexAI - 'frequency_penalty optional_params["stop_sequences"] = value if param == "max_tokens": optional_params["max_output_tokens"] = value - if param == "response_format" and value["type"] == "json_object": # type: ignore + if param == "response_format": # type: ignore if value["type"] == "json_object": # type: ignore - optional_params["response_mime_type"] = "application/json" - elif value["type"] == "text": # type: ignore - optional_params["response_mime_type"] = "text/plain" - if "response_schema" in value: # type: ignore - optional_params["response_schema"] = value["response_schema"] # type: ignore + if value["type"] == "json_object": # type: ignore + optional_params["response_mime_type"] = "application/json" + elif value["type"] == "text": # type: ignore + optional_params["response_mime_type"] = "text/plain" + if "response_schema" in value: # type: ignore + optional_params["response_schema"] = value["response_schema"] # type: ignore + elif value["type"] == "json_schema": # type: ignore + if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore + optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore if param == "tools" and isinstance(value, list): gtool_func_declarations = [] for tool in value: @@ -396,6 +400,9 @@ class VertexGeminiConfig: optional_params["response_mime_type"] = "text/plain" if "response_schema" in value: optional_params["response_schema"] = value["response_schema"] + elif value["type"] == "json_schema": # type: ignore + if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore + optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore if param == "frequency_penalty": optional_params["frequency_penalty"] = value if param == "presence_penalty": diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 4338d63ba60..53bb9fd8030 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -1192,7 +1192,15 @@ def vertex_httpx_mock_post_valid_response(*args, **kwargs): "role": "model", "parts": [ { - "text": '[{"recipe_name": "Chocolate Chip Cookies"}, {"recipe_name": "Oatmeal Raisin Cookies"}, {"recipe_name": "Peanut Butter Cookies"}, {"recipe_name": "Sugar Cookies"}, {"recipe_name": "Snickerdoodles"}]\n' + "text": """{ + "recipes": [ + {"recipe_name": "Chocolate Chip Cookies"}, + {"recipe_name": "Oatmeal Raisin Cookies"}, + {"recipe_name": "Peanut Butter Cookies"}, + {"recipe_name": "Sugar Cookies"}, + {"recipe_name": "Snickerdoodles"} + ] + }""" } ], }, @@ -1253,13 +1261,15 @@ def vertex_httpx_mock_post_valid_response_anthropic(*args, **kwargs): "id": "toolu_vrtx_01YMnYZrToPPfcmY2myP2gEB", "name": "json_tool_call", "input": { - "values": [ - {"recipe_name": "Chocolate Chip Cookies"}, - {"recipe_name": "Oatmeal Raisin Cookies"}, - {"recipe_name": "Peanut Butter Cookies"}, - {"recipe_name": "Snickerdoodle Cookies"}, - {"recipe_name": "Sugar Cookies"}, - ] + "values": { + "recipes": [ + {"recipe_name": "Chocolate Chip Cookies"}, + {"recipe_name": "Oatmeal Raisin Cookies"}, + {"recipe_name": "Peanut Butter Cookies"}, + {"recipe_name": "Snickerdoodle Cookies"}, + {"recipe_name": "Sugar Cookies"}, + ] + } }, } ], @@ -1377,16 +1387,19 @@ async def test_gemini_pro_json_schema_args_sent_httpx( from litellm.llms.custom_httpx.http_handler import HTTPHandler response_schema = { - "type": "array", - "items": { - "type": "object", - "properties": { - "recipe_name": { - "type": "string", + "type": "object", + "properties": { + "recipes": { + "type": "array", + "items": { + "type": "object", + "properties": {"recipe_name": {"type": "string"}}, + "required": ["recipe_name"], }, - }, - "required": ["recipe_name"], + } }, + "required": ["recipes"], + "additionalProperties": False, } client = HTTPHandler() @@ -1448,6 +1461,105 @@ async def test_gemini_pro_json_schema_args_sent_httpx( ) +@pytest.mark.parametrize( + "model, vertex_location, supports_response_schema", + [ + ("vertex_ai_beta/gemini-1.5-pro-001", "us-central1", True), + ("gemini/gemini-1.5-pro", None, True), + ("vertex_ai_beta/gemini-1.5-flash", "us-central1", False), + ("vertex_ai/claude-3-5-sonnet@20240620", "us-east5", False), + ], +) +@pytest.mark.parametrize( + "invalid_response", + [True, False], +) +@pytest.mark.parametrize( + "enforce_validation", + [True, False], +) +@pytest.mark.asyncio +async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema( + model, + supports_response_schema, + vertex_location, + invalid_response, + enforce_validation, +): + from typing import List + + from pydantic import BaseModel + + load_vertex_ai_credentials() + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + litellm.set_verbose = True + + messages = [{"role": "user", "content": "List 5 cookie recipes"}] + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + class Recipe(BaseModel): + recipe_name: str + + class ResponseSchema(BaseModel): + recipes: List[Recipe] + + client = HTTPHandler() + + httpx_response = MagicMock() + if invalid_response is True: + if "claude" in model: + httpx_response.side_effect = ( + vertex_httpx_mock_post_invalid_schema_response_anthropic + ) + else: + httpx_response.side_effect = vertex_httpx_mock_post_invalid_schema_response + else: + if "claude" in model: + httpx_response.side_effect = vertex_httpx_mock_post_valid_response_anthropic + else: + httpx_response.side_effect = vertex_httpx_mock_post_valid_response + with patch.object(client, "post", new=httpx_response) as mock_call: + print("SENDING CLIENT POST={}".format(client.post)) + try: + resp = completion( + model=model, + messages=messages, + response_format=ResponseSchema, + vertex_location=vertex_location, + client=client, + ) + print("Received={}".format(resp)) + if invalid_response is True and enforce_validation is True: + pytest.fail("Expected this to fail") + except litellm.JSONSchemaValidationError as e: + if invalid_response is False: + pytest.fail("Expected this to pass. Got={}".format(e)) + + mock_call.assert_called_once() + if "claude" not in model: + print(mock_call.call_args.kwargs) + print(mock_call.call_args.kwargs["json"]["generationConfig"]) + + if supports_response_schema: + assert ( + "response_schema" + in mock_call.call_args.kwargs["json"]["generationConfig"] + ) + else: + assert ( + "response_schema" + not in mock_call.call_args.kwargs["json"]["generationConfig"] + ) + assert ( + "Use this JSON schema:" + in mock_call.call_args.kwargs["json"]["contents"][0]["parts"][1][ + "text" + ] + ) + + @pytest.mark.parametrize("provider", ["vertex_ai_beta"]) # "vertex_ai", @pytest.mark.asyncio async def test_gemini_pro_httpx_custom_api_base(provider): diff --git a/litellm/utils.py b/litellm/utils.py index ed155ab143a..f106132689f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -645,6 +645,30 @@ def client(original_function): input=model_response, model=model ) ### JSON SCHEMA VALIDATION ### + try: + if ( + optional_params is not None + and "response_format" in optional_params + and _parsing._completions.is_basemodel_type( + optional_params["response_format"] + ) + ): + json_response_format = ( + type_to_response_format_param( + response_format=optional_params[ + "response_format" + ] + ) + ) + if json_response_format is not None: + litellm.litellm_core_utils.json_validation_rule.validate_schema( + schema=json_response_format[ + "json_schema" + ]["schema"], + response=model_response, + ) + except TypeError: + pass if ( optional_params is not None and "response_format" in optional_params diff --git a/poetry.lock b/poetry.lock index d1b428ac7c0..12b89473f74 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1311,6 +1311,76 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jiter" +version = "0.5.0" +description = "Fast iterable JSON parser." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jiter-0.5.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b599f4e89b3def9a94091e6ee52e1d7ad7bc33e238ebb9c4c63f211d74822c3f"}, + {file = "jiter-0.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a063f71c4b06225543dddadbe09d203dc0c95ba352d8b85f1221173480a71d5"}, + {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc0d5b8b3dd12e91dd184b87273f864b363dfabc90ef29a1092d269f18c7e28"}, + {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c22541f0b672f4d741382a97c65609332a783501551445ab2df137ada01e019e"}, + {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63314832e302cc10d8dfbda0333a384bf4bcfce80d65fe99b0f3c0da8945a91a"}, + {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a25fbd8a5a58061e433d6fae6d5298777c0814a8bcefa1e5ecfff20c594bd749"}, + {file = "jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:503b2c27d87dfff5ab717a8200fbbcf4714516c9d85558048b1fc14d2de7d8dc"}, + {file = "jiter-0.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6d1f3d27cce923713933a844872d213d244e09b53ec99b7a7fdf73d543529d6d"}, + {file = "jiter-0.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c95980207b3998f2c3b3098f357994d3fd7661121f30669ca7cb945f09510a87"}, + {file = "jiter-0.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:afa66939d834b0ce063f57d9895e8036ffc41c4bd90e4a99631e5f261d9b518e"}, + {file = "jiter-0.5.0-cp310-none-win32.whl", hash = "sha256:f16ca8f10e62f25fd81d5310e852df6649af17824146ca74647a018424ddeccf"}, + {file = "jiter-0.5.0-cp310-none-win_amd64.whl", hash = "sha256:b2950e4798e82dd9176935ef6a55cf6a448b5c71515a556da3f6b811a7844f1e"}, + {file = "jiter-0.5.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d4c8e1ed0ef31ad29cae5ea16b9e41529eb50a7fba70600008e9f8de6376d553"}, + {file = "jiter-0.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6f16e21276074a12d8421692515b3fd6d2ea9c94fd0734c39a12960a20e85f3"}, + {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5280e68e7740c8c128d3ae5ab63335ce6d1fb6603d3b809637b11713487af9e6"}, + {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:583c57fc30cc1fec360e66323aadd7fc3edeec01289bfafc35d3b9dcb29495e4"}, + {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26351cc14507bdf466b5f99aba3df3143a59da75799bf64a53a3ad3155ecded9"}, + {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4829df14d656b3fb87e50ae8b48253a8851c707da9f30d45aacab2aa2ba2d614"}, + {file = "jiter-0.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42a4bdcf7307b86cb863b2fb9bb55029b422d8f86276a50487982d99eed7c6e"}, + {file = "jiter-0.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04d461ad0aebf696f8da13c99bc1b3e06f66ecf6cfd56254cc402f6385231c06"}, + {file = "jiter-0.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e6375923c5f19888c9226582a124b77b622f8fd0018b843c45eeb19d9701c403"}, + {file = "jiter-0.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2cec323a853c24fd0472517113768c92ae0be8f8c384ef4441d3632da8baa646"}, + {file = "jiter-0.5.0-cp311-none-win32.whl", hash = "sha256:aa1db0967130b5cab63dfe4d6ff547c88b2a394c3410db64744d491df7f069bb"}, + {file = "jiter-0.5.0-cp311-none-win_amd64.whl", hash = "sha256:aa9d2b85b2ed7dc7697597dcfaac66e63c1b3028652f751c81c65a9f220899ae"}, + {file = "jiter-0.5.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9f664e7351604f91dcdd557603c57fc0d551bc65cc0a732fdacbf73ad335049a"}, + {file = "jiter-0.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:044f2f1148b5248ad2c8c3afb43430dccf676c5a5834d2f5089a4e6c5bbd64df"}, + {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:702e3520384c88b6e270c55c772d4bd6d7b150608dcc94dea87ceba1b6391248"}, + {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:528d742dcde73fad9d63e8242c036ab4a84389a56e04efd854062b660f559544"}, + {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8cf80e5fe6ab582c82f0c3331df27a7e1565e2dcf06265afd5173d809cdbf9ba"}, + {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:44dfc9ddfb9b51a5626568ef4e55ada462b7328996294fe4d36de02fce42721f"}, + {file = "jiter-0.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c451f7922992751a936b96c5f5b9bb9312243d9b754c34b33d0cb72c84669f4e"}, + {file = "jiter-0.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:308fce789a2f093dca1ff91ac391f11a9f99c35369117ad5a5c6c4903e1b3e3a"}, + {file = "jiter-0.5.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7f5ad4a7c6b0d90776fdefa294f662e8a86871e601309643de30bf94bb93a64e"}, + {file = "jiter-0.5.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ea189db75f8eca08807d02ae27929e890c7d47599ce3d0a6a5d41f2419ecf338"}, + {file = "jiter-0.5.0-cp312-none-win32.whl", hash = "sha256:e3bbe3910c724b877846186c25fe3c802e105a2c1fc2b57d6688b9f8772026e4"}, + {file = "jiter-0.5.0-cp312-none-win_amd64.whl", hash = "sha256:a586832f70c3f1481732919215f36d41c59ca080fa27a65cf23d9490e75b2ef5"}, + {file = "jiter-0.5.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:f04bc2fc50dc77be9d10f73fcc4e39346402ffe21726ff41028f36e179b587e6"}, + {file = "jiter-0.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6f433a4169ad22fcb550b11179bb2b4fd405de9b982601914ef448390b2954f3"}, + {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad4a6398c85d3a20067e6c69890ca01f68659da94d74c800298581724e426c7e"}, + {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6baa88334e7af3f4d7a5c66c3a63808e5efbc3698a1c57626541ddd22f8e4fbf"}, + {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ece0a115c05efca597c6d938f88c9357c843f8c245dbbb53361a1c01afd7148"}, + {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:335942557162ad372cc367ffaf93217117401bf930483b4b3ebdb1223dbddfa7"}, + {file = "jiter-0.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:649b0ee97a6e6da174bffcb3c8c051a5935d7d4f2f52ea1583b5b3e7822fbf14"}, + {file = "jiter-0.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f4be354c5de82157886ca7f5925dbda369b77344b4b4adf2723079715f823989"}, + {file = "jiter-0.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5206144578831a6de278a38896864ded4ed96af66e1e63ec5dd7f4a1fce38a3a"}, + {file = "jiter-0.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8120c60f8121ac3d6f072b97ef0e71770cc72b3c23084c72c4189428b1b1d3b6"}, + {file = "jiter-0.5.0-cp38-none-win32.whl", hash = "sha256:6f1223f88b6d76b519cb033a4d3687ca157c272ec5d6015c322fc5b3074d8a5e"}, + {file = "jiter-0.5.0-cp38-none-win_amd64.whl", hash = "sha256:c59614b225d9f434ea8fc0d0bec51ef5fa8c83679afedc0433905994fb36d631"}, + {file = "jiter-0.5.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:0af3838cfb7e6afee3f00dc66fa24695199e20ba87df26e942820345b0afc566"}, + {file = "jiter-0.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:550b11d669600dbc342364fd4adbe987f14d0bbedaf06feb1b983383dcc4b961"}, + {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:489875bf1a0ffb3cb38a727b01e6673f0f2e395b2aad3c9387f94187cb214bbf"}, + {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b250ca2594f5599ca82ba7e68785a669b352156260c5362ea1b4e04a0f3e2389"}, + {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ea18e01f785c6667ca15407cd6dabbe029d77474d53595a189bdc813347218e"}, + {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:462a52be85b53cd9bffd94e2d788a09984274fe6cebb893d6287e1c296d50653"}, + {file = "jiter-0.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92cc68b48d50fa472c79c93965e19bd48f40f207cb557a8346daa020d6ba973b"}, + {file = "jiter-0.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1c834133e59a8521bc87ebcad773608c6fa6ab5c7a022df24a45030826cf10bc"}, + {file = "jiter-0.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab3a71ff31cf2d45cb216dc37af522d335211f3a972d2fe14ea99073de6cb104"}, + {file = "jiter-0.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cccd3af9c48ac500c95e1bcbc498020c87e1781ff0345dd371462d67b76643eb"}, + {file = "jiter-0.5.0-cp39-none-win32.whl", hash = "sha256:368084d8d5c4fc40ff7c3cc513c4f73e02c85f6009217922d0823a48ee7adf61"}, + {file = "jiter-0.5.0-cp39-none-win_amd64.whl", hash = "sha256:ce03f7b4129eb72f1687fa11300fbf677b02990618428934662406d2a76742a1"}, + {file = "jiter-0.5.0.tar.gz", hash = "sha256:1d916ba875bcab5c5f7d927df998c4cb694d27dceddf3392e58beaf10563368a"}, +] + [[package]] name = "jsonschema" version = "4.22.0" @@ -1691,23 +1761,24 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "openai" -version = "1.30.1" +version = "1.40.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.30.1-py3-none-any.whl", hash = "sha256:c9fb3c3545c118bbce8deb824397b9433a66d0d0ede6a96f7009c95b76de4a46"}, - {file = "openai-1.30.1.tar.gz", hash = "sha256:4f85190e577cba0b066e1950b8eb9b11d25bc7ebcc43a86b326ce1bfa564ec74"}, + {file = "openai-1.40.0-py3-none-any.whl", hash = "sha256:eb6909abaacd62ef28c275a5c175af29f607b40645b0a49d2856bbed62edb2e7"}, + {file = "openai-1.40.0.tar.gz", hash = "sha256:1b7b316e27b2333b063ee62b6539b74267c7282498d9a02fc4ccb38a9c14336c"}, ] [package.dependencies] anyio = ">=3.5.0,<5" distro = ">=1.7.0,<2" httpx = ">=0.23.0,<1" +jiter = ">=0.4.0,<1" pydantic = ">=1.9.0,<3" sniffio = "*" tqdm = ">4" -typing-extensions = ">=4.7,<5" +typing-extensions = ">=4.11,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] @@ -2267,7 +2338,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -3414,4 +3484,4 @@ proxy = ["PyJWT", "apscheduler", "backoff", "cryptography", "fastapi", "fastapi- [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "6025cae7749c94755d17362f77adf76f834863dba2126501cd3111d53a9c5779" +content-hash = "dd2242834589eb08430e4acbd470d1bdcf4438fe0bed7ff6ea5b48a7cba0eb10" diff --git a/pyproject.toml b/pyproject.toml index c36b40c6178..c331ddc31cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ documentation = "https://docs.litellm.ai" [tool.poetry.dependencies] python = ">=3.8.1,<4.0, !=3.9.7" -openai = ">=1.27.0" +openai = ">=1.40.0" python-dotenv = ">=0.2.0" tiktoken = ">=0.7.0" importlib-metadata = ">=6.8.0" diff --git a/requirements.txt b/requirements.txt index e6cc0722761..e72f386f8ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # LITELLM PROXY DEPENDENCIES # anyio==4.2.0 # openai + http req. -openai==1.34.0 # openai req. +openai==1.40.0 # openai req. fastapi==0.111.0 # server dep backoff==2.2.1 # server dep pyyaml==6.0.0 # server dep From 2dd27a4e12ed0552d771895a4b36a616dd392d14 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 6 Aug 2024 19:35:33 -0700 Subject: [PATCH 4/8] feat(utils.py): support validating json schema client-side if user opts in --- docs/my-website/docs/completion/json_mode.md | 109 ++++++++++++------- litellm/__init__.py | 1 + litellm/proxy/_new_secret_config.yaml | 2 +- litellm/utils.py | 67 ++++++++---- 4 files changed, 117 insertions(+), 62 deletions(-) diff --git a/docs/my-website/docs/completion/json_mode.md b/docs/my-website/docs/completion/json_mode.md index 3c3bca3adba..bf159cd07e7 100644 --- a/docs/my-website/docs/completion/json_mode.md +++ b/docs/my-website/docs/completion/json_mode.md @@ -69,7 +69,10 @@ To use Structured Outputs, simply specify response_format: { "type": "json_schema", "json_schema": … , "strict": true } ``` -Works for OpenAI models +Works for: +- OpenAI models +- Google AI Studio - Gemini models +- Vertex AI models (Gemini + Anthropic) @@ -202,15 +205,15 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ ## Validate JSON Schema -:::info -Support for doing this in the openai 'json_schema' format will be [coming soon](https://github.com/BerriAI/litellm/issues/5074#issuecomment-2272355842) +Not all vertex models support passing the json_schema to them (e.g. `gemini-1.5-flash`). To solve this, LiteLLM supports client-side validation of the json schema. -::: +``` +litellm.enable_json_schema_validation=True +``` +If `litellm.enable_json_schema_validation=True` is set, LiteLLM will validate the json response using `jsonvalidator`. -For VertexAI models, LiteLLM supports passing the `response_schema` and validating the JSON output. - -This works across Gemini (`vertex_ai_beta/`) + Anthropic (`vertex_ai/`) models. +[**See Code**](https://github.com/BerriAI/litellm/blob/671d8ac496b6229970c7f2a3bdedd6cb84f0746b/litellm/litellm_core_utils/json_validation_rule.py#L4) @@ -218,33 +221,28 @@ This works across Gemini (`vertex_ai_beta/`) + Anthropic (`vertex_ai/`) models. ```python # !gcloud auth application-default login - run this to add vertex credentials to your env - +import litellm, os from litellm import completion +from pydantic import BaseModel -messages = [{"role": "user", "content": "List 5 cookie recipes"}] -response_schema = { - "type": "array", - "items": { - "type": "object", - "properties": { - "recipe_name": { - "type": "string", - }, - }, - "required": ["recipe_name"], - }, -} +messages=[ + {"role": "system", "content": "Extract the event information."}, + {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, + ] + +litellm.enable_json_schema_validation = True +litellm.set_verbose = True # see the raw request made by litellm + +class CalendarEvent(BaseModel): + name: str + date: str + participants: list[str] resp = completion( - model="vertex_ai_beta/gemini-1.5-pro", + model="gemini/gemini-1.5-pro", messages=messages, - response_format={ - "type": "json_object", - "response_schema": response_schema, - "enforce_validation": True, # client-side json schema validation - }, - vertex_location="us-east5", + response_format=CalendarEvent, ) print("Received={}".format(resp)) @@ -252,26 +250,63 @@ print("Received={}".format(resp)) +1. Create config.yaml +```yaml +model_list: + - model_name: "gemini-1.5-flash" + litellm_params: + model: "gemini/gemini-1.5-flash" + api_key: os.environ/GEMINI_API_KEY + +litellm_settings: + enable_json_schema_validation: True +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + ```bash curl http://0.0.0.0:4000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $LITELLM_API_KEY" \ -d '{ - "model": "vertex_ai_beta/gemini-1.5-pro", - "messages": [{"role": "user", "content": "List 5 cookie recipes"}] + "model": "gemini-1.5-flash", + "messages": [ + {"role": "system", "content": "Extract the event information."}, + {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, + ], "response_format": { "type": "json_object", - "enforce_validation: true, "response_schema": { - "type": "array", - "items": { + "type": "json_schema", + "json_schema": { + "name": "math_reasoning", + "schema": { "type": "object", "properties": { - "recipe_name": { - "type": "string", - }, + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "explanation": { "type": "string" }, + "output": { "type": "string" } + }, + "required": ["explanation", "output"], + "additionalProperties": false + } + }, + "final_answer": { "type": "string" } }, - "required": ["recipe_name"], + "required": ["steps", "final_answer"], + "additionalProperties": false + }, + "strict": true }, } }, diff --git a/litellm/__init__.py b/litellm/__init__.py index dfc3f3fc1bb..9c8513e142d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -144,6 +144,7 @@ enable_preview_features: bool = False return_response_headers: bool = ( False # get response headers from LLM Api providers - example x-remaining-requests, ) +enable_json_schema_validation: bool = False ################## logging: bool = True enable_caching_on_provider_specific_optional_params: bool = ( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 1bf073513b4..a77ddd24465 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -4,4 +4,4 @@ model_list: model: "*" litellm_settings: - callbacks: ["lakera_prompt_injection"] \ No newline at end of file + enable_json_schema_validation: true \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index f106132689f..50e2e2bf2f6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -631,8 +631,8 @@ def client(original_function): call_type == CallTypes.completion.value or call_type == CallTypes.acompletion.value ): - is_coroutine = check_coroutine(original_function) - if is_coroutine == True: + is_coroutine = check_coroutine(original_response) + if is_coroutine is True: pass else: if isinstance(original_response, ModelResponse): @@ -645,30 +645,49 @@ def client(original_function): input=model_response, model=model ) ### JSON SCHEMA VALIDATION ### - try: - if ( - optional_params is not None - and "response_format" in optional_params - and _parsing._completions.is_basemodel_type( - optional_params["response_format"] - ) - ): - json_response_format = ( - type_to_response_format_param( - response_format=optional_params[ + if litellm.enable_json_schema_validation is True: + try: + if ( + optional_params is not None + and "response_format" in optional_params + and optional_params["response_format"] + is not None + ): + json_response_format: Optional[dict] = None + if ( + isinstance( + optional_params["response_format"], + dict, + ) + and optional_params[ + "response_format" + ].get("json_schema") + is not None + ): + json_response_format = optional_params[ "response_format" ] - ) - ) - if json_response_format is not None: - litellm.litellm_core_utils.json_validation_rule.validate_schema( - schema=json_response_format[ - "json_schema" - ]["schema"], - response=model_response, - ) - except TypeError: - pass + elif ( + _parsing._completions.is_basemodel_type( + optional_params["response_format"] + ) + ): + json_response_format = ( + type_to_response_format_param( + response_format=optional_params[ + "response_format" + ] + ) + ) + if json_response_format is not None: + litellm.litellm_core_utils.json_validation_rule.validate_schema( + schema=json_response_format[ + "json_schema" + ]["schema"], + response=model_response, + ) + except TypeError: + pass if ( optional_params is not None and "response_format" in optional_params From a2e792d32e9ff66d93cfbcf4b4fff6e8fd6b2cab Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 7 Aug 2024 13:08:53 -0700 Subject: [PATCH 5/8] test: update build requirements --- .circleci/config.yml | 2 +- litellm/tests/test_completion.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f697be521a4..bcfa4c28758 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.34.0 + pip install openai==1.40.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 7450824f525..a5fd63e92b9 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -4092,7 +4092,7 @@ def test_completion_gemini(model): if "InternalServerError" in str(e): pass else: - pytest.fail(f"Error occurred: {e}") + pytest.fail(f"Error occurred:{e}") # test_completion_gemini() From 37dc359efbcc5f550e5fa56ae1d0f1cc942366f3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 7 Aug 2024 13:14:29 -0700 Subject: [PATCH 6/8] fix(utils.py): fix linting error for python3.8 --- litellm/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index ee0bed3f7b7..98c8b018419 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -160,6 +160,7 @@ from typing import ( Literal, Optional, Tuple, + Type, Union, cast, get_args, @@ -6155,7 +6156,7 @@ def _should_retry(status_code: int): def type_to_response_format_param( - response_format: Optional[Union[type[BaseModel], dict]], + response_format: Optional[Union[Type[BaseModel], dict]], ) -> Optional[dict]: """ Re-implementation of openai's 'type_to_response_format_param' function From c0ef2e9dd0e681bd094e989f43586cdea2e16b23 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 7 Aug 2024 13:21:35 -0700 Subject: [PATCH 7/8] fix(main.py): fix linting error for python3.8 --- litellm/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 0e281b5edcb..e1660b21a99 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -31,6 +31,7 @@ from typing import ( Literal, Mapping, Optional, + Type, Union, ) @@ -608,7 +609,7 @@ def completion( logit_bias: Optional[dict] = None, user: Optional[str] = None, # openai v1.0+ new params - response_format: Optional[Union[dict, type[BaseModel]]] = None, + response_format: Optional[Union[dict, Type[BaseModel]]] = None, seed: Optional[int] = None, tools: Optional[List] = None, tool_choice: Optional[Union[str, dict]] = None, From 55ad5bb7c6269d4fa65d15f132898b42a3b7a4a0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 7 Aug 2024 13:23:04 -0700 Subject: [PATCH 8/8] fix(config.yml): fix build and test --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bcfa4c28758..a1348b12ccf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -165,7 +165,7 @@ jobs: pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install aiohttp - pip install openai + pip install "openai==1.40.0" python -m pip install --upgrade pip python -m pip install -r .circleci/requirements.txt pip install "pytest==7.3.1"