From 4295f3972a90b52ac48696282a614fe246be5121 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Tue, 22 Jul 2025 17:17:05 -0600 Subject: [PATCH 01/32] initial pass at adding Heroku chat provider --- litellm/__init__.py | 2 ++ litellm/constants.py | 1 + .../get_llm_provider_logic.py | 2 ++ litellm/llms/heroku/chat/transformation.py | 28 +++++++++++++++++ litellm/main.py | 31 +++++++++++++++++++ 5 files changed, 64 insertions(+) create mode 100644 litellm/llms/heroku/chat/transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 192b210e865..138a84493c0 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -216,6 +216,7 @@ nlp_cloud_key: Optional[str] = None novita_api_key: Optional[str] = None snowflake_key: Optional[str] = None nebius_key: Optional[str] = None +heroku_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], @@ -1151,6 +1152,7 @@ from .llms.azure.azure import ( AzureOpenAIAssistantsAPIConfig, ) +from .llms.heroku.chat.transformation import HerokuChatConfig from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig from .llms.azure.completion.transformation import AzureOpenAITextConfig from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig diff --git a/litellm/constants.py b/litellm/constants.py index afdd95385cb..4bbf6c40505 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -279,6 +279,7 @@ LITELLM_CHAT_PROVIDERS = [ "dashscope", "moonshot", "v0", + "heroku", ] LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [ diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 84c25d49323..e8f1477b90c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -350,6 +350,8 @@ def get_llm_provider( # noqa: PLR0915 # bytez models elif model.startswith("bytez/"): custom_llm_provider = "bytez" + elif model.startswith("heroku/"): + custom_llm_provider = "heroku" if not custom_llm_provider: if litellm.suppress_debug_info is False: print() # noqa diff --git a/litellm/llms/heroku/chat/transformation.py b/litellm/llms/heroku/chat/transformation.py new file mode 100644 index 00000000000..a6bceb525a6 --- /dev/null +++ b/litellm/llms/heroku/chat/transformation.py @@ -0,0 +1,28 @@ +from typing import Optional, List, Union +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.types.llms.openai import AllMessageValues + +class HerokuChatConfig(BaseConfig): + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + headers.update({"Authorization": f"Bearer {api_key}"}) + return headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + return f"{api_base}/v1/chat/completions" \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index e0d41260a3f..ce62581a409 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -151,6 +151,7 @@ from .llms.custom_llm import CustomLLM, custom_chat_llm_router from .llms.databricks.embed.handler import DatabricksEmbeddingHandler from .llms.deprecated_providers import aleph_alpha, palm from .llms.groq.chat.handler import GroqChatCompletion +from .llms.heroku.chat.transformation import HerokuChatConfig from .llms.huggingface.embedding.handler import HuggingFaceEmbedding from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion from .llms.ollama.completion import handler as ollama @@ -254,6 +255,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler() sagemaker_chat_completion = SagemakerChatHandler() bytez_transformation = BytezChatConfig() +heroku_transformation = HerokuChatConfig() ####### COMPLETION ENDPOINTS ################ @@ -1768,6 +1770,35 @@ def completion( # type: ignore # noqa: PLR0915 additional_args={"headers": headers}, ) raise e + elif custom_llm_provider == "heroku": + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=encoding, + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + elif custom_llm_provider == "xai": ## COMPLETION CALL try: From 9a9f8826935d5156c4e4a69b0ee72b5b1caa011e Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Wed, 23 Jul 2025 17:06:11 -0600 Subject: [PATCH 02/32] adds test for chat tranformation --- litellm/llms/heroku/chat/transformation.py | 93 ++++++++++++++----- .../heroku/test_heroku_chat_transformation.py | 33 +++++++ 2 files changed, 101 insertions(+), 25 deletions(-) create mode 100644 tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py diff --git a/litellm/llms/heroku/chat/transformation.py b/litellm/llms/heroku/chat/transformation.py index a6bceb525a6..6626de69009 100644 --- a/litellm/llms/heroku/chat/transformation.py +++ b/litellm/llms/heroku/chat/transformation.py @@ -1,28 +1,71 @@ -from typing import Optional, List, Union -from litellm.llms.base_llm.chat.transformation import BaseConfig -from litellm.types.llms.openai import AllMessageValues +""" +Heroku Chat Completions API -class HerokuChatConfig(BaseConfig): - def validate_environment( +this is OpenAI compatible - no translation needed / occurs +""" +import os + +from typing import Optional, List, Union +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + +# Base error class for Heroku +class HerokuError(Exception): + pass + +class HerokuChatConfig(OpenAIGPTConfig): + max_tokens: Optional[int] = None + stop: Optional[List[str]] = None + stream: Optional[bool] = None + temperature: Optional[float] = None + tool_choice: Optional[str] = None + tools: Optional[list] = None + top_p: Optional[int] = None + + def __init__( self, - headers: dict, - model: str, - messages: List[AllMessageValues], - optional_params: dict, - litellm_params: dict, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - ) -> dict: - headers.update({"Authorization": f"Bearer {api_key}"}) - return headers - - def get_complete_url( - self, - api_base: Optional[str], - api_key: Optional[str], - model: str, - optional_params: dict, - litellm_params: dict, + max_tokens: Optional[int] = None, + stop: Optional[List[str]] = None, stream: Optional[bool] = None, - ) -> str: - return f"{api_base}/v1/chat/completions" \ No newline at end of file + temperature: Optional[float] = None, + tool_choice: Optional[str] = None, + tools: Optional[list] = None, + top_p: Optional[int] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + def get_supported_openai_params(self, model: str) -> list: + return [ + "max_tokens", + "stop", + "stream", + "temperature", + "tool_choice", + "top_p", + "tools", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_openai_params = self.get_supported_openai_params(model=model) + for param, value in non_default_params.items(): + if param in supported_openai_params: + optional_params[param] = value + return optional_params + + def get_complete_url(self, api_base: Optional[str], api_key: Optional[str], model: str, optional_params: dict, litellm_params: dict, stream: Optional[bool] = None) -> str: + api_base = api_base or os.getenv("HEROKU_API_BASE") + if not api_base: + raise HerokuError("No api base was set. Please provide an api_base, or set the HEROKU_API_BASE environment variable.") + + if not api_base.endswith("/v1/chat/completions"): + api_base = f"{api_base}/v1/chat/completions" + + return api_base \ No newline at end of file diff --git a/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py b/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py new file mode 100644 index 00000000000..140105c2a26 --- /dev/null +++ b/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py @@ -0,0 +1,33 @@ +import os +import pytest +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from unittest.mock import patch +from litellm.llms.heroku.chat.transformation import HerokuChatConfig + +class TestHerokuChatConfig: + def test_default_api_base(self): + """Test that default API base is used when none is provided""" + config = HerokuChatConfig() + headers = {} + api_key = "fake-heroku-key" + + # Call validate_environment without specifying api_base + result = config.validate_environment( + headers=headers, + model="claude-3-5-haiku", + messages=[{"role": "user", "content": "Hey"}], + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base=None, # Not providing api_base + ) + + # set env var for api_base + os.environ["HEROKU_API_BASE"] = "https://mia.heroku.com" + + print('****************************************') + print(config.get_complete_url(api_base=None, api_key=api_key, model="claude-3-5-haiku", optional_params={}, litellm_params={}, stream=False)) + print('****************************************') + # Verify headers are still set correctly + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" \ No newline at end of file From da6eac4aacd4a6f347338bc99e7b27455cad8886 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Fri, 25 Jul 2025 09:18:54 -0600 Subject: [PATCH 03/32] passing tests added necessary provider models to model prices and context window files --- litellm/__init__.py | 5 + .../get_llm_provider_logic.py | 7 ++ litellm/llms/heroku/chat/transformation.py | 92 +++++++++---------- ...odel_prices_and_context_window_backup.json | 20 ++++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 20 ++++ .../heroku/test_heroku_chat_transformation.py | 64 +++++++++++-- 8 files changed, 157 insertions(+), 54 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 138a84493c0..c1fe040a371 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -501,6 +501,7 @@ nebius_models: List = [] nebius_embedding_models: List = [] deepgram_models: List = [] elevenlabs_models: List = [] +heroku_models: List = [] dashscope_models: List = [] moonshot_models: List = [] v0_models: List = [] @@ -681,6 +682,8 @@ def add_known_models(): deepgram_models.append(key) elif value.get("litellm_provider") == "elevenlabs": elevenlabs_models.append(key) + elif value.get("litellm_provider") == "heroku": + heroku_models.append(key) elif value.get("litellm_provider") == "dashscope": dashscope_models.append(key) elif value.get("litellm_provider") == "moonshot": @@ -775,6 +778,7 @@ model_list = ( + nscale_models + deepgram_models + elevenlabs_models + + heroku_models + dashscope_models + moonshot_models + v0_models @@ -846,6 +850,7 @@ models_by_provider: dict = { "featherless_ai": featherless_ai_models, "deepgram": deepgram_models, "elevenlabs": elevenlabs_models, + "heroku": heroku_models, "dashscope": dashscope_models, "moonshot": moonshot_models, "v0": v0_models, diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index e8f1477b90c..88db7f6dbcd 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -672,6 +672,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.NscaleConfig()._get_openai_compatible_provider_info( api_base=api_base, api_key=api_key ) + elif custom_llm_provider == "heroku": + ( + api_base, + dynamic_api_key, + ) = litellm.HerokuChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "dashscope": ( api_base, diff --git a/litellm/llms/heroku/chat/transformation.py b/litellm/llms/heroku/chat/transformation.py index 6626de69009..a1c865af747 100644 --- a/litellm/llms/heroku/chat/transformation.py +++ b/litellm/llms/heroku/chat/transformation.py @@ -5,7 +5,11 @@ this is OpenAI compatible - no translation needed / occurs """ import os -from typing import Optional, List, Union +from typing import Optional, List, Tuple, Union, Coroutine, Any, Literal, overload +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + handle_messages_with_content_list_to_str_conversion, +) +from litellm.types.llms.openai import AllMessageValues from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig # Base error class for Heroku @@ -13,59 +17,53 @@ class HerokuError(Exception): pass class HerokuChatConfig(OpenAIGPTConfig): - max_tokens: Optional[int] = None - stop: Optional[List[str]] = None - stream: Optional[bool] = None - temperature: Optional[float] = None - tool_choice: Optional[str] = None - tools: Optional[list] = None - top_p: Optional[int] = None + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[True] + ) -> Coroutine[Any, Any, List[AllMessageValues]]: + ... - def __init__( + @overload + def _transform_messages( self, - max_tokens: Optional[int] = None, - stop: Optional[List[str]] = None, - stream: Optional[bool] = None, - temperature: Optional[float] = None, - tool_choice: Optional[str] = None, - tools: Optional[list] = None, - top_p: Optional[int] = None, - ) -> None: - locals_ = locals().copy() - for key, value in locals_.items(): - if key != "self" and value is not None: - setattr(self.__class__, key, value) - - def get_supported_openai_params(self, model: str) -> list: - return [ - "max_tokens", - "stop", - "stream", - "temperature", - "tool_choice", - "top_p", - "tools", - ] - - def map_openai_params( - self, - non_default_params: dict, - optional_params: dict, + messages: List[AllMessageValues], model: str, - drop_params: bool, - ) -> dict: - supported_openai_params = self.get_supported_openai_params(model=model) - for param, value in non_default_params.items(): - if param in supported_openai_params: - optional_params[param] = value - return optional_params - - def get_complete_url(self, api_base: Optional[str], api_key: Optional[str], model: str, optional_params: dict, litellm_params: dict, stream: Optional[bool] = None) -> str: + is_async: Literal[False] = False, + ) -> List[AllMessageValues]: + ... + + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: bool = False + ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: + """ + Heroku does not support content in list format. + See: https://devcenter.heroku.com/articles/heroku-inference-api-v1-chat-completions#content-object + """ + messages = handle_messages_with_content_list_to_str_conversion(messages) + if is_async: + return super()._transform_messages( + messages=messages, model=model, is_async=True + ) + else: + return super()._transform_messages( + messages=messages, model=model, is_async=False + ) + + def _get_openai_compatible_provider_info(self, api_base: Optional[str], api_key: Optional[str]) -> Tuple[Optional[str], Optional[str]]: api_base = api_base or os.getenv("HEROKU_API_BASE") if not api_base: raise HerokuError("No api base was set. Please provide an api_base, or set the HEROKU_API_BASE environment variable.") + api_key = api_key or os.getenv("HEROKU_API_KEY") + if not api_key: + raise HerokuError("No api key was set. Please provide an api_key, or set the HEROKU_API_KEY environment variable.") + + return api_base, api_key + + def get_complete_url(self, api_base: Optional[str], api_key: Optional[str], model: str, optional_params: dict, litellm_params: dict, stream: Optional[bool] = None) -> str: + api_base, _ = self._get_openai_compatible_provider_info(api_base, api_key) + if not api_base.endswith("/v1/chat/completions"): api_base = f"{api_base}/v1/chat/completions" - + return api_base \ No newline at end of file diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2ab758e722b..c68c1093a4b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16670,5 +16670,25 @@ "supports_vision": false, "supports_system_messages": true, "supports_tool_choice": false + }, + "heroku/claude-4-sonnet": { + "max_tokens": 8192, + "litellm_provider": "heroku", + "mode": "chat" + }, + "heroku/claude-3-7-sonnet": { + "max_tokens": 8192, + "litellm_provider": "heroku", + "mode": "chat" + }, + "heroku/claude-3-5-sonnet-latest": { + "max_tokens": 8192, + "litellm_provider": "heroku", + "mode": "chat" + }, + "heroku/claude-3-5-haiku": { + "max_tokens": 4096, + "litellm_provider": "heroku", + "mode": "chat" } } diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 88d58582d0c..899b15a78fb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2314,6 +2314,7 @@ class LlmProviders(str, Enum): NSCALE = "nscale" PG_VECTOR = "pg_vector" RECRAFT = "recraft" + HEROKU = "heroku" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 676e40f6b2d..9f8b2dcc587 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6880,6 +6880,8 @@ class ProviderConfigManager: return litellm.OpenAIGPTConfig() elif litellm.LlmProviders.NSCALE == provider: return litellm.NscaleConfig() + elif litellm.LlmProviders.HEROKU == provider: + return litellm.HerokuChatConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2ab758e722b..c68c1093a4b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16670,5 +16670,25 @@ "supports_vision": false, "supports_system_messages": true, "supports_tool_choice": false + }, + "heroku/claude-4-sonnet": { + "max_tokens": 8192, + "litellm_provider": "heroku", + "mode": "chat" + }, + "heroku/claude-3-7-sonnet": { + "max_tokens": 8192, + "litellm_provider": "heroku", + "mode": "chat" + }, + "heroku/claude-3-5-sonnet-latest": { + "max_tokens": 8192, + "litellm_provider": "heroku", + "mode": "chat" + }, + "heroku/claude-3-5-haiku": { + "max_tokens": 4096, + "litellm_provider": "heroku", + "mode": "chat" } } diff --git a/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py b/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py index 140105c2a26..4359bce3f70 100644 --- a/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py +++ b/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py @@ -1,9 +1,14 @@ import os import pytest +import litellm +from litellm import completion from litellm.llms.custom_httpx.http_handler import HTTPHandler from unittest.mock import patch from litellm.llms.heroku.chat.transformation import HerokuChatConfig +os.environ["HEROKU_API_BASE"] = "https://us.inference.heroku.com" +os.environ["HEROKU_API_KEY"] = "fake-heroku-key" + class TestHerokuChatConfig: def test_default_api_base(self): """Test that default API base is used when none is provided""" @@ -22,12 +27,57 @@ class TestHerokuChatConfig: api_base=None, # Not providing api_base ) - # set env var for api_base - os.environ["HEROKU_API_BASE"] = "https://mia.heroku.com" - - print('****************************************') - print(config.get_complete_url(api_base=None, api_key=api_key, model="claude-3-5-haiku", optional_params={}, litellm_params={}, stream=False)) - print('****************************************') # Verify headers are still set correctly assert result["Authorization"] == f"Bearer {api_key}" - assert result["Content-Type"] == "application/json" \ No newline at end of file + assert result["Content-Type"] == "application/json" + + @pytest.mark.respx() + def test_heroku_chat_mock(self, respx_mock): + """Test that the Heroku chat API is called correctly""" + + litellm.disable_aiohttp_transport = True + + model = "heroku/claude-3-5-haiku" + model_name = "claude-3-5-haiku" + + respx_mock.post("https://us.inference.heroku.com/v1/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": model_name, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "It's me, Mia! How are you?", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21, + }, + }, + status_code=200, + ) + + response = completion( + model=model, + messages=[ + {"role": "user", "content": "write code for saying hey from LiteLLM"} + ], + extended_thinking={ "enabled": True, "include_reasoning":True } + ) + + # Verify the request was made with correct headers + assert len(respx_mock.calls) == 1 + request = respx_mock.calls[0].request + + assert request.headers["Authorization"] == f"Bearer {os.environ['HEROKU_API_KEY']}" + assert request.headers["Content-Type"] == "application/json" + + assert response.choices[0].message.content == "It's me, Mia! How are you?" \ No newline at end of file From 79e43088097066496b44f2121c1fb87d3980ab5d Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Fri, 25 Jul 2025 16:02:26 -0600 Subject: [PATCH 04/32] adds provider docs --- docs/my-website/docs/providers/heroku.md | 77 ++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 docs/my-website/docs/providers/heroku.md diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md new file mode 100644 index 00000000000..d10387f958a --- /dev/null +++ b/docs/my-website/docs/providers/heroku.md @@ -0,0 +1,77 @@ +# Heroku + +## Provision a Model + +To use the Heroku provider for LiteLLM, you must first configure a Heroku app, and attach one of the models listed in the [Supported Models](#supported-models) section. + +To get configure a Heroku app with an attached model, please refer to [Heroku's documentation](https://devcenter.heroku.com/articles/heroku-inference). + +## Supported Models + +The Heroku provider for LiteLLM currently, only supports the following models for the [`v1/chat/completions`](https://devcenter.heroku.com/articles/heroku-inference-api-v1-chat-completions) endpoint: + +| Model | Region | +|-----------------------------------|---------| +| [`heroku/claude-sonnet-4`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-4-sonnet) | US, EU | +| [`heroku/claude-3-7-sonnet`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-7-sonnet) | US, EU | +| [`heroku/claude-3-5-sonnet-latest`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-5-sonnet-latest) | US | +| [`heroku/claude-3-5-haiku`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-5-haiku) | US | +| [`heroku/claude-3`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-haiku) | EU | + +## Environment Variables + +When a model is attached to a Heroku app, three config variables are set: + +- `INFERENCE_KEY`: The API key used for authenticating requests to the model. +- `INFERENCE_MODEL_ID`: The name of the model. E.g. `claude-3-5-haiku`. +- `INFERENCE_URL`: The base URL for calling the model. + +It is important to note that the values for `INFERENCE_KEY` and `INFERENCE_URL` will be required for making calls to your model. More details follow in the [Usage Examples](#usage-examples) section. + +For a deeper explanation of these variables, see the official [Heroku documentation](https://devcenter.heroku.com/articles/heroku-inference#model-resource-config-vars). + +## Usage Examples +### Using Config Variables + +The Heroku provider is aware of the following config variables, and will use them, if present: + +- `HEROKU_API_KEY`: This value corresponds to the [`api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this to the value of Heroku's `INFERENCE_KEY` config variable. +- `HEROKU_API_BASE`: This value corresponds to the [`api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this to the value of Heroku's `INFERENCE_URL` config variable. + +In this example, we don't explicitly pass the `api_key` and `api_base`. We, instead, set the config variables which will be used by the Heroku provider. + +```python +import os +from litellm import completion + +os.environ["HEROKU_API_BASE"] = "https://us.inference.heroku.com" +os.environ["HEROKU_API_KEY"] = "fake-heroku-key" + +response = completion( + model="heroku/claude-3-5-haiku", + messages=[ + {"role": "user", "content": "write code for saying hey from LiteLLM"} + ] +) + +print(response) +``` + +### Explicitly Setting `api_key` and `api_base` + +```python +from litellm import completion + +response = completion( + model="heroku/claude-sonnet-4", + api_key="fake-heroku-key", + api_base="https://us.inference.heroku.com", + messages=[ + {"role": "user", "content": "write code for saying hey from LiteLLM"} + ], +) +``` + +## Misc + +Note that in both of the above examples, the model name has the `heroku/` prefix. This is necessary, as it allows LiteLLM to know what model provider to use. \ No newline at end of file From 0eb05ad23fd26c1ecfa55e7c1be4e17cb890ceb7 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Fri, 25 Jul 2025 16:11:39 -0600 Subject: [PATCH 05/32] adds sidebar link to Heroku provider docs --- docs/my-website/sidebars.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 54ff6461c69..c8ddce32ea2 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -461,7 +461,8 @@ const sidebars = { "providers/featherless_ai", "providers/nebius", "providers/dashscope", - "providers/bytez" + "providers/bytez", + "providers/heroku" ], }, { From c51b8d0a6df6e7f1ca3edc86f1c2e816a48902d8 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Fri, 25 Jul 2025 16:14:36 -0600 Subject: [PATCH 06/32] doc edits --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index d10387f958a..24974b0fa0e 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -8,7 +8,7 @@ To get configure a Heroku app with an attached model, please refer to [Heroku's ## Supported Models -The Heroku provider for LiteLLM currently, only supports the following models for the [`v1/chat/completions`](https://devcenter.heroku.com/articles/heroku-inference-api-v1-chat-completions) endpoint: +The Heroku provider for LiteLLM currently, only supports [chat](https://devcenter.heroku.com/articles/heroku-inference-api-v1-chat-completions). Supported chat models are: | Model | Region | |-----------------------------------|---------| From dd109e7c371536c58366573e900f44bdfe7522e6 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Fri, 25 Jul 2025 16:28:56 -0600 Subject: [PATCH 07/32] adds heroku to supported providers table --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 528dd53581c..9a7c1a6c96e 100644 --- a/README.md +++ b/README.md @@ -343,6 +343,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | | | [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | | | [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [Heroku](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | | | | | [**Read the Docs**](https://docs.litellm.ai/docs/) From a7abcf22766b7b4431ba2dc049af05a32de3da75 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Mon, 28 Jul 2025 09:43:15 -0600 Subject: [PATCH 08/32] fixes linter error --- litellm/llms/heroku/chat/transformation.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/litellm/llms/heroku/chat/transformation.py b/litellm/llms/heroku/chat/transformation.py index a1c865af747..a64d8afe63a 100644 --- a/litellm/llms/heroku/chat/transformation.py +++ b/litellm/llms/heroku/chat/transformation.py @@ -51,17 +51,15 @@ class HerokuChatConfig(OpenAIGPTConfig): def _get_openai_compatible_provider_info(self, api_base: Optional[str], api_key: Optional[str]) -> Tuple[Optional[str], Optional[str]]: api_base = api_base or os.getenv("HEROKU_API_BASE") - if not api_base: - raise HerokuError("No api base was set. Please provide an api_base, or set the HEROKU_API_BASE environment variable.") - api_key = api_key or os.getenv("HEROKU_API_KEY") - if not api_key: - raise HerokuError("No api key was set. Please provide an api_key, or set the HEROKU_API_KEY environment variable.") - + return api_base, api_key def get_complete_url(self, api_base: Optional[str], api_key: Optional[str], model: str, optional_params: dict, litellm_params: dict, stream: Optional[bool] = None) -> str: api_base, _ = self._get_openai_compatible_provider_info(api_base, api_key) + + if not api_base: + raise HerokuError("No api base was set. Please provide an api_base, or set the HEROKU_API_BASE environment variable.") if not api_base.endswith("/v1/chat/completions"): api_base = f"{api_base}/v1/chat/completions" From 555579f42bbc051984dbded0522e4ce3eb876394 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Wed, 6 Aug 2025 14:23:55 -0600 Subject: [PATCH 09/32] adds tool calling test --- .../heroku/test_heroku_chat_transformation.py | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py b/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py index 4359bce3f70..f70392db040 100644 --- a/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py +++ b/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py @@ -80,4 +80,87 @@ class TestHerokuChatConfig: assert request.headers["Authorization"] == f"Bearer {os.environ['HEROKU_API_KEY']}" assert request.headers["Content-Type"] == "application/json" - assert response.choices[0].message.content == "It's me, Mia! How are you?" \ No newline at end of file + assert response.choices[0].message.content == "It's me, Mia! How are you?" + + @pytest.mark.respx() + def test_heroku_tool_calling(self, respx_mock): + """Test that the Heroku tool calling API is called correctly""" + config = HerokuChatConfig() + headers = {} + api_key = "fake-heroku-key" + + litellm.disable_aiohttp_transport = True + + model = "heroku/claude-4-sonnet" + + respx_mock.post("https://us.inference.heroku.com/v1/chat/completions").respond( + json={ + "id": "chatcmpl-1859428879fc791b17d73", + "object": "chat.completion", + "created": 1754506683, + "model": "claude-4-sonnet", + "system_fingerprint": "heroku-inf-cp42st", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "refusal": None, + "tool_calls": [ + { + "id": "tooluse_dV3Vtnb-S9-Z_YFicSv2Gw", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\":\"Portland, OR\"}" + } + } + ], + "content": "Let me check the current weather in Portland for you." + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 354, + "completion_tokens": 69, + "total_tokens": 423 + } + }, + status_code=200, + ) + + response = completion( + model=model, + messages=[{"role": "user", "content": "What's the weather in Portland?"}], + tools=[{ + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. Portland, OR" + } + }, + "required": [ + "location" + ] + } + } + }], + tool_choice="auto", + ) + print(response) + assert response.choices[0].message.content == "Let me check the current weather in Portland for you." + assert response.choices[0].message.tool_calls[0].id == "tooluse_dV3Vtnb-S9-Z_YFicSv2Gw" + assert response.choices[0].message.tool_calls[0].type == "function" + assert response.choices[0].message.tool_calls[0].function.name == "get_current_weather" + assert response.choices[0].message.tool_calls[0].function.arguments == "{\"location\":\"Portland, OR\"}" + + assert response.usage.prompt_tokens == 354 + assert response.usage.completion_tokens == 69 + assert response.usage.total_tokens == 423 \ No newline at end of file From ecda9b1f22c051429bf94000d93623bfe04b49d1 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Wed, 6 Aug 2025 14:30:14 -0600 Subject: [PATCH 10/32] removes redefinition of GroqChatCompletion --- litellm/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 41be77973b9..f9be5ef8e2f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -151,7 +151,6 @@ from .llms.deprecated_providers import aleph_alpha, palm from .llms.groq.chat.handler import GroqChatCompletion from .llms.heroku.chat.transformation import HerokuChatConfig from .llms.gemini.common_utils import get_api_key_from_env -from .llms.groq.chat.handler import GroqChatCompletion from .llms.huggingface.embedding.handler import HuggingFaceEmbedding from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion from .llms.oci.chat.transformation import OCIChatConfig From 791c6126a451075c87cf3e8edf5f36a4dce73147 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Wed, 6 Aug 2025 15:14:36 -0600 Subject: [PATCH 11/32] updates price and context window for heroku models --- ...odel_prices_and_context_window_backup.json | 20 +++++++++++++++---- model_prices_and_context_window.json | 20 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0de4a0f1182..786fd578ce8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -17639,21 +17639,33 @@ "heroku/claude-4-sonnet": { "max_tokens": 8192, "litellm_provider": "heroku", - "mode": "chat" + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "heroku/claude-3-7-sonnet": { "max_tokens": 8192, "litellm_provider": "heroku", - "mode": "chat" + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "heroku/claude-3-5-sonnet-latest": { "max_tokens": 8192, "litellm_provider": "heroku", - "mode": "chat" + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "heroku/claude-3-5-haiku": { "max_tokens": 4096, "litellm_provider": "heroku", - "mode": "chat" + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0de4a0f1182..786fd578ce8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -17639,21 +17639,33 @@ "heroku/claude-4-sonnet": { "max_tokens": 8192, "litellm_provider": "heroku", - "mode": "chat" + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "heroku/claude-3-7-sonnet": { "max_tokens": 8192, "litellm_provider": "heroku", - "mode": "chat" + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "heroku/claude-3-5-sonnet-latest": { "max_tokens": 8192, "litellm_provider": "heroku", - "mode": "chat" + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true }, "heroku/claude-3-5-haiku": { "max_tokens": 4096, "litellm_provider": "heroku", - "mode": "chat" + "mode": "chat", + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true } } From 51b52534fb4f50dfb4f4190335f3867643ed29f0 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Mon, 11 Aug 2025 09:40:40 -0600 Subject: [PATCH 12/32] fixes misconfigured OCI models by setting supports_tool_choice: true --- ...odel_prices_and_context_window_backup.json | 20 +++++++++---------- model_prices_and_context_window.json | 20 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2502ce56350..651f394f7a9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18019,7 +18019,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/meta.llama-4-scout-17b-16e-instruct": { @@ -18032,7 +18032,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/meta.llama-3.3-70b-instruct": { @@ -18045,7 +18045,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/meta.llama-3.2-90b-vision-instruct": { @@ -18058,7 +18058,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/meta.llama-3.1-405b-instruct": { @@ -18071,7 +18071,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, @@ -18085,7 +18085,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/xai.grok-3": { @@ -18098,7 +18098,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/xai.grok-3-mini": { @@ -18111,7 +18111,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/xai.grok-3-fast": { @@ -18124,7 +18124,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/xai.grok-3-mini-fast": { @@ -18137,7 +18137,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2502ce56350..651f394f7a9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18019,7 +18019,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/meta.llama-4-scout-17b-16e-instruct": { @@ -18032,7 +18032,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/meta.llama-3.3-70b-instruct": { @@ -18045,7 +18045,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/meta.llama-3.2-90b-vision-instruct": { @@ -18058,7 +18058,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/meta.llama-3.1-405b-instruct": { @@ -18071,7 +18071,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, @@ -18085,7 +18085,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/xai.grok-3": { @@ -18098,7 +18098,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/xai.grok-3-mini": { @@ -18111,7 +18111,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/xai.grok-3-fast": { @@ -18124,7 +18124,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" }, "oci/xai.grok-3-mini-fast": { @@ -18137,7 +18137,7 @@ "mode": "chat", "supports_function_calling": true, "supports_response_schema": false, - "supports_tool_choice": false, + "supports_tool_choice": true, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing" } } From cc3ec33fe9ea6f7e27168d1f2de47c988f0af27e Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Mon, 11 Aug 2025 09:52:20 -0600 Subject: [PATCH 13/32] removes unused imports that are causing linting failures --- litellm/caching/caching_handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index f41b745bb1c..7580752c307 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -18,7 +18,6 @@ import asyncio import datetime import inspect import threading -from functools import lru_cache, wraps from typing import ( TYPE_CHECKING, Any, @@ -36,7 +35,6 @@ from pydantic import BaseModel import litellm from litellm._logging import print_verbose, verbose_logger -from litellm._service_logger import ServiceLogging from litellm.caching import InMemoryCache from litellm.caching.caching import S3Cache from litellm.litellm_core_utils.logging_utils import ( From e2165ff76e72b937951b943ef0a8da3b95eb8f7b Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:40:43 -0600 Subject: [PATCH 14/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 24974b0fa0e..204c81ba3d9 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -2,7 +2,7 @@ ## Provision a Model -To use the Heroku provider for LiteLLM, you must first configure a Heroku app, and attach one of the models listed in the [Supported Models](#supported-models) section. +To use Heroku with LiteLLM, [configure a Heroku app and attach a supported model](https://devcenter.heroku.com/articles/heroku-inference#provision-access-to-an-ai-model-resource). To get configure a Heroku app with an attached model, please refer to [Heroku's documentation](https://devcenter.heroku.com/articles/heroku-inference). From b20b28a912b41042d4b4a43d3bb71b02f978f4d2 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:40:54 -0600 Subject: [PATCH 15/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 204c81ba3d9..19117f6167e 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -4,7 +4,6 @@ To use Heroku with LiteLLM, [configure a Heroku app and attach a supported model](https://devcenter.heroku.com/articles/heroku-inference#provision-access-to-an-ai-model-resource). -To get configure a Heroku app with an attached model, please refer to [Heroku's documentation](https://devcenter.heroku.com/articles/heroku-inference). ## Supported Models From e97fa833b329ab1a82b096f5c33711ca08e8c37a Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:41:08 -0600 Subject: [PATCH 16/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 19117f6167e..6ef7dc6316e 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -7,7 +7,7 @@ To use Heroku with LiteLLM, [configure a Heroku app and attach a supported model ## Supported Models -The Heroku provider for LiteLLM currently, only supports [chat](https://devcenter.heroku.com/articles/heroku-inference-api-v1-chat-completions). Supported chat models are: +Heroku for LiteLLM supports various [chat](https://devcenter.heroku.com/articles/heroku-inference-api-v1-chat-completions) models: | Model | Region | |-----------------------------------|---------| From 00c9f1fe0e84f154e697e9f031c90213bd168910 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:41:22 -0600 Subject: [PATCH 17/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 6ef7dc6316e..66b56c93c0f 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -19,7 +19,7 @@ Heroku for LiteLLM supports various [chat](https://devcenter.heroku.com/articles ## Environment Variables -When a model is attached to a Heroku app, three config variables are set: +When you attach a model to a Heroku app, three config variables are set: - `INFERENCE_KEY`: The API key used for authenticating requests to the model. - `INFERENCE_MODEL_ID`: The name of the model. E.g. `claude-3-5-haiku`. From 1be33ec47f04d9323a2202507cb023cfc4e78ec7 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:41:38 -0600 Subject: [PATCH 18/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 66b56c93c0f..4c09c2bb05b 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -22,7 +22,7 @@ Heroku for LiteLLM supports various [chat](https://devcenter.heroku.com/articles When you attach a model to a Heroku app, three config variables are set: - `INFERENCE_KEY`: The API key used for authenticating requests to the model. -- `INFERENCE_MODEL_ID`: The name of the model. E.g. `claude-3-5-haiku`. +- `INFERENCE_MODEL_ID`: The name of the model, e.g. `claude-3-5-haiku`. - `INFERENCE_URL`: The base URL for calling the model. It is important to note that the values for `INFERENCE_KEY` and `INFERENCE_URL` will be required for making calls to your model. More details follow in the [Usage Examples](#usage-examples) section. From fa587c2ebaaeba8aceb41d9d8f91821c6bcc6da6 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:42:27 -0600 Subject: [PATCH 19/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 4c09c2bb05b..10ac001e918 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -25,7 +25,7 @@ When you attach a model to a Heroku app, three config variables are set: - `INFERENCE_MODEL_ID`: The name of the model, e.g. `claude-3-5-haiku`. - `INFERENCE_URL`: The base URL for calling the model. -It is important to note that the values for `INFERENCE_KEY` and `INFERENCE_URL` will be required for making calls to your model. More details follow in the [Usage Examples](#usage-examples) section. +Both `INFERENCE_KEY` and `INFERENCE_URL` are required to make calls to your model. For a deeper explanation of these variables, see the official [Heroku documentation](https://devcenter.heroku.com/articles/heroku-inference#model-resource-config-vars). From 417b18a7273354c11dec9bdfec4696e7ee1ebdb0 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:42:46 -0600 Subject: [PATCH 20/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 10ac001e918..6ba8af4e899 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -27,7 +27,7 @@ When you attach a model to a Heroku app, three config variables are set: Both `INFERENCE_KEY` and `INFERENCE_URL` are required to make calls to your model. -For a deeper explanation of these variables, see the official [Heroku documentation](https://devcenter.heroku.com/articles/heroku-inference#model-resource-config-vars). +For more information on these variables, see the [Heroku documentation](https://devcenter.heroku.com/articles/heroku-inference#model-resource-config-vars). ## Usage Examples ### Using Config Variables From 3cfb85228f3eda89831d60a6a77a9a6a164f6fa4 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:44:52 -0600 Subject: [PATCH 21/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 6ba8af4e899..98464f75d6f 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -37,7 +37,7 @@ The Heroku provider is aware of the following config variables, and will use the - `HEROKU_API_KEY`: This value corresponds to the [`api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this to the value of Heroku's `INFERENCE_KEY` config variable. - `HEROKU_API_BASE`: This value corresponds to the [`api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this to the value of Heroku's `INFERENCE_URL` config variable. -In this example, we don't explicitly pass the `api_key` and `api_base`. We, instead, set the config variables which will be used by the Heroku provider. +In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which will be used by Heroku: ```python import os From 5857d17fadfb1a7efa7c24025e8619e804f155c5 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:45:06 -0600 Subject: [PATCH 22/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 98464f75d6f..1b61f96caf2 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -32,7 +32,7 @@ For more information on these variables, see the [Heroku documentation](https:// ## Usage Examples ### Using Config Variables -The Heroku provider is aware of the following config variables, and will use them, if present: +Heroku uses the following LiteLLM API config variables: - `HEROKU_API_KEY`: This value corresponds to the [`api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this to the value of Heroku's `INFERENCE_KEY` config variable. - `HEROKU_API_BASE`: This value corresponds to the [`api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this to the value of Heroku's `INFERENCE_URL` config variable. From d4e320a4ceb3918919fd9b74767c6b542e849f11 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:45:21 -0600 Subject: [PATCH 23/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 1b61f96caf2..db85d442315 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -71,6 +71,8 @@ response = completion( ) ``` +> Include the `heroku/` prefix in the model name so LiteLLM knows the model provider to use. + ## Misc Note that in both of the above examples, the model name has the `heroku/` prefix. This is necessary, as it allows LiteLLM to know what model provider to use. \ No newline at end of file From 7b0ecb1847c997fab9645cddf4caf27c218a4848 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:45:36 -0600 Subject: [PATCH 24/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index db85d442315..f3350e46d43 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -56,6 +56,8 @@ response = completion( print(response) ``` +> Include the `heroku/` prefix in the model name so LiteLLM knows the model provider to use. + ### Explicitly Setting `api_key` and `api_base` ```python From 586c0a7be57cb742164287d4ba4b140ffcfc7fd4 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:45:47 -0600 Subject: [PATCH 25/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index f3350e46d43..64ffe8c998a 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -76,5 +76,3 @@ response = completion( > Include the `heroku/` prefix in the model name so LiteLLM knows the model provider to use. ## Misc - -Note that in both of the above examples, the model name has the `heroku/` prefix. This is necessary, as it allows LiteLLM to know what model provider to use. \ No newline at end of file From 1342abf16325e28a703ac9f76269bef75596136a Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:46:16 -0600 Subject: [PATCH 26/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 64ffe8c998a..45584b2cb8b 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -34,7 +34,7 @@ For more information on these variables, see the [Heroku documentation](https:// Heroku uses the following LiteLLM API config variables: -- `HEROKU_API_KEY`: This value corresponds to the [`api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this to the value of Heroku's `INFERENCE_KEY` config variable. +- `HEROKU_API_KEY`: This value corresponds to [LiteLLM's `api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this to the value of Heroku's `INFERENCE_KEY` config variable. - `HEROKU_API_BASE`: This value corresponds to the [`api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this to the value of Heroku's `INFERENCE_URL` config variable. In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which will be used by Heroku: From cccf41c0dfad45882da5c016ba3eb04b361ca185 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:46:48 -0600 Subject: [PATCH 27/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 45584b2cb8b..0a96ef2d571 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -35,7 +35,7 @@ For more information on these variables, see the [Heroku documentation](https:// Heroku uses the following LiteLLM API config variables: - `HEROKU_API_KEY`: This value corresponds to [LiteLLM's `api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this to the value of Heroku's `INFERENCE_KEY` config variable. -- `HEROKU_API_BASE`: This value corresponds to the [`api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this to the value of Heroku's `INFERENCE_URL` config variable. +- `HEROKU_API_BASE`: This value corresponds to [LiteLLM's `api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this to the value of Heroku's `INFERENCE_URL` config variable. In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which will be used by Heroku: From bb259a651c4def811ca77fcdbb2f092ee8466565 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore Date: Tue, 12 Aug 2025 09:11:16 -0600 Subject: [PATCH 28/32] removes extraneous heading from readme --- docs/my-website/docs/providers/heroku.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 0a96ef2d571..cfdf84f417b 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -74,5 +74,3 @@ response = completion( ``` > Include the `heroku/` prefix in the model name so LiteLLM knows the model provider to use. - -## Misc From ede74a66d27b3bf4ad5e0acc075b92d38e599b45 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:13:11 -0600 Subject: [PATCH 29/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index cfdf84f417b..5110961f95d 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -22,7 +22,7 @@ Heroku for LiteLLM supports various [chat](https://devcenter.heroku.com/articles When you attach a model to a Heroku app, three config variables are set: - `INFERENCE_KEY`: The API key used for authenticating requests to the model. -- `INFERENCE_MODEL_ID`: The name of the model, e.g. `claude-3-5-haiku`. +- `INFERENCE_MODEL_ID`: The name of the model, for example`claude-3-5-haiku`. - `INFERENCE_URL`: The base URL for calling the model. Both `INFERENCE_KEY` and `INFERENCE_URL` are required to make calls to your model. From fffe6cb6fb133dd531bf75085a6e548300c7503f Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:13:20 -0600 Subject: [PATCH 30/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 5110961f95d..18279142e66 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -34,7 +34,7 @@ For more information on these variables, see the [Heroku documentation](https:// Heroku uses the following LiteLLM API config variables: -- `HEROKU_API_KEY`: This value corresponds to [LiteLLM's `api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this to the value of Heroku's `INFERENCE_KEY` config variable. +- `HEROKU_API_KEY`: This value corresponds to [LiteLLM's `api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this variable to the value of Heroku's `INFERENCE_KEY` config variable. - `HEROKU_API_BASE`: This value corresponds to [LiteLLM's `api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this to the value of Heroku's `INFERENCE_URL` config variable. In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which will be used by Heroku: From 50de2a09e4ef79c803fe49457103104903adc0b5 Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:18:26 -0600 Subject: [PATCH 31/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index 18279142e66..a03837b3514 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -35,7 +35,7 @@ For more information on these variables, see the [Heroku documentation](https:// Heroku uses the following LiteLLM API config variables: - `HEROKU_API_KEY`: This value corresponds to [LiteLLM's `api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this variable to the value of Heroku's `INFERENCE_KEY` config variable. -- `HEROKU_API_BASE`: This value corresponds to [LiteLLM's `api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this to the value of Heroku's `INFERENCE_URL` config variable. +- `HEROKU_API_BASE`: This value corresponds to [LiteLLM's `api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this variable to the value of Heroku's `INFERENCE_URL` config variable. In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which will be used by Heroku: From a9d4f45a103a8b8f44ae1242b007dd8f2141e21b Mon Sep 17 00:00:00 2001 From: Timothy Lowrimore <154477569+tlowrimore-heroku@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:18:37 -0600 Subject: [PATCH 32/32] Update docs/my-website/docs/providers/heroku.md Co-authored-by: Claire Riley --- docs/my-website/docs/providers/heroku.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md index a03837b3514..bf37ed64b19 100644 --- a/docs/my-website/docs/providers/heroku.md +++ b/docs/my-website/docs/providers/heroku.md @@ -37,7 +37,7 @@ Heroku uses the following LiteLLM API config variables: - `HEROKU_API_KEY`: This value corresponds to [LiteLLM's `api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this variable to the value of Heroku's `INFERENCE_KEY` config variable. - `HEROKU_API_BASE`: This value corresponds to [LiteLLM's `api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this variable to the value of Heroku's `INFERENCE_URL` config variable. -In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which will be used by Heroku: +In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which Heroku will use: ```python import os