mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(huggingface): honor HUGGINGFACE_API_BASE and env endpoints for chat (#51449)
* fix(huggingface): honor HUGGINGFACE_API_BASE and env endpoints for chat With only HUGGINGFACE_API_BASE set, the chat URL became the string 'None'. With either HF_API_BASE or HUGGINGFACE_API_BASE set, a model name without a slash such as huggingface/tgi failed with 'not enough values to unpack' because the request was still routed through the Inference Providers mapping. Treat an endpoint from the environment the same way as an explicit api_base. * fix(huggingface): keep string image_url conversion when the endpoint comes from HF_API_BASE * fix(huggingface): build the endpoint request once so the type gate count stays flat --------- Co-authored-by: Abelo9996 <Abelo9996@users.noreply.github.com> Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
3637bdaea2
commit
331a673114
2 changed files with 53 additions and 4 deletions
|
|
@ -24,6 +24,10 @@ logger: Final = logging.getLogger(__name__)
|
|||
BASE_URL: Final = "https://router.huggingface.co"
|
||||
|
||||
|
||||
def _api_base_from_env() -> str | None:
|
||||
return os.getenv("HF_API_BASE") or os.getenv("HUGGINGFACE_API_BASE")
|
||||
|
||||
|
||||
def _build_chat_completion_url(model_url: str) -> str:
|
||||
# Strip trailing /
|
||||
model_url = model_url.rstrip("/")
|
||||
|
|
@ -96,8 +100,8 @@ class HuggingFaceChatConfig(OpenAIGPTConfig):
|
|||
if api_base is not None:
|
||||
complete_url = api_base
|
||||
complete_url = _build_chat_completion_url(complete_url)
|
||||
elif os.getenv("HF_API_BASE") or os.getenv("HUGGINGFACE_API_BASE"):
|
||||
complete_url = str(os.getenv("HF_API_BASE")) or str(os.getenv("HUGGINGFACE_API_BASE"))
|
||||
elif env_api_base := _api_base_from_env():
|
||||
complete_url = env_api_base
|
||||
elif model.startswith(("http://", "https://")):
|
||||
complete_url = model
|
||||
complete_url = _build_chat_completion_url(complete_url)
|
||||
|
|
@ -129,8 +133,12 @@ class HuggingFaceChatConfig(OpenAIGPTConfig):
|
|||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
if litellm_params.get("api_base"):
|
||||
return dict(ChatCompletionRequest(model=model, messages=messages, **optional_params))
|
||||
lacks_explicit_api_base: Final = not litellm_params.get("api_base")
|
||||
if not lacks_explicit_api_base or _api_base_from_env():
|
||||
endpoint_messages: Final = (
|
||||
self._transform_messages(messages=messages, model=model) if lacks_explicit_api_base else messages
|
||||
)
|
||||
return dict(ChatCompletionRequest(model=model, messages=endpoint_messages, **optional_params))
|
||||
if "max_retries" in optional_params:
|
||||
logger.warning("`max_retries` is not supported. It will be ignored.")
|
||||
optional_params.pop("max_retries", None)
|
||||
|
|
|
|||
|
|
@ -427,6 +427,47 @@ def test_completion_with_api_base_uses_endpoint_url(respx_mock: respx.MockRouter
|
|||
assert response.choices[0].message.content == "Hugging Face response"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("env_var", ["HF_API_BASE", "HUGGINGFACE_API_BASE"])
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_completion_with_env_api_base_uses_endpoint_url(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, env_var: str
|
||||
) -> None:
|
||||
monkeypatch.setenv(env_var, "https://custom.huggingface.test/v1/chat/completions")
|
||||
route: Final = respx_mock.post("https://custom.huggingface.test/v1/chat/completions")
|
||||
route.return_value = httpx.Response(200, json=HUGGINGFACE_COMPLETION_RESPONSE)
|
||||
messages: Final = [{"role": "user", "content": "This is a test message"}]
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="huggingface/tgi",
|
||||
messages=messages,
|
||||
api_key="test-api-key",
|
||||
client=HTTPHandler(),
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {"model": "tgi", "messages": messages}
|
||||
assert response.choices[0].message.content == "Hugging Face response"
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_completion_with_env_api_base_converts_string_image_url(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("HF_API_BASE", "https://custom.huggingface.test/v1/chat/completions")
|
||||
route: Final = respx_mock.post("https://custom.huggingface.test/v1/chat/completions")
|
||||
route.return_value = httpx.Response(200, json=HUGGINGFACE_COMPLETION_RESPONSE)
|
||||
image_url: Final = "https://example.test/cat.png"
|
||||
|
||||
litellm.completion(
|
||||
model="huggingface/tgi",
|
||||
messages=[{"role": "user", "content": [{"type": "image_url", "image_url": image_url}]}],
|
||||
api_key="test-api-key",
|
||||
client=HTTPHandler(),
|
||||
)
|
||||
|
||||
sent_part: Final = json.loads(route.calls[0].request.content)["messages"][0]["content"][0]
|
||||
assert sent_part["image_url"] == {"url": image_url}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
async def test_async_completion_with_api_base_uses_endpoint_url(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue