scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

Merge pull request #38233 from BerriAI/litellm_together_api_host

Browse source 
fix(together_ai): default endpoints to api.together.ai instead of api.together.xyz
This commit is contained in:
Mateo Wang 2026-08-25 11:17:17 -07:00 • committed by GitHub
commit 41aa1caff1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 148 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
basedpyright-code-budget.json
View file

@ -1,6 +1,6 @@
{
"reportAny": {
"limit": 19955
"limit": 19949
},
"reportArgumentType": {
"limit": 2566
@ -54,7 +54,7 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5663
"limit": 5661
},
"reportMissingTypeArgument": {
"limit": 15555
@ -105,10 +105,10 @@
"limit": 109
},
"reportUnknownMemberType": {
"limit": 39011
"limit": 39009
},
"reportUnknownParameterType": {
"limit": 19885
"limit": 19883
},
"reportUnknownVariableType": {
"limit": 30569

1
litellm/constants.py
View file

@ -750,6 +750,7 @@ openai_compatible_endpoints: Final[list] = [
"api.groq.com/openai/v1",
"https://integrate.api.nvidia.com/v1",
"api.deepseek.com/v1",
"api.together.ai/v1",
"api.together.xyz/v1",
"app.empower.dev/api/v1",
"https://api.friendli.ai/serverless/v1",

10
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -272,6 +272,14 @@ def get_llm_provider(
elif endpoint == "api.deepseek.com/v1":
custom_llm_provider = "deepseek"
dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
elif endpoint == "api.together.ai/v1" or endpoint == "api.together.xyz/v1":
custom_llm_provider = "together_ai"
dynamic_api_key = api_key or (
get_secret_str("TOGETHER_API_KEY")
or get_secret_str("TOGETHER_AI_API_KEY")
or get_secret_str("TOGETHERAI_API_KEY")
or get_secret_str("TOGETHER_AI_TOKEN")
)
elif endpoint == "ollama.com":
custom_llm_provider = "ollama"
dynamic_api_key = get_secret_str("OLLAMA_API_KEY")
@ -707,7 +715,7 @@ def _get_openai_compatible_provider_info(
dynamic_api_key,
) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "together_ai":
api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.xyz/v1"
api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.ai/v1"
dynamic_api_key = api_key or (
get_secret_str("TOGETHER_API_KEY")
or get_secret_str("TOGETHER_AI_API_KEY")

43
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -643,49 +643,6 @@ def claude_2_1_pt(
return prompt
### TOGETHER AI
def get_model_info(token, model):
try:
headers: Final = {"Authorization": f"Bearer {token}"}
client: Final = HTTPHandler(concurrent_limit=1)
response: Final = client.get("https://api.together.xyz/models/info", headers=headers)
if response.status_code == 200:
model_info: Final = response.json()
for m in model_info:
if m["name"].lower().strip() == model.strip():
return m["config"].get("prompt_format", None), m["config"].get("chat_template", None)
return None, None
else:
return None, None
except Exception: # safely fail a prompt template request
return None, None
## OLD TOGETHER AI FLOW
# def format_prompt_togetherai(messages, prompt_format, chat_template):
# if prompt_format is None:
# return default_pt(messages)
# human_prompt, assistant_prompt = prompt_format.split("{prompt}")
# if chat_template is not None:
# prompt = hf_chat_template(
# model=None, messages=messages, chat_template=chat_template
# )
# elif prompt_format is not None:
# prompt = custom_prompt(
# role_dict={},
# messages=messages,
# initial_prompt_value=human_prompt,
# final_prompt_value=assistant_prompt,
# )
# else:
# prompt = default_pt(messages)
# return prompt
### IBM Granite

12
litellm/llms/together_ai/rerank/handler.py
View file

@ -16,11 +16,16 @@ from litellm.llms.together_ai.rerank.transformation import TogetherAIRerankConfi
from litellm.types.rerank import RerankRequest, RerankResponse
def _rerank_url(api_base: str) -> str:
return f"{api_base.rstrip('/')}/rerank"
class TogetherAIRerank(BaseLLM):
def rerank(
self,
model: str,
api_key: str,
api_base: str,
query: str,
documents: list[str | dict[str, Any]],
top_n: int | None = None,
@ -46,10 +51,10 @@ class TogetherAIRerank(BaseLLM):
raise ValueError("TogetherAI does not support max_chunks_per_doc")
if _is_async:
return self.async_rerank(request_data_dict, api_key) # Call async method
return self.async_rerank(request_data_dict, api_key, api_base)
response: Final = client.post(
"https://api.together.xyz/v1/rerank",
_rerank_url(api_base),
headers={
"accept": "application/json",
"content-type": "application/json",
@ -69,11 +74,12 @@ class TogetherAIRerank(BaseLLM):
self,
request_data_dict: dict[str, Any],
api_key: str,
api_base: str,
) -> RerankResponse:
client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.TOGETHER_AI) # Use async client
response: Final = await client.post(
"https://api.together.xyz/v1/rerank",
_rerank_url(api_base),
headers={
"accept": "application/json",
"content-type": "application/json",

3
litellm/rerank_api/main.py
View file

@ -277,6 +277,8 @@ def rerank(
if api_key is None:
raise ValueError("TogetherAI API key is required, please set 'TOGETHERAI_API_KEY' in your environment")
api_base = dynamic_api_base or optional_params.api_base or litellm.api_base or "https://api.together.ai/v1"
response = together_rerank.rerank(
model=model,
query=query,
@ -286,6 +288,7 @@ def rerank(
return_documents=return_documents,
max_chunks_per_doc=max_chunks_per_doc,
api_key=api_key,
api_base=api_base,
_is_async=_is_async,
)
elif _custom_llm_provider == litellm.LlmProviders.JINA_AI:

6
ruff-strict-budget.json
View file

@ -1,6 +1,6 @@
{
"ANN001": {
"limit": 3018
"limit": 3016
},
"ANN002": {
"limit": 71
@ -9,7 +9,7 @@
"limit": 827
},
"ANN201": {
"limit": 2016
"limit": 2015
},
"ANN202": {
"limit": 852
@ -57,7 +57,7 @@
"limit": 3
},
"BLE001": {
"limit": 2919
"limit": 2918
},
"C401": {
"limit": 8

51
tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py
View file

@ -133,3 +133,54 @@ class TestGetLlmProviderRejectsAttackerSmuggledApiBase:
assert provider == "groq"
assert dynamic_api_key == "server-real-groq-key"
class TestTogetherApiBaseResolvesProvider:
"""
Regression for the Together host migration: both the current
``api.together.ai`` host and the legacy ``api.together.xyz`` host must
resolve to ``together_ai`` when passed as ``api_base``. Before the fix
the endpoint list carried the legacy host but the provider-mapping
chain had no branch for it, so the match fell through with a None
provider and the deployment failed with "LLM Provider NOT provided".
"""
@pytest.mark.parametrize(
"api_base",
[
"https://api.together.ai/v1",
"https://api.together.xyz/v1",
],
)
def test_together_api_base_resolves_to_together_ai(self, api_base, monkeypatch):
monkeypatch.setenv("TOGETHER_API_KEY", "together-key-from-env")
model, provider, dynamic_api_key, returned_api_base = get_llm_provider(
model="some-model",
api_base=api_base,
)
assert provider == "together_ai"
assert dynamic_api_key == "together-key-from-env"
assert returned_api_base == api_base
assert model == "some-model"
def test_explicit_api_key_beats_together_env_key(self, monkeypatch):
monkeypatch.setenv("TOGETHER_API_KEY", "together-key-from-env")
_, provider, dynamic_api_key, _ = get_llm_provider(
model="some-model",
api_base="https://api.together.ai/v1",
api_key="explicit-caller-key",
)
assert provider == "together_ai"
assert dynamic_api_key == "explicit-caller-key"
def test_together_default_api_base_is_together_ai(self, monkeypatch):
monkeypatch.delenv("TOGETHER_AI_API_BASE", raising=False)
_, provider, _, api_base = get_llm_provider(model="together_ai/some-model")
assert provider == "together_ai"
assert api_base == "https://api.together.ai/v1"

67
tests/test_litellm/rerank_api/test_main.py
View file

@ -1,6 +1,10 @@
import logging
from unittest.mock import MagicMock, patch
import httpx
import pytest
import respx
import litellm
@ -62,3 +66,66 @@ def test_rerank_does_not_log_request_content_at_info(caplog):
assert all(
r.levelno == logging.DEBUG for r in optional_params_logs
), "optional_rerank_params must be logged at DEBUG, not INFO"
TOGETHER_RERANK_BODY = {
"id": "rerank-mock-id",
"results": [{"index": 0, "relevance_score": 0.95}],
"usage": {"prompt_tokens": 10, "total_tokens": 10},
}
def test_together_rerank_defaults_to_together_ai_host(respx_mock: respx.MockRouter, monkeypatch):
"""Regression for the Together host migration: rerank used to hardcode
https://api.together.xyz/v1/rerank. The default must now be api.together.ai."""
monkeypatch.delenv("TOGETHER_AI_API_BASE", raising=False)
mock_route = respx_mock.post("https://api.together.ai/v1/rerank")
mock_route.return_value = httpx.Response(200, json=TOGETHER_RERANK_BODY)
response = litellm.rerank(
model="together_ai/mixedbread-ai/mxbai-rerank-large-v2",
query=MARKER_QUERY,
documents=[MARKER_DOC],
api_key="fake-together-key",
)
assert mock_route.called
assert response.results[0]["relevance_score"] == 0.95
def test_together_rerank_honors_api_base(respx_mock: respx.MockRouter):
"""Regression: a custom api_base was silently ignored by the Together rerank handler."""
mock_route = respx_mock.post("https://custom-together.example/v1/rerank")
mock_route.return_value = httpx.Response(200, json=TOGETHER_RERANK_BODY)
litellm.rerank(
model="together_ai/mixedbread-ai/mxbai-rerank-large-v2",
query=MARKER_QUERY,
documents=[MARKER_DOC],
api_key="fake-together-key",
api_base="https://custom-together.example/v1",
)
assert mock_route.called
assert mock_route.calls[0].request.headers["authorization"] == "Bearer fake-together-key"
@pytest.mark.asyncio
async def test_together_rerank_async_honors_env_api_base(respx_mock: respx.MockRouter, monkeypatch):
"""Regression: TOGETHER_AI_API_BASE was honored by chat but ignored by rerank."""
monkeypatch.setenv("TOGETHER_AI_API_BASE", "https://env-together.example/v1")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
mock_route = respx_mock.post("https://env-together.example/v1/rerank")
mock_route.return_value = httpx.Response(200, json=TOGETHER_RERANK_BODY)
response = await litellm.arerank(
model="together_ai/mixedbread-ai/mxbai-rerank-large-v2",
query=MARKER_QUERY,
documents=[MARKER_DOC],
api_key="fake-together-key",
)
assert mock_route.called
assert response.results[0]["relevance_score"] == 0.95

2
type-discipline-budget.json
View file

@ -3,7 +3,7 @@
"limit": 22804
},
"LIT002": {
"limit": 26873
"limit": 26872
},
"LIT003": {
"limit": 269