mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-19 00:01:29 +00:00
revert(e2e): unmount Gemini, its api_base means two things
Build 227 mounted Gemini and turned TestGeminiFiles::test_gemini_file_upload
red. litellm's two Gemini endpoints disagree about what api_base means.
Chat composes {api_base}/models/{model}:{endpoint} and defaults api_base to
https://generativelanguage.googleapis.com/v1beta, so the version lives
inside it. File upload composes {api_base}/upload/v1beta/files and defaults
to the host root, so the version lives outside it. A single api_base cannot
satisfy both, and a registration carries no signal about which endpoint the
deployment will be used for, so the edge cannot route one and not the other.
Backing it out rather than working around it. The cache must never turn a
passing test red, which is the same rule the Bedrock model allowlist
follows, and Gemini was 7 of roughly 1030 edge calls in that build. Anyone
pointing litellm's Gemini provider at an AI gateway or a corporate proxy
hits this too, so the fix belongs in litellm; mounting Gemini is one line
once it lands.
This reverts commit 8a553ceb58.
This commit is contained in:
Yuneng Jiang
parent
c447c3312d
commit
1972a30def
5 changed files with 7 additions and 215 deletions
|
|
@ -912,7 +912,7 @@ def test_anthropic_stream_requires_start_finish_and_stop() -> None:
|
|||
assert not successful_response("anthropic", url, 200, headers, start + finish)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider,suffix", [("openai", "/v1"), ("anthropic", ""), ("gemini", "")])
|
||||
@pytest.mark.parametrize("provider,suffix", [("openai", "/v1"), ("anthropic", "")])
|
||||
def test_normal_registration_routes_supported_providers(provider: str, suffix: str) -> None:
|
||||
params: Final = LiteLLMParamsBody(model=f"{provider}/test", api_key="os.environ/SYNTHETIC_KEY", timeout=12)
|
||||
routed: Final = route_cache_model(params, lambda mount: f"http://edge.invalid/{mount}", enabled=True)
|
||||
|
|
@ -924,8 +924,6 @@ def test_normal_registration_routes_supported_providers(provider: str, suffix: s
|
|||
@pytest.mark.parametrize("params", [
|
||||
LiteLLMParamsBody(model="bedrock/test"),
|
||||
LiteLLMParamsBody(model="azure/test"),
|
||||
LiteLLMParamsBody(model="vertex_ai/gemini-2.5-flash"),
|
||||
LiteLLMParamsBody(model="gemini/gemini-2.5-flash", api_base="https://custom.invalid"),
|
||||
LiteLLMParamsBody(model="openai/test", api_base="https://custom.invalid/v1"),
|
||||
LiteLLMParamsBody(model="openai/test", api_base=""),
|
||||
LiteLLMParamsBody(model="openai/test", litellm_credential_name="named-credential"),
|
||||
|
|
@ -1294,153 +1292,3 @@ class TestBedrockStreams:
|
|||
|
||||
def test_a_stream_that_errored_before_it_started_is_not_recordable(self) -> None:
|
||||
assert not successful_response(BEDROCK_MOUNT, CONVERSE_STREAM_URL, 503, {}, CONVERSE_STREAM_OK)
|
||||
|
||||
|
||||
GEMINI_MODEL: Final = "gemini-2.5-flash"
|
||||
GEMINI_API_VERSION: Final = "/v1beta"
|
||||
GEMINI_GENERATE_PATH: Final = f"/models/{GEMINI_MODEL}:generateContent"
|
||||
GEMINI_STREAM_PATH: Final = f"/models/{GEMINI_MODEL}:streamGenerateContent"
|
||||
GEMINI_USAGE: Final = {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 25}
|
||||
|
||||
|
||||
def gemini_body(finish_reason: str | None, usage: bool = True, candidates: bool = True) -> JsonValue:
|
||||
candidate: Final[dict[str, JsonValue]] = {"content": {"parts": [{"text": "OK"}], "role": "model"}, "index": 0}
|
||||
return {
|
||||
"candidates": [{**candidate, "finishReason": finish_reason} if finish_reason else candidate]
|
||||
if candidates else [],
|
||||
**({"usageMetadata": GEMINI_USAGE} if usage else {}),
|
||||
"modelVersion": GEMINI_MODEL,
|
||||
}
|
||||
|
||||
|
||||
def gemini_unary(finish_reason: str | None = "STOP", usage: bool = True, candidates: bool = True) -> bytes:
|
||||
return json.dumps(gemini_body(finish_reason, usage, candidates)).encode()
|
||||
|
||||
|
||||
def gemini_stream(*finish_reasons: str | None) -> bytes:
|
||||
return b"".join(
|
||||
b"data: " + json.dumps(gemini_body(reason)).encode() + b"\r\n\r\n" for reason in finish_reasons
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def gemini_edge(cache: CacheEdge, provider: Provider, path: str) -> Generator[str, None, None]:
|
||||
upstream: Final = f"http://127.0.0.1:{provider.server_port}{GEMINI_API_VERSION}"
|
||||
running: Final = start_provider_edge(cache, mounts={"gemini": upstream})
|
||||
try:
|
||||
yield running.edge.api_base("gemini") + path
|
||||
finally:
|
||||
running.shutdown()
|
||||
|
||||
|
||||
class TestGemini:
|
||||
"""Gemini reaches the edge by path prefix alone: litellm composes
|
||||
`{api_base}/models/{model}:{endpoint}` and sends a static `x-goog-api-key`,
|
||||
so nothing has to be re-signed and nothing leaves the cache key. The response
|
||||
grammar is its own though, and the streaming one is the interesting half: every
|
||||
chunk repeats `usageMetadata`, so only `finishReason` on the last chunk
|
||||
separates a finished turn from a dropped connection."""
|
||||
|
||||
@pytest.mark.parametrize("path,response", [
|
||||
(GEMINI_GENERATE_PATH, gemini_unary()),
|
||||
(GEMINI_STREAM_PATH, gemini_stream(None, None, "STOP")),
|
||||
], ids=["generate", "stream"])
|
||||
def test_a_finished_turn_replays_on_the_next_run(
|
||||
self, store: RedisResponseStore, provider: Provider, path: str, response: bytes,
|
||||
) -> None:
|
||||
provider.stream = path == GEMINI_STREAM_PATH
|
||||
provider.response = response
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, path) as url:
|
||||
assert call(url, MARKED).body == response
|
||||
assert len(provider.hits) == 1
|
||||
|
||||
@pytest.mark.parametrize("reason", ["MAX_TOKENS", "SAFETY", "RECITATION"])
|
||||
def test_a_turn_the_provider_ended_for_its_own_reasons_is_still_finished(
|
||||
self, store: RedisResponseStore, provider: Provider, reason: str,
|
||||
) -> None:
|
||||
"""Reading `finishReason` as a string rather than comparing it to STOP is
|
||||
deliberate. A turn cut off by the token limit or a safety filter is over,
|
||||
and rejecting those would send every one of them upstream forever."""
|
||||
provider.response = gemini_unary(reason)
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, GEMINI_GENERATE_PATH) as url:
|
||||
assert call(url, MARKED).body == provider.response
|
||||
assert len(provider.hits) == 1
|
||||
|
||||
@pytest.mark.parametrize("response", [
|
||||
gemini_unary(None),
|
||||
gemini_unary("STOP", usage=False),
|
||||
gemini_unary("STOP", candidates=False),
|
||||
b'{"error":{"code":400,"message":"API key not valid","status":"INVALID_ARGUMENT"}}',
|
||||
], ids=["no-finish-reason", "no-usage", "no-candidates", "error-body"])
|
||||
def test_an_unfinished_or_failed_turn_never_enters_the_cache(
|
||||
self, store: RedisResponseStore, provider: Provider, response: bytes,
|
||||
) -> None:
|
||||
provider.response = response
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, GEMINI_GENERATE_PATH) as url:
|
||||
assert call(url, MARKED).body == response
|
||||
assert len(provider.hits) == 2
|
||||
|
||||
@pytest.mark.parametrize("response", [
|
||||
gemini_stream(None, None),
|
||||
gemini_stream("STOP", None),
|
||||
gemini_stream(),
|
||||
], ids=["cut-before-the-reason", "reason-then-another-chunk", "empty"])
|
||||
def test_a_stream_that_never_named_a_reason_calls_the_provider_every_time(
|
||||
self, store: RedisResponseStore, provider: Provider, response: bytes,
|
||||
) -> None:
|
||||
provider.stream = True
|
||||
provider.response = response
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, GEMINI_STREAM_PATH) as url:
|
||||
assert call(url, MARKED).body == response
|
||||
assert len(provider.hits) == 2
|
||||
|
||||
def test_a_response_whose_candidates_did_not_all_finish_is_not_recordable(
|
||||
self, store: RedisResponseStore, provider: Provider,
|
||||
) -> None:
|
||||
"""A request for more than one candidate is answered by more than one, and
|
||||
the turn is over only when every one of them names a reason. Holding the
|
||||
whole list to that rule rather than its first entry is what keeps a
|
||||
half-finished answer from being stored and replayed as a finished one."""
|
||||
finished: Final = json.loads(gemini_unary("STOP"))["candidates"][0]
|
||||
unfinished: Final = json.loads(gemini_unary(None))["candidates"][0]
|
||||
provider.response = json.dumps(
|
||||
{"candidates": [finished, {**unfinished, "index": 1}], "usageMetadata": GEMINI_USAGE}
|
||||
).encode()
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, GEMINI_GENERATE_PATH) as url:
|
||||
assert call(url, MARKED).body == provider.response
|
||||
assert len(provider.hits) == 2
|
||||
|
||||
@pytest.mark.parametrize("path,cacheable", [
|
||||
(GEMINI_GENERATE_PATH, True),
|
||||
(GEMINI_STREAM_PATH, True),
|
||||
(f"/models/{GEMINI_MODEL}:countTokens", False),
|
||||
(f"/models/{GEMINI_MODEL}:embedContent", False),
|
||||
("/v1/chat/completions", False),
|
||||
(f"/files/{GEMINI_MODEL}:generateContent", False),
|
||||
])
|
||||
@pytest.mark.parametrize("version", ["", GEMINI_API_VERSION], ids=["bare", "versioned"])
|
||||
def test_only_the_generate_endpoints_are_cacheable(self, version: str, path: str, cacheable: bool) -> None:
|
||||
"""The mount's upstream base carries the API version, so the path the cache
|
||||
sees is the upstream one and starts `/v1beta`. A rule anchored at the start
|
||||
of the path would pass every test against a stub with no version prefix and
|
||||
then cache nothing at all in a real run."""
|
||||
assert cacheable_endpoint("gemini", "POST", f"https://gemini.invalid{version}{path}", MARKED) is cacheable
|
||||
|
||||
def test_the_bodies_these_tests_build_match_a_real_gemini_response(self) -> None:
|
||||
"""The shapes above are hand-built so a test can express the turn it means.
|
||||
This holds them to the fields a live `generativelanguage.googleapis.com`
|
||||
answer carries, captured 2026-09-16 against gemini-2.5-flash."""
|
||||
captured: Final = json.loads(
|
||||
'{"candidates":[{"content":{"parts":[{"text":"OK"}],"role":"model"},"finishReason":"STOP",'
|
||||
'"index":0}],"usageMetadata":{"promptTokenCount":7,"candidatesTokenCount":1,'
|
||||
'"totalTokenCount":25},"modelVersion":"gemini-2.5-flash","responseId":"1J6qauKFI8ut1MkPgNjI4AI"}'
|
||||
)
|
||||
built: Final = json.loads(gemini_unary())
|
||||
assert captured.keys() >= built.keys()
|
||||
assert captured["candidates"][0].keys() >= built["candidates"][0].keys()
|
||||
assert successful_response("gemini", GEMINI_GENERATE_PATH, 200, {}, json.dumps(captured).encode())
|
||||
|
|
|
|||
|
|
@ -1,23 +1,13 @@
|
|||
# Shared provider-response cache
|
||||
|
||||
`E2E_PROVIDER_CACHE=1` enables automatic response reuse in the live E2E mode. Standard OpenAI, Anthropic and Gemini model registrations use the provider edge, as do Anthropic-on-Bedrock registrations that carry no AWS identity of their own. Existing custom API bases, named credentials, mocked models and realtime WebSocket deployments keep their existing routing. Other provider protocols remain live
|
||||
`E2E_PROVIDER_CACHE=1` enables automatic response reuse in the live E2E mode. Standard OpenAI and Anthropic model registrations use the provider edge, as do Anthropic-on-Bedrock registrations that carry no AWS identity of their own. Existing custom API bases, named credentials, mocked models and realtime WebSocket deployments keep their existing routing. Other provider protocols remain live
|
||||
|
||||
The edge caches complete successful POST responses for `/v1/chat/completions`, `/v1/messages`, `/v1/embeddings` and `/v1/responses` on the OpenAI and Anthropic mounts, SSE streams included, for `/model/{id}/converse` and `/model/{id}/invoke` on a Bedrock mount, and for `models/{model}:generateContent` and `:streamGenerateContent` on the Gemini mount. Unsupported endpoints pass through. Each endpoint family has its own completeness rule, so a truncated embedding or a Responses run that never reached `response.completed` is not stored
|
||||
The edge caches complete successful POST responses for `/v1/chat/completions`, `/v1/messages`, `/v1/embeddings` and `/v1/responses` on the OpenAI and Anthropic mounts, SSE streams included, and for `/model/{id}/converse` and `/model/{id}/invoke` on a Bedrock mount. Unsupported endpoints pass through. Each endpoint family has its own completeness rule, so a truncated embedding or a Responses run that never reached `response.completed` is not stored
|
||||
|
||||
Bedrock's streaming endpoints, `converse-stream` and `invoke-with-response-stream`, cache too. AWS frames those as binary `vnd.amazon.eventstream` rather than SSE, so botocore's own parser reads the frames and validates both CRCs, and each endpoint is then held to its terminal grammar. That matters more than the endpoint count suggests: the Claude Code compat cells drive the real CLI, which always streams, so streaming is most of the suite's Bedrock traffic
|
||||
|
||||
Two details of that rule are worth knowing before changing it. A ConverseStream ends with `metadata`, not with `messageStop`, and the `metadata` frame is what carries the token usage litellm prices the call from, so the rule requires it: a stream cut between the two still names a stop reason but would replay as a free call. And a dropped connection is invisible to the parser, which yields the frames it did receive and silently discards a trailing partial one, so the body is also checked against the frame lengths it declares. A stream cut one byte short parses clean and has to be caught that way
|
||||
|
||||
## Gemini
|
||||
|
||||
Gemini needs nothing that Bedrock needed. litellm composes `{api_base}/models/{model}:{endpoint}` from a custom api_base, so a path-prefixed mount reaches it, and the credential travels as a static `x-goog-api-key` header that no host rewrite invalidates. Nothing is re-signed and nothing is excluded from the key, so a recording still cannot cross credentials
|
||||
|
||||
The mount's upstream base carries the API version, which is the one detail worth remembering: the path the cache rules see is the upstream one, `/v1beta/models/...`, not the one the proxy sent. A rule anchored at the start of that path would look right against a local stub and then cache nothing at all in a real run
|
||||
|
||||
A finished turn names a `finishReason` on every candidate and reports `usageMetadata`. The reason is read as a string rather than compared to `STOP`, because `MAX_TOKENS` and the safety reasons end a turn just as finally and rejecting them would send every one of them upstream forever. Streaming is the more interesting half: Gemini repeats `usageMetadata` on every chunk and names a `finishReason` only on the last one, so the terminator is the final event rather than any event, and a stream the connection cut short ends on a chunk with usage and no reason
|
||||
|
||||
Vertex is not mounted. litellm grafts the default Vertex path onto an api_base only when that api_base has no path of its own, so a Vertex mount needs a root-mounted edge on its own port rather than a path prefix. Gemini and Vertex are separate providers in litellm and the Gemini mount does not cover Vertex deployments
|
||||
|
||||
## Request identity
|
||||
|
||||
A recording belongs to one test. The key is a keyed digest over the test's node id, the method, the URL, the effective outbound headers (including authentication and HTTP-library defaults), body presence and the body bytes, with one normalization: a 12-hex-digit run, the shape `unique_marker()` mints, is replaced by a placeholder in both the URL and a UTF-8 body. Nothing else is normalized away. No prompts, JSON values or credentials are rewritten, and the rule is the one `fixture_canonical.py` already applies for record/replay, so there is a single definition of what a marker is
|
||||
|
|
@ -40,7 +30,9 @@ Only deployments that carry no AWS identity of their own route to the edge. A de
|
|||
|
||||
Which models route is an explicit allowlist in `provider_cache_routing.py`, mirroring the runner role's IAM policy, which names its models one by one. That coupling is deliberate: the edge re-signs with the run pod's identity, so a model the role cannot invoke comes back 403 from Bedrock rather than falling back. An unlisted model keeps its direct path and loses only caching, so adding a Bedrock model to the suite can never turn it red. Adding one to the edge is a policy edit in litellm-ops plus a line here
|
||||
|
||||
Vertex is not mounted. litellm's `_check_custom_proxy` rewrites a path-prefixed Vertex `api_base` into `{api_base}:{endpoint}`, dropping project, location and model, so a mount under a path prefix cannot work without either a root-mounted edge on its own port or a change in litellm. Gemini is a separate provider there and does have a working path-prefixed form, so it is mounted; see the Gemini section
|
||||
Vertex and Gemini are not mounted, for different reasons. litellm grafts the default Vertex path onto an `api_base` only when that `api_base` has no path of its own, so a path-prefixed Vertex mount instead becomes `{api_base}:{endpoint}`, dropping project, location and model. Vertex needs a root-mounted edge on its own port, or a change in litellm
|
||||
|
||||
Gemini reaches a path-prefixed mount perfectly well and was mounted for one build, then backed out, because litellm's two Gemini endpoints disagree about what `api_base` means. Chat composes `{api_base}/models/{model}:{endpoint}` and defaults `api_base` to `https://generativelanguage.googleapis.com/v1beta`, so the version has to be inside it. File upload composes `{api_base}/upload/v1beta/files` and defaults to the host root, so the version has to be outside it. One `api_base` cannot satisfy both, and a deployment gives no signal at registration time about which it will be used for, so mounting Gemini turned `TestGeminiFiles::test_gemini_file_upload` red in build 227. Anyone pointing litellm's Gemini provider at an AI gateway or a corporate proxy hits the same thing; it is a litellm bug rather than a cache limitation, and mounting Gemini is one line once it is fixed
|
||||
|
||||
Recordings are shared across workers and builds through dedicated Redis, separate from the candidate's own cache. They expire 86,400 seconds after capture starts, based on Redis time. Reads never extend expiry. There is no scheduled recapture: the next miss calls the provider again. Bounded coordination reduces duplicate concurrent calls, but slow or failed captures may lead to extra live calls after the wait expires
|
||||
|
||||
|
|
|
|||
|
|
@ -37,10 +37,6 @@ SIGNATURE_HEADERS: Final = frozenset(
|
|||
{"authorization", "x-amz-date", "x-amz-security-token", "x-amz-content-sha256"}
|
||||
)
|
||||
BEDROCK_MOUNT_PREFIX: Final = "bedrock"
|
||||
GEMINI_MOUNT: Final = "gemini"
|
||||
GEMINI_MODELS_SEGMENT: Final = "/models"
|
||||
GEMINI_GENERATE_SUFFIX: Final = ":generateContent"
|
||||
GEMINI_STREAM_SUFFIX: Final = ":streamGenerateContent"
|
||||
BEDROCK_CONVERSE_SUFFIX: Final = "/converse"
|
||||
BEDROCK_INVOKE_SUFFIX: Final = "/invoke"
|
||||
BEDROCK_CONVERSE_STREAM_SUFFIX: Final = "/converse-stream"
|
||||
|
|
@ -162,21 +158,12 @@ def is_bedrock(mount: str) -> bool:
|
|||
return mount.partition("/")[0] == BEDROCK_MOUNT_PREFIX
|
||||
|
||||
|
||||
def is_gemini(mount: str) -> bool:
|
||||
return mount == GEMINI_MOUNT
|
||||
|
||||
|
||||
def cacheable_endpoint(mount: str, method: str, url: str, body: bytes | None) -> bool:
|
||||
if method != "POST" or body is None or len(body) > MAX_REQUEST_BYTES:
|
||||
return False
|
||||
path: Final = urlsplit(url).path
|
||||
if is_bedrock(mount):
|
||||
return path.startswith("/model/") and path.endswith(BEDROCK_SUFFIXES)
|
||||
if is_gemini(mount):
|
||||
collection, _, resource = path.rpartition("/")
|
||||
return collection.endswith(GEMINI_MODELS_SEGMENT) and resource.endswith(
|
||||
(GEMINI_GENERATE_SUFFIX, GEMINI_STREAM_SUFFIX)
|
||||
)
|
||||
return path in OPENAI_JSON_PATHS
|
||||
|
||||
|
||||
|
|
@ -203,8 +190,6 @@ def successful_response(mount: str, url: str, status: int, headers: Mapping[str,
|
|||
for value in values
|
||||
):
|
||||
return False
|
||||
if is_gemini(mount):
|
||||
return complete_gemini_stream(values)
|
||||
if urlsplit(url).path == "/v1/responses":
|
||||
return complete_responses_stream(values)
|
||||
if urlsplit(url).path == "/v1/chat/completions":
|
||||
|
|
@ -216,8 +201,6 @@ def successful_response(mount: str, url: str, status: int, headers: Mapping[str,
|
|||
return False
|
||||
if not isinstance(value, dict) or value.get("error") is not None:
|
||||
return False
|
||||
if is_gemini(mount):
|
||||
return complete_gemini_candidates(value)
|
||||
path: Final = urlsplit(url).path
|
||||
if path == "/v1/messages":
|
||||
return value.get("type") == "message" and isinstance(value.get("content"), list) and isinstance(value.get("stop_reason"), str)
|
||||
|
|
@ -236,35 +219,6 @@ def successful_response(mount: str, url: str, status: int, headers: Mapping[str,
|
|||
)
|
||||
|
||||
|
||||
def complete_gemini_candidates(value: Mapping[str, JsonValue]) -> bool:
|
||||
"""A finished Gemini turn names a ``finishReason`` on every candidate and
|
||||
reports the usage litellm prices the call from. ``finishReason`` is read as a
|
||||
string rather than compared to ``STOP`` because ``MAX_TOKENS`` and the safety
|
||||
reasons end a turn just as finally, and a cache that rejected them would send
|
||||
every one of them upstream forever."""
|
||||
candidates: Final = value.get("candidates")
|
||||
return (
|
||||
isinstance(value.get("usageMetadata"), dict)
|
||||
and isinstance(candidates, list)
|
||||
and bool(candidates)
|
||||
and all(
|
||||
isinstance(candidate, dict) and isinstance(candidate.get("finishReason"), str)
|
||||
for candidate in candidates
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def complete_gemini_stream(values: tuple[JsonValue, ...]) -> bool:
|
||||
"""Gemini repeats ``usageMetadata`` on every chunk but names a
|
||||
``finishReason`` only on the last one, so the terminator is the final event
|
||||
rather than any event. A stream the connection cut short ends on a chunk that
|
||||
carries usage and no reason, which is exactly what this rejects."""
|
||||
if not values:
|
||||
return False
|
||||
last: Final = values[-1]
|
||||
return isinstance(last, dict) and complete_gemini_candidates(last)
|
||||
|
||||
|
||||
def complete_bedrock_response(url: str, body: bytes) -> bool:
|
||||
"""Converse answers with ``output`` plus a ``stopReason``; InvokeModel on an
|
||||
Anthropic model answers the Anthropic message shape. Either way a truncated
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ BEDROCK_EDGE_MODELS: Final = frozenset(
|
|||
}
|
||||
)
|
||||
ENV_REFERENCE_PREFIX: Final = "os.environ/"
|
||||
EDGE_PROVIDERS: Final = frozenset({"openai", "anthropic", "gemini"})
|
||||
|
||||
|
||||
def bedrock_region(declared: str | None) -> str:
|
||||
|
|
@ -82,7 +81,7 @@ def route_cache_model(
|
|||
return route_bedrock(params, base_for, mode)
|
||||
if mode == "realtime" or params.api_base is not None:
|
||||
return params
|
||||
if provider not in EDGE_PROVIDERS:
|
||||
if provider not in {"openai", "anthropic"}:
|
||||
return params
|
||||
base: Final = base_for(provider)
|
||||
if base is None:
|
||||
|
|
|
|||
|
|
@ -104,7 +104,6 @@ EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType(
|
|||
{
|
||||
"openai": "https://api.openai.com",
|
||||
"anthropic": "https://api.anthropic.com",
|
||||
"gemini": "https://generativelanguage.googleapis.com/v1beta",
|
||||
**{
|
||||
f"bedrock/{region}": f"https://bedrock-runtime.{region}.amazonaws.com"
|
||||
for region in BEDROCK_REGIONS
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue