mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-20 00:11:50 +00:00
feat(e2e): mount Gemini on the provider cache
Gemini needs none of the machinery Bedrock needed. litellm composes
{api_base}/models/{model}:{endpoint} from a custom api_base, so a plain
path-prefixed mount reaches it, and the credential travels as a static
x-goog-api-key header that no host rewrite invalidates. Nothing is
re-signed and nothing leaves the cache key, so a recording still cannot
cross credentials.
A finished turn names a finishReason on every candidate and reports
usageMetadata. The reason is read as a string rather than compared to
STOP: MAX_TOKENS and the safety reasons end a turn just as finally, and
rejecting them would send every one of them upstream forever. Streaming
is the half worth care. Gemini repeats usageMetadata on every chunk and
names a finishReason only on the last, so the terminator is the final
event rather than any event, and a stream the connection cut short ends
on a chunk carrying usage and no reason.
The mount's upstream base carries the API version, so the path the rules
see is /v1beta/models/..., not the one the proxy sent. The first version
of this anchored the rule at the start of that path, which passed every
test against a stub with no version prefix and would have cached nothing
at all in a real run. Caught by replaying the rules over responses
captured from live gemini-2.5-flash, which is also why the tests now
mount their stub under the version prefix.
Vertex stays unmounted and is a separate provider here: litellm grafts
the default Vertex path onto an api_base only when that api_base has no
path of its own, so Vertex needs a root-mounted edge on its own port.
This commit is contained in:
Yuneng Jiang
parent
30a691ed55
commit
8a553ceb58
5 changed files with 215 additions and 5 deletions
|
|
@ -861,7 +861,7 @@ def test_anthropic_stream_requires_start_finish_and_stop() -> None:
|
|||
assert not successful_response("anthropic", url, 200, headers, start + finish)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider,suffix", [("openai", "/v1"), ("anthropic", "")])
|
||||
@pytest.mark.parametrize("provider,suffix", [("openai", "/v1"), ("anthropic", ""), ("gemini", "")])
|
||||
def test_normal_registration_routes_supported_providers(provider: str, suffix: str) -> None:
|
||||
params: Final = LiteLLMParamsBody(model=f"{provider}/test", api_key="os.environ/SYNTHETIC_KEY", timeout=12)
|
||||
routed: Final = route_cache_model(params, lambda mount: f"http://edge.invalid/{mount}", enabled=True)
|
||||
|
|
@ -873,6 +873,8 @@ def test_normal_registration_routes_supported_providers(provider: str, suffix: s
|
|||
@pytest.mark.parametrize("params", [
|
||||
LiteLLMParamsBody(model="bedrock/test"),
|
||||
LiteLLMParamsBody(model="azure/test"),
|
||||
LiteLLMParamsBody(model="vertex_ai/gemini-2.5-flash"),
|
||||
LiteLLMParamsBody(model="gemini/gemini-2.5-flash", api_base="https://custom.invalid"),
|
||||
LiteLLMParamsBody(model="openai/test", api_base="https://custom.invalid/v1"),
|
||||
LiteLLMParamsBody(model="openai/test", api_base=""),
|
||||
LiteLLMParamsBody(model="openai/test", litellm_credential_name="named-credential"),
|
||||
|
|
@ -1241,3 +1243,153 @@ class TestBedrockStreams:
|
|||
|
||||
def test_a_stream_that_errored_before_it_started_is_not_recordable(self) -> None:
|
||||
assert not successful_response(BEDROCK_MOUNT, CONVERSE_STREAM_URL, 503, {}, CONVERSE_STREAM_OK)
|
||||
|
||||
|
||||
GEMINI_MODEL: Final = "gemini-2.5-flash"
|
||||
GEMINI_API_VERSION: Final = "/v1beta"
|
||||
GEMINI_GENERATE_PATH: Final = f"/models/{GEMINI_MODEL}:generateContent"
|
||||
GEMINI_STREAM_PATH: Final = f"/models/{GEMINI_MODEL}:streamGenerateContent"
|
||||
GEMINI_USAGE: Final = {"promptTokenCount": 7, "candidatesTokenCount": 1, "totalTokenCount": 25}
|
||||
|
||||
|
||||
def gemini_body(finish_reason: str | None, usage: bool = True, candidates: bool = True) -> JsonValue:
|
||||
candidate: Final[dict[str, JsonValue]] = {"content": {"parts": [{"text": "OK"}], "role": "model"}, "index": 0}
|
||||
return {
|
||||
"candidates": [{**candidate, "finishReason": finish_reason} if finish_reason else candidate]
|
||||
if candidates else [],
|
||||
**({"usageMetadata": GEMINI_USAGE} if usage else {}),
|
||||
"modelVersion": GEMINI_MODEL,
|
||||
}
|
||||
|
||||
|
||||
def gemini_unary(finish_reason: str | None = "STOP", usage: bool = True, candidates: bool = True) -> bytes:
|
||||
return json.dumps(gemini_body(finish_reason, usage, candidates)).encode()
|
||||
|
||||
|
||||
def gemini_stream(*finish_reasons: str | None) -> bytes:
|
||||
return b"".join(
|
||||
b"data: " + json.dumps(gemini_body(reason)).encode() + b"\r\n\r\n" for reason in finish_reasons
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def gemini_edge(cache: CacheEdge, provider: Provider, path: str) -> Generator[str, None, None]:
|
||||
upstream: Final = f"http://127.0.0.1:{provider.server_port}{GEMINI_API_VERSION}"
|
||||
running: Final = start_provider_edge(cache, mounts={"gemini": upstream})
|
||||
try:
|
||||
yield running.edge.api_base("gemini") + path
|
||||
finally:
|
||||
running.shutdown()
|
||||
|
||||
|
||||
class TestGemini:
|
||||
"""Gemini reaches the edge by path prefix alone: litellm composes
|
||||
`{api_base}/models/{model}:{endpoint}` and sends a static `x-goog-api-key`,
|
||||
so nothing has to be re-signed and nothing leaves the cache key. The response
|
||||
grammar is its own though, and the streaming one is the interesting half: every
|
||||
chunk repeats `usageMetadata`, so only `finishReason` on the last chunk
|
||||
separates a finished turn from a dropped connection."""
|
||||
|
||||
@pytest.mark.parametrize("path,response", [
|
||||
(GEMINI_GENERATE_PATH, gemini_unary()),
|
||||
(GEMINI_STREAM_PATH, gemini_stream(None, None, "STOP")),
|
||||
], ids=["generate", "stream"])
|
||||
def test_a_finished_turn_replays_on_the_next_run(
|
||||
self, store: RedisResponseStore, provider: Provider, path: str, response: bytes,
|
||||
) -> None:
|
||||
provider.stream = path == GEMINI_STREAM_PATH
|
||||
provider.response = response
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, path) as url:
|
||||
assert call(url, MARKED).body == response
|
||||
assert len(provider.hits) == 1
|
||||
|
||||
@pytest.mark.parametrize("reason", ["MAX_TOKENS", "SAFETY", "RECITATION"])
|
||||
def test_a_turn_the_provider_ended_for_its_own_reasons_is_still_finished(
|
||||
self, store: RedisResponseStore, provider: Provider, reason: str,
|
||||
) -> None:
|
||||
"""Reading `finishReason` as a string rather than comparing it to STOP is
|
||||
deliberate. A turn cut off by the token limit or a safety filter is over,
|
||||
and rejecting those would send every one of them upstream forever."""
|
||||
provider.response = gemini_unary(reason)
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, GEMINI_GENERATE_PATH) as url:
|
||||
assert call(url, MARKED).body == provider.response
|
||||
assert len(provider.hits) == 1
|
||||
|
||||
@pytest.mark.parametrize("response", [
|
||||
gemini_unary(None),
|
||||
gemini_unary("STOP", usage=False),
|
||||
gemini_unary("STOP", candidates=False),
|
||||
b'{"error":{"code":400,"message":"API key not valid","status":"INVALID_ARGUMENT"}}',
|
||||
], ids=["no-finish-reason", "no-usage", "no-candidates", "error-body"])
|
||||
def test_an_unfinished_or_failed_turn_never_enters_the_cache(
|
||||
self, store: RedisResponseStore, provider: Provider, response: bytes,
|
||||
) -> None:
|
||||
provider.response = response
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, GEMINI_GENERATE_PATH) as url:
|
||||
assert call(url, MARKED).body == response
|
||||
assert len(provider.hits) == 2
|
||||
|
||||
@pytest.mark.parametrize("response", [
|
||||
gemini_stream(None, None),
|
||||
gemini_stream("STOP", None),
|
||||
gemini_stream(),
|
||||
], ids=["cut-before-the-reason", "reason-then-another-chunk", "empty"])
|
||||
def test_a_stream_that_never_named_a_reason_calls_the_provider_every_time(
|
||||
self, store: RedisResponseStore, provider: Provider, response: bytes,
|
||||
) -> None:
|
||||
provider.stream = True
|
||||
provider.response = response
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, GEMINI_STREAM_PATH) as url:
|
||||
assert call(url, MARKED).body == response
|
||||
assert len(provider.hits) == 2
|
||||
|
||||
def test_a_response_whose_candidates_did_not_all_finish_is_not_recordable(
|
||||
self, store: RedisResponseStore, provider: Provider,
|
||||
) -> None:
|
||||
"""A request for more than one candidate is answered by more than one, and
|
||||
the turn is over only when every one of them names a reason. Holding the
|
||||
whole list to that rule rather than its first entry is what keeps a
|
||||
half-finished answer from being stored and replayed as a finished one."""
|
||||
finished: Final = json.loads(gemini_unary("STOP"))["candidates"][0]
|
||||
unfinished: Final = json.loads(gemini_unary(None))["candidates"][0]
|
||||
provider.response = json.dumps(
|
||||
{"candidates": [finished, {**unfinished, "index": 1}], "usageMetadata": GEMINI_USAGE}
|
||||
).encode()
|
||||
for _ in range(2):
|
||||
with gemini_edge(cache_edge(store), provider, GEMINI_GENERATE_PATH) as url:
|
||||
assert call(url, MARKED).body == provider.response
|
||||
assert len(provider.hits) == 2
|
||||
|
||||
@pytest.mark.parametrize("path,cacheable", [
|
||||
(GEMINI_GENERATE_PATH, True),
|
||||
(GEMINI_STREAM_PATH, True),
|
||||
(f"/models/{GEMINI_MODEL}:countTokens", False),
|
||||
(f"/models/{GEMINI_MODEL}:embedContent", False),
|
||||
("/v1/chat/completions", False),
|
||||
(f"/files/{GEMINI_MODEL}:generateContent", False),
|
||||
])
|
||||
@pytest.mark.parametrize("version", ["", GEMINI_API_VERSION], ids=["bare", "versioned"])
|
||||
def test_only_the_generate_endpoints_are_cacheable(self, version: str, path: str, cacheable: bool) -> None:
|
||||
"""The mount's upstream base carries the API version, so the path the cache
|
||||
sees is the upstream one and starts `/v1beta`. A rule anchored at the start
|
||||
of the path would pass every test against a stub with no version prefix and
|
||||
then cache nothing at all in a real run."""
|
||||
assert cacheable_endpoint("gemini", "POST", f"https://gemini.invalid{version}{path}", MARKED) is cacheable
|
||||
|
||||
def test_the_bodies_these_tests_build_match_a_real_gemini_response(self) -> None:
|
||||
"""The shapes above are hand-built so a test can express the turn it means.
|
||||
This holds them to the fields a live `generativelanguage.googleapis.com`
|
||||
answer carries, captured 2026-09-16 against gemini-2.5-flash."""
|
||||
captured: Final = json.loads(
|
||||
'{"candidates":[{"content":{"parts":[{"text":"OK"}],"role":"model"},"finishReason":"STOP",'
|
||||
'"index":0}],"usageMetadata":{"promptTokenCount":7,"candidatesTokenCount":1,'
|
||||
'"totalTokenCount":25},"modelVersion":"gemini-2.5-flash","responseId":"1J6qauKFI8ut1MkPgNjI4AI"}'
|
||||
)
|
||||
built: Final = json.loads(gemini_unary())
|
||||
assert captured.keys() >= built.keys()
|
||||
assert captured["candidates"][0].keys() >= built["candidates"][0].keys()
|
||||
assert successful_response("gemini", GEMINI_GENERATE_PATH, 200, {}, json.dumps(captured).encode())
|
||||
|
|
|
|||
|
|
@ -1,13 +1,23 @@
|
|||
# Shared provider-response cache
|
||||
|
||||
`E2E_PROVIDER_CACHE=1` enables automatic response reuse in the live E2E mode. Standard OpenAI and Anthropic model registrations use the provider edge, as do Anthropic-on-Bedrock registrations that carry no AWS identity of their own. Existing custom API bases, named credentials, mocked models and realtime WebSocket deployments keep their existing routing. Other provider protocols remain live
|
||||
`E2E_PROVIDER_CACHE=1` enables automatic response reuse in the live E2E mode. Standard OpenAI, Anthropic and Gemini model registrations use the provider edge, as do Anthropic-on-Bedrock registrations that carry no AWS identity of their own. Existing custom API bases, named credentials, mocked models and realtime WebSocket deployments keep their existing routing. Other provider protocols remain live
|
||||
|
||||
The edge caches complete successful POST responses for `/v1/chat/completions`, `/v1/messages`, `/v1/embeddings` and `/v1/responses` on the OpenAI and Anthropic mounts, SSE streams included, and for `/model/{id}/converse` and `/model/{id}/invoke` on a Bedrock mount. Unsupported endpoints pass through. Each endpoint family has its own completeness rule, so a truncated embedding or a Responses run that never reached `response.completed` is not stored
|
||||
The edge caches complete successful POST responses for `/v1/chat/completions`, `/v1/messages`, `/v1/embeddings` and `/v1/responses` on the OpenAI and Anthropic mounts, SSE streams included, for `/model/{id}/converse` and `/model/{id}/invoke` on a Bedrock mount, and for `models/{model}:generateContent` and `:streamGenerateContent` on the Gemini mount. Unsupported endpoints pass through. Each endpoint family has its own completeness rule, so a truncated embedding or a Responses run that never reached `response.completed` is not stored
|
||||
|
||||
Bedrock's streaming endpoints, `converse-stream` and `invoke-with-response-stream`, cache too. AWS frames those as binary `vnd.amazon.eventstream` rather than SSE, so botocore's own parser reads the frames and validates both CRCs, and each endpoint is then held to its terminal grammar. That matters more than the endpoint count suggests: the Claude Code compat cells drive the real CLI, which always streams, so streaming is most of the suite's Bedrock traffic
|
||||
|
||||
Two details of that rule are worth knowing before changing it. A ConverseStream ends with `metadata`, not with `messageStop`, and the `metadata` frame is what carries the token usage litellm prices the call from, so the rule requires it: a stream cut between the two still names a stop reason but would replay as a free call. And a dropped connection is invisible to the parser, which yields the frames it did receive and silently discards a trailing partial one, so the body is also checked against the frame lengths it declares. A stream cut one byte short parses clean and has to be caught that way
|
||||
|
||||
## Gemini
|
||||
|
||||
Gemini needs nothing that Bedrock needed. litellm composes `{api_base}/models/{model}:{endpoint}` from a custom api_base, so a path-prefixed mount reaches it, and the credential travels as a static `x-goog-api-key` header that no host rewrite invalidates. Nothing is re-signed and nothing is excluded from the key, so a recording still cannot cross credentials
|
||||
|
||||
The mount's upstream base carries the API version, which is the one detail worth remembering: the path the cache rules see is the upstream one, `/v1beta/models/...`, not the one the proxy sent. A rule anchored at the start of that path would look right against a local stub and then cache nothing at all in a real run
|
||||
|
||||
A finished turn names a `finishReason` on every candidate and reports `usageMetadata`. The reason is read as a string rather than compared to `STOP`, because `MAX_TOKENS` and the safety reasons end a turn just as finally and rejecting them would send every one of them upstream forever. Streaming is the more interesting half: Gemini repeats `usageMetadata` on every chunk and names a `finishReason` only on the last one, so the terminator is the final event rather than any event, and a stream the connection cut short ends on a chunk with usage and no reason
|
||||
|
||||
Vertex is not mounted. litellm grafts the default Vertex path onto an api_base only when that api_base has no path of its own, so a Vertex mount needs a root-mounted edge on its own port rather than a path prefix. Gemini and Vertex are separate providers in litellm and the Gemini mount does not cover Vertex deployments
|
||||
|
||||
## Request identity
|
||||
|
||||
A recording belongs to one test. The key is a keyed digest over the test's node id, the method, the URL, the effective outbound headers (including authentication and HTTP-library defaults), body presence and the body bytes, with one normalization: a 12-hex-digit run, the shape `unique_marker()` mints, is replaced by a placeholder in both the URL and a UTF-8 body. Nothing else is normalized away. No prompts, JSON values or credentials are rewritten, and the rule is the one `fixture_canonical.py` already applies for record/replay, so there is a single definition of what a marker is
|
||||
|
|
@ -30,7 +40,7 @@ Only deployments that carry no AWS identity of their own route to the edge. A de
|
|||
|
||||
Which models route is an explicit allowlist in `provider_cache_routing.py`, mirroring the runner role's IAM policy, which names its models one by one. That coupling is deliberate: the edge re-signs with the run pod's identity, so a model the role cannot invoke comes back 403 from Bedrock rather than falling back. An unlisted model keeps its direct path and loses only caching, so adding a Bedrock model to the suite can never turn it red. Adding one to the edge is a policy edit in litellm-ops plus a line here
|
||||
|
||||
Vertex and Gemini are not mounted. litellm's `_check_custom_proxy` rewrites a path-prefixed Vertex `api_base` into `{api_base}:{endpoint}`, dropping project, location and model, so a mount under a path prefix cannot work without either a root-mounted edge on its own port or a change in litellm
|
||||
Vertex is not mounted. litellm's `_check_custom_proxy` rewrites a path-prefixed Vertex `api_base` into `{api_base}:{endpoint}`, dropping project, location and model, so a mount under a path prefix cannot work without either a root-mounted edge on its own port or a change in litellm. Gemini is a separate provider there and does have a working path-prefixed form, so it is mounted; see the Gemini section
|
||||
|
||||
Recordings are shared across workers and builds through dedicated Redis, separate from the candidate's own cache. They expire 86,400 seconds after capture starts, based on Redis time. Reads never extend expiry. There is no scheduled recapture: the next miss calls the provider again. Bounded coordination reduces duplicate concurrent calls, but slow or failed captures may lead to extra live calls after the wait expires
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ SIGNATURE_HEADERS: Final = frozenset(
|
|||
{"authorization", "x-amz-date", "x-amz-security-token", "x-amz-content-sha256"}
|
||||
)
|
||||
BEDROCK_MOUNT_PREFIX: Final = "bedrock"
|
||||
GEMINI_MOUNT: Final = "gemini"
|
||||
GEMINI_MODELS_SEGMENT: Final = "/models"
|
||||
GEMINI_GENERATE_SUFFIX: Final = ":generateContent"
|
||||
GEMINI_STREAM_SUFFIX: Final = ":streamGenerateContent"
|
||||
BEDROCK_CONVERSE_SUFFIX: Final = "/converse"
|
||||
BEDROCK_INVOKE_SUFFIX: Final = "/invoke"
|
||||
BEDROCK_CONVERSE_STREAM_SUFFIX: Final = "/converse-stream"
|
||||
|
|
@ -154,12 +158,21 @@ def is_bedrock(mount: str) -> bool:
|
|||
return mount.partition("/")[0] == BEDROCK_MOUNT_PREFIX
|
||||
|
||||
|
||||
def is_gemini(mount: str) -> bool:
|
||||
return mount == GEMINI_MOUNT
|
||||
|
||||
|
||||
def cacheable_endpoint(mount: str, method: str, url: str, body: bytes | None) -> bool:
|
||||
if method != "POST" or body is None or len(body) > MAX_REQUEST_BYTES:
|
||||
return False
|
||||
path: Final = urlsplit(url).path
|
||||
if is_bedrock(mount):
|
||||
return path.startswith("/model/") and path.endswith(BEDROCK_SUFFIXES)
|
||||
if is_gemini(mount):
|
||||
collection, _, resource = path.rpartition("/")
|
||||
return collection.endswith(GEMINI_MODELS_SEGMENT) and resource.endswith(
|
||||
(GEMINI_GENERATE_SUFFIX, GEMINI_STREAM_SUFFIX)
|
||||
)
|
||||
return path in OPENAI_JSON_PATHS
|
||||
|
||||
|
||||
|
|
@ -186,6 +199,8 @@ def successful_response(mount: str, url: str, status: int, headers: Mapping[str,
|
|||
for value in values
|
||||
):
|
||||
return False
|
||||
if is_gemini(mount):
|
||||
return complete_gemini_stream(values)
|
||||
if urlsplit(url).path == "/v1/responses":
|
||||
return complete_responses_stream(values)
|
||||
if urlsplit(url).path == "/v1/chat/completions":
|
||||
|
|
@ -197,6 +212,8 @@ def successful_response(mount: str, url: str, status: int, headers: Mapping[str,
|
|||
return False
|
||||
if not isinstance(value, dict) or value.get("error") is not None:
|
||||
return False
|
||||
if is_gemini(mount):
|
||||
return complete_gemini_candidates(value)
|
||||
path: Final = urlsplit(url).path
|
||||
if path == "/v1/messages":
|
||||
return value.get("type") == "message" and isinstance(value.get("content"), list) and isinstance(value.get("stop_reason"), str)
|
||||
|
|
@ -215,6 +232,35 @@ def successful_response(mount: str, url: str, status: int, headers: Mapping[str,
|
|||
)
|
||||
|
||||
|
||||
def complete_gemini_candidates(value: Mapping[str, JsonValue]) -> bool:
|
||||
"""A finished Gemini turn names a ``finishReason`` on every candidate and
|
||||
reports the usage litellm prices the call from. ``finishReason`` is read as a
|
||||
string rather than compared to ``STOP`` because ``MAX_TOKENS`` and the safety
|
||||
reasons end a turn just as finally, and a cache that rejected them would send
|
||||
every one of them upstream forever."""
|
||||
candidates: Final = value.get("candidates")
|
||||
return (
|
||||
isinstance(value.get("usageMetadata"), dict)
|
||||
and isinstance(candidates, list)
|
||||
and bool(candidates)
|
||||
and all(
|
||||
isinstance(candidate, dict) and isinstance(candidate.get("finishReason"), str)
|
||||
for candidate in candidates
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def complete_gemini_stream(values: tuple[JsonValue, ...]) -> bool:
|
||||
"""Gemini repeats ``usageMetadata`` on every chunk but names a
|
||||
``finishReason`` only on the last one, so the terminator is the final event
|
||||
rather than any event. A stream the connection cut short ends on a chunk that
|
||||
carries usage and no reason, which is exactly what this rejects."""
|
||||
if not values:
|
||||
return False
|
||||
last: Final = values[-1]
|
||||
return isinstance(last, dict) and complete_gemini_candidates(last)
|
||||
|
||||
|
||||
def complete_bedrock_response(url: str, body: bytes) -> bool:
|
||||
"""Converse answers with ``output`` plus a ``stopReason``; InvokeModel on an
|
||||
Anthropic model answers the Anthropic message shape. Either way a truncated
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ BEDROCK_EDGE_MODELS: Final = frozenset(
|
|||
}
|
||||
)
|
||||
ENV_REFERENCE_PREFIX: Final = "os.environ/"
|
||||
EDGE_PROVIDERS: Final = frozenset({"openai", "anthropic", "gemini"})
|
||||
|
||||
|
||||
def bedrock_region(declared: str | None) -> str:
|
||||
|
|
@ -81,7 +82,7 @@ def route_cache_model(
|
|||
return route_bedrock(params, base_for, mode)
|
||||
if mode == "realtime" or params.api_base is not None:
|
||||
return params
|
||||
if provider not in {"openai", "anthropic"}:
|
||||
if provider not in EDGE_PROVIDERS:
|
||||
return params
|
||||
base: Final = base_for(provider)
|
||||
if base is None:
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType(
|
|||
{
|
||||
"openai": "https://api.openai.com",
|
||||
"anthropic": "https://api.anthropic.com",
|
||||
"gemini": "https://generativelanguage.googleapis.com/v1beta",
|
||||
**{
|
||||
f"bedrock/{region}": f"https://bedrock-runtime.{region}.amazonaws.com"
|
||||
for region in BEDROCK_REGIONS
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue