scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

Merge pull request #1415 from BerriAI/litellm_bump_httpx_pool_limits

Browse source 
fix(router.py): bump httpx pool limits
This commit is contained in:
Krish Dholakia 2024-01-11 13:46:31 +05:30 • committed by GitHub
commit 40054f89b5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 52 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
litellm/router.py
View file

@ -1363,6 +1363,12 @@ class Router:
api_version=api_version,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1378,6 +1384,12 @@ class Router:
api_version=api_version,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1393,6 +1405,12 @@ class Router:
api_version=api_version,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1408,6 +1426,12 @@ class Router:
api_version=api_version,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1471,9 +1495,10 @@ class Router:
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
)
),
),
)
self.cache.set_cache(
@ -1491,9 +1516,10 @@ class Router:
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
)
),
),
)
self.cache.set_cache(
@ -1513,6 +1539,12 @@ class Router:
base_url=api_base,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1527,6 +1559,12 @@ class Router:
base_url=api_base,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1542,6 +1580,12 @@ class Router:
base_url=api_base,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1557,6 +1601,12 @@ class Router:
base_url=api_base,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,