diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index 4af242b7d6b..85eb7b0ce77 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -11,9 +11,9 @@ on: - litellm/deploy/docker/Dockerfile.dockerignore - litellm/deploy/docker/docker-entrypoint.sh - litellm/runtime/migrations/run.py - - gateway/main.py - - gateway/launch.py - - gateway/routes/allowlist.py + - litellm/runtime/gateway/main.py + - litellm/runtime/gateway/launch.py + - litellm/runtime/gateway/routes/allowlist.py - litellm/cli/** - litellm/runtime/backend/main.py - litellm/runtime/backend/routes/allowlist.py diff --git a/litellm/deploy/docker/Dockerfile b/litellm/deploy/docker/Dockerfile index ac13b66533f..0b109b12640 100644 --- a/litellm/deploy/docker/Dockerfile +++ b/litellm/deploy/docker/Dockerfile @@ -181,7 +181,7 @@ COPY --from=builder /app/.venv /app/.venv COPY --from=liteadmin-builder /opt/liteadmin /opt/liteadmin COPY --from=builder /app/litellm/deploy/docker/docker-entrypoint.sh /app/docker-entrypoint.sh COPY --from=builder /app/litellm/schema.prisma /app/litellm/schema.prisma -COPY --from=builder /app/gateway /app/gateway +COPY --from=builder /app/litellm/runtime/gateway /app/gateway COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py # enterprise/ is imported by source path at runtime (proxy_cli puts the # working directory on sys.path; litellm/proxy/hooks resolves diff --git a/litellm/deploy/helm/litellm/templates/ingress.yaml b/litellm/deploy/helm/litellm/templates/ingress.yaml index 8cd56f2f2f8..78d74adc1ef 100644 --- a/litellm/deploy/helm/litellm/templates/ingress.yaml +++ b/litellm/deploy/helm/litellm/templates/ingress.yaml @@ -86,7 +86,7 @@ {{- $uiPaths = append $uiPaths (dict "path" "/*.txt" "pathType" "ImplementationSpecific") }} {{- end }} {{/* - Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py. + Gateway data-plane prefixes — must mirror litellm/runtime/gateway/routes/allowlist.py. Versioned paths are listed explicitly to avoid routing management routes (e.g. /v1/access_group, /v2/key/info, /v1/tool/*, /v1/agents, /v1/workflows, /v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*, diff --git a/litellm/deploy/terraform/litellm/README.md b/litellm/deploy/terraform/litellm/README.md index 70011dbb393..22f768e3718 100644 --- a/litellm/deploy/terraform/litellm/README.md +++ b/litellm/deploy/terraform/litellm/README.md @@ -46,7 +46,7 @@ starts the process named by its first argument, so each workload only differs in the argument the module passes The load balancer routes gateway path prefixes (mirrored verbatim from -`gateway/routes/allowlist.py`) to the gateway, UI asset paths (`/`, +`litellm/runtime/gateway/routes/allowlist.py`) to the gateway, UI asset paths (`/`, `/litellm-asset-prefix/*`, `/_next/*`, `/favicon.ico`) to the UI, and everything else to the backend. diff --git a/litellm/deploy/terraform/litellm/aws/locals.tf b/litellm/deploy/terraform/litellm/aws/locals.tf index e15310ead3e..9a772ed7a81 100644 --- a/litellm/deploy/terraform/litellm/aws/locals.tf +++ b/litellm/deploy/terraform/litellm/aws/locals.tf @@ -1,4 +1,4 @@ -# Gateway path prefixes — mirrored verbatim from gateway/routes/allowlist.py +# Gateway path prefixes — mirrored verbatim from litellm/runtime/gateway/routes/allowlist.py # and the helm ingress in litellm/deploy/helm/litellm/templates/ingress.yaml. Anything not in # this list and not a UI asset path falls through to the backend (management # API) catch-all rule on the ALB. diff --git a/litellm/deploy/terraform/litellm/gcp/locals.tf b/litellm/deploy/terraform/litellm/gcp/locals.tf index ad3a0d385d9..9bbe8bfd879 100644 --- a/litellm/deploy/terraform/litellm/gcp/locals.tf +++ b/litellm/deploy/terraform/litellm/gcp/locals.tf @@ -1,4 +1,4 @@ -# Gateway path prefixes — mirrored verbatim from gateway/routes/allowlist.py +# Gateway path prefixes — mirrored verbatim from litellm/runtime/gateway/routes/allowlist.py # and litellm/deploy/helm/litellm/templates/ingress.yaml. URL maps use the "path matcher" # rule with `paths` lists; up to 10 path globs per rule, up to 50 rules # per matcher. Easily fits the gateway list in one rule per chunk-of-10. diff --git a/litellm/runtime/backend/main.py b/litellm/runtime/backend/main.py index 5ceb4fcc51b..435fc61e3ef 100644 --- a/litellm/runtime/backend/main.py +++ b/litellm/runtime/backend/main.py @@ -16,7 +16,7 @@ from starlette.applications import Starlette from starlette.routing import Mount from starlette.types import Lifespan -# See gateway/main.py for why we assemble DATABASE_URL(s) here before +# See litellm/runtime/gateway/main.py for why we assemble DATABASE_URL(s) here before # importing proxy_server. from litellm.proxy.db.db_url_settings import DatabaseURLSettings @@ -45,7 +45,7 @@ def _is_backend_route(route) -> bool: return any(path.startswith(prefix) for prefix in BACKEND_PATH_PREFIXES) -# See gateway/main.py for why the trim runs inside the lifespan instead of at +# See litellm/runtime/gateway/main.py for why the trim runs inside the lifespan instead of at # module scope. _proxy_lifespan: Final = app.router.lifespan_context diff --git a/gateway/launch.py b/litellm/runtime/gateway/launch.py similarity index 97% rename from gateway/launch.py rename to litellm/runtime/gateway/launch.py index c620c29fa7b..20025ce9374 100644 --- a/gateway/launch.py +++ b/litellm/runtime/gateway/launch.py @@ -1,6 +1,6 @@ """Gateway supervisor: assemble DATABASE_URL, start the in-container PgBouncer, then run uvicorn. -``gateway/main.py`` assembles ``DATABASE_URL`` inside every uvicorn worker, which +``litellm/runtime/gateway/main.py`` assembles ``DATABASE_URL`` inside every uvicorn worker, which is fine for a plain Postgres URL but not for the pooler: PgBouncer must be started exactly once per pod, before the workers fork, and the workers must be handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in diff --git a/gateway/main.py b/litellm/runtime/gateway/main.py similarity index 100% rename from gateway/main.py rename to litellm/runtime/gateway/main.py diff --git a/gateway/routes/__init__.py b/litellm/runtime/gateway/routes/__init__.py similarity index 100% rename from gateway/routes/__init__.py rename to litellm/runtime/gateway/routes/__init__.py diff --git a/gateway/routes/allowlist.py b/litellm/runtime/gateway/routes/allowlist.py similarity index 100% rename from gateway/routes/allowlist.py rename to litellm/runtime/gateway/routes/allowlist.py diff --git a/pyproject.toml b/pyproject.toml index f2586bfc0a0..faecaaac034 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -443,7 +443,7 @@ version_files = [ [tool.pytest.ini_options] norecursedirs = ["*.egg", ".*", "_darcs", "build", "CVS", "dist", "node_modules", "venv", "{arch}", "**/litellm/ui", "**/litellm/vscode-extension", "**/litellm/enterprise", "**/litellm/proxy-extras", "**/litellm/packaging", "**/litellm/rust", "**/litellm/scripts", "**/litellm/runtime"] -pythonpath = ["litellm/scripts"] +pythonpath = ["litellm/scripts", "litellm/runtime"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" markers = [ @@ -510,6 +510,7 @@ source = ["litellm"] relative_files = true [tool.ruff] +src = [".", "litellm/runtime"] lint.ignore = ["F405", "E402", "F403"] # The second group is the strict gate's graduates: rules the codebase already has zero # violations of, so they hard-fail here instead of being counted by the strict gate. @@ -570,6 +571,7 @@ banned-module-level-imports = ["prisma"] "litellm/proxy/utils.py" = ["F401"] [tool.pyright] +extraPaths = ["litellm/runtime"] include = ["litellm"] ignore = [] exclude = ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "tests/e2e_harness/claude_code", "tests/e2e/ui", "litellm/types/utils.py", "litellm/proxy/_types.py", "litellm/scripts", "litellm/runtime", "litellm/deploy", "litellm/ui", "litellm/vscode-extension", "litellm/enterprise", "litellm/proxy-extras", "litellm/packaging", "litellm/rust", "litellm/proxy/discovery_endpoints/skill_scanner_adapter"] diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index d682fcbad2d..a55c4a0126e 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -35,8 +35,10 @@ def test_a_uvicorn_worker_reimports_the_launcher_without_loading_litellm(): sys.executable, "-I", "-c", - 'import runpy, sys; runpy.run_module("gateway.launch", run_name="__mp_main__", alter_sys=True); ' + 'import runpy, sys; sys.path.insert(0, sys.argv[1]); ' + 'runpy.run_module("gateway.launch", run_name="__mp_main__", alter_sys=True); ' 'print("litellm" in sys.modules)', + str(Path(__file__).resolve().parents[3] / "litellm" / "runtime"), ], capture_output=True, text=True, diff --git a/tests/unit/proxy/test_component_allowlists.py b/tests/unit/proxy/test_component_allowlists.py index 2ff4461f622..8bd8cc8d772 100644 --- a/tests/unit/proxy/test_component_allowlists.py +++ b/tests/unit/proxy/test_component_allowlists.py @@ -106,6 +106,7 @@ for _key, _previous in _PRE_DB_ENV.items(): _COVERAGE_PROBE: Final = """ import json, os, sys sys.path.insert(0, os.environ["LITELLM_COMPONENT_ALLOWLIST_REPO_ROOT"]) +sys.path.insert(0, os.path.join(os.environ["LITELLM_COMPONENT_ALLOWLIST_REPO_ROOT"], "litellm", "runtime")) from starlette.routing import Mount from litellm.runtime.backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES @@ -286,7 +287,7 @@ def test_gateway_plus_backend_covers_full_app(): uncovered: Final = report["uncovered"] assert not uncovered, ( f"{len(uncovered)} route(s) are not exposed on either component. " - f"Update gateway/routes/allowlist.py or litellm/runtime/backend/routes/allowlist.py to cover:\n " + f"Update litellm/runtime/gateway/routes/allowlist.py or litellm/runtime/backend/routes/allowlist.py to cover:\n " + "\n ".join(uncovered) ) diff --git a/tests/unit/rust_bridge/test_verify_linux_native_wheel.py b/tests/unit/rust_bridge/test_verify_linux_native_wheel.py index 5c05a2e1e5a..9d9ebb6e6c0 100644 --- a/tests/unit/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/unit/rust_bridge/test_verify_linux_native_wheel.py @@ -147,6 +147,7 @@ def test_accepts_expected_release_wheel_tags(tmp_path: Path) -> None: "litellm/packaging/litellm-core/pyproject.toml", "litellm/scripts/default_branch.py", "litellm/runtime/backend/main.py", + "litellm/runtime/gateway/launch.py", "litellm/schema.prisma", ), )