mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
chore: remove bundled root skills and relocate GitGuardian config
This commit is contained in:
Devin AI
parent
6043307625
commit
9df55d637d
10 changed files with 2 additions and 558 deletions
|
|
@ -1,73 +0,0 @@
|
|||
---
|
||||
name: database-access
|
||||
description: Organize database repositories and proxy dependency wiring, manage connection ownership, or port Prisma operations to Rust and SQLx without expanding migration scope
|
||||
metadata:
|
||||
internal: true
|
||||
---
|
||||
|
||||
# Database access
|
||||
|
||||
## Situations
|
||||
|
||||
- Adding a query or an operation to a repository, read [adding a query](references/python-repository.md#adding-a-query)
|
||||
- Moving existing Prisma calls behind a repository, read [moving existing access](references/python-repository.md#moving-existing-access-into-a-repository)
|
||||
- Porting a repository from Prisma to Rust and SQLx, read [porting to SQLx](references/python-repository.md#porting-a-repository-to-sqlx)
|
||||
- Injecting a repository into a route, read [Python and FastAPI](references/python-fastapi.md) or [Rust and Axum](references/rust-axum.md)
|
||||
|
||||
## Adding new database access
|
||||
|
||||
- Put Python database access behind an operation in `litellm/repositories`
|
||||
- Give each repository its own FastAPI provider, so a route builds only the repositories it uses
|
||||
- A provider borrows existing handles and performs no database I/O during construction
|
||||
- Define the operations a route calls as a Protocol in `litellm/models/<feature>.py`, next to its domain types
|
||||
- The Protocol takes and returns domain types, never Prisma models, raw rows or serialized payloads
|
||||
- Parsing and serialization of stored values live in the repository
|
||||
- Add a Protocol when a repository gets a provider, not up front for every repository
|
||||
- Keep query execution inside the repository, with one implementation per repository and no backend switch or fallback
|
||||
- A repository is either all Prisma or all native, never a mix of both
|
||||
- Prisma is fine for the implementation, adding a query does not require a Rust port
|
||||
- Native Rust callers use `litellm/rust/crates/db` directly
|
||||
|
||||
## Framework wiring
|
||||
|
||||
- Python routes receive a per-repository dependency such as `UserBannerRepositoryDep` and call its operations, such as `banners.get_banner()`
|
||||
- `DatabaseDep` lives in `litellm/proxy/deps/database.py`, repository providers in `litellm/proxy/deps/repositories.py`
|
||||
- A provider returns the feature's Protocol, so routes never see which backend implements it
|
||||
- Dependencies used by one feature only, like the banner audit binding, stay next to that feature's routes
|
||||
- Repositories never import FastAPI, so jobs and helpers can use them without a request
|
||||
- Use named FastAPI providers with `Annotated[..., Depends(...)]`, ordinary helpers and jobs receive explicit arguments
|
||||
- Preserve each route's no-database behavior, authorization precedence and background-task scheduling
|
||||
- Read [Python and FastAPI](references/python-fastapi.md) for provider examples, lifespan state and route verification
|
||||
- Read [Rust and Axum](references/rust-axum.md) when adding database access to Rust handlers
|
||||
|
||||
## Connection ownership
|
||||
|
||||
- Keep connection setup and shutdown separate from repositories, which own no pools
|
||||
- Reuse existing handles, never connect per request or disconnect from a repository operation
|
||||
- Give each pool one owner for setup and shutdown, borrowed handles never close it
|
||||
- Finish requests, jobs and queued database work before closing connections
|
||||
- Clean up resources opened before a startup failure, without a second disconnect fallback
|
||||
- Add a backend to the existing lifecycle owner and pass its handle only to repositories that need it
|
||||
- Preserve cancellation, timeout, error and transaction semantics across the native bridge
|
||||
|
||||
## Changing ownership or reloading resources
|
||||
|
||||
- Inventory initialization, reload, tests, background jobs and shutdown together
|
||||
- Move every writer when changing authoritative storage, copying a replaceable global into state is insufficient
|
||||
- A frozen repository does not make its clients immutable
|
||||
- Decide whether each request captures one resource generation or reads it again during processing
|
||||
- Build and validate replacements before publishing them, preserving the old generation on failure
|
||||
- Serialize reloads and coordinate publication with shutdown
|
||||
- Drain streams and background users before retiring old resources, retaining every generation grows resource use
|
||||
- Define multi-worker coordination separately from an in-process swap
|
||||
- Choose mechanisms such as `ArcSwap` only after defining publication, capture and retirement
|
||||
|
||||
## Migration context and boundaries
|
||||
|
||||
- Long-term direction is to centralize Python database access in repositories, then replace their Prisma implementations with Rust APIs one repository at a time
|
||||
- Prisma and native repositories coexist during the transition, unrelated existing calls can stay as they are
|
||||
- Prisma and SQLx keep separate client pools to PgBouncer, which pools connections to PostgreSQL
|
||||
- Size both layers together and preserve reader/writer routing and pooling-mode compatibility
|
||||
- PgBouncer does not share transactions between Prisma and SQLx, keep each atomic unit of work on one backend and transaction
|
||||
- Never retry a failed Rust write through Prisma when it may already have committed
|
||||
- Prisma owns schema migrations today, eventual SQLx migration ownership and Prisma removal are separate planned steps, not part of a query port
|
||||
|
|
@ -1,101 +0,0 @@
|
|||
# Python and FastAPI
|
||||
|
||||
## Existing pattern
|
||||
|
||||
- `UserBannerStore` in `litellm/models/user_banner.py` is the Protocol the banner routes call
|
||||
- `get_database` in `litellm/proxy/deps/database.py` reads the lifespan-owned handle from request state
|
||||
- `get_user_banner_repository` in `litellm/proxy/deps/repositories.py` builds the banner repository from that handle
|
||||
- Choose each repository's replica routing once in its provider, never at call sites
|
||||
- A handler receives `UserBannerStore | None`, never a Prisma client or a concrete repository
|
||||
- `UserBannerRepository` implements `UserBannerStore` structurally, keeping the existing Prisma table, serialization and configuration-sync behavior
|
||||
- The banner audit provider binds the legacy audit callable at request time, without performing audit I/O
|
||||
- A native repository can later implement the same Protocol without changing handler signatures, see [porting a repository to SQLx](python-repository.md#porting-a-repository-to-sqlx)
|
||||
|
||||
## Dependency injection
|
||||
|
||||
- Add one provider and one `Annotated` alias per repository in `litellm/proxy/deps/repositories.py`
|
||||
- Annotate the provider and alias with the feature's Protocol, not the concrete repository
|
||||
- A route depends only on the repositories it calls, so a request never builds repositories it does not use
|
||||
- Do not build repositories once at lifespan, a reconnect can replace the Prisma client they wrap
|
||||
|
||||
```python
|
||||
class UserBannerStore(Protocol):
|
||||
async def get_banner(self) -> UserBanner: ...
|
||||
|
||||
async def save_banner(self, banner: UserBanner) -> None: ...
|
||||
```
|
||||
|
||||
```python
|
||||
async def get_user_banner_repository(database: DatabaseDep) -> UserBannerStore | None:
|
||||
return None if database is None else UserBannerRepository(database.prisma)
|
||||
|
||||
|
||||
UserBannerRepositoryDep: TypeAlias = Annotated[UserBannerStore | None, Depends(get_user_banner_repository)]
|
||||
```
|
||||
|
||||
- FastAPI caches each provider within a request, so routes and sub-dependencies share one instance
|
||||
- Ordinary helpers and jobs receive explicit arguments, never unresolved `Depends` defaults
|
||||
- A route still reading the `prisma_client` global or building a repository inline switches to the repository's `...RepositoryDep`, keeping its no-database behavior and error precedence
|
||||
|
||||
```python
|
||||
async def get_user_banner(banners: UserBannerRepositoryDep) -> UserBanner:
|
||||
if banners is None:
|
||||
return UserBanner()
|
||||
return await banners.get_banner()
|
||||
```
|
||||
|
||||
## Lifespan state
|
||||
|
||||
- Prisma startup owns connection setup, authentication refresh, health checks and migrations
|
||||
- `Database` holds the initialized Prisma client and the `NativeDatabase` handle, without holding fields per feature
|
||||
- The native handle opens no pool yet, its `connect` and `close` are noops
|
||||
- The proxy lifespan builds the handle with `Database.open`, registers `Database.aclose` on its exit stack and publishes it in request state
|
||||
- Callers only open and close `Database`, which closes Prisma and the native handle together, and `open` closes Prisma itself when the native connect fails
|
||||
- Starlette shallow-copies lifespan state into request state, which is distinct from `app.state`
|
||||
- The database module never imports a repository, legacy globals borrow its handle without owning shutdown
|
||||
- Drain requests, jobs, logging, spend and audit work before database teardown
|
||||
- The exit stack closes the handle on exit, including startup failures after the client is created
|
||||
- When adding native resources, initialize them in the serving process, respecting runtime fork restrictions
|
||||
- Await native shutdown instead of depending on Python garbage collection
|
||||
- Apply the [connection ownership rules](../SKILL.md#connection-ownership) when changing the owner
|
||||
|
||||
## Timing and errors
|
||||
|
||||
- FastAPI resolves dependencies before entering the handler
|
||||
- Handler exception handling cannot catch failures raised during dependency resolution
|
||||
- A universal required-database dependency can override authorization and validation behavior
|
||||
- Banner GET without a database returns the default disabled banner
|
||||
- Banner PATCH checks the admin role before returning the existing missing-database error
|
||||
- Invalid stored banner data falls back to the default, but storage failures still propagate
|
||||
- Banner writes finish before the success audit is queued as a FastAPI background task
|
||||
- Queue post-response work with `BackgroundTasks`, not bare `asyncio.create_task`, so the in-flight drain waits for it before the database closes
|
||||
- Do not move background work into an awaited provider merely to remove an import
|
||||
|
||||
## Verification
|
||||
|
||||
- Exercise real FastAPI handlers using `dependency_overrides` for external boundaries
|
||||
- Keep production parsing and persistence code real
|
||||
- Restore overrides after each test
|
||||
- Verify complete response bodies, status codes and OpenAPI against the base
|
||||
- Include combined failure conditions to pin error precedence
|
||||
- Check saved values, fresh revisions, audit contents and failed-write behavior
|
||||
- Check lifespan state with a running lifespan, not only a TestClient that skips startup
|
||||
- Verify providers do not connect, query, disconnect or audit eagerly
|
||||
- Use real PostgreSQL and live proxy requests when changing persistence behavior
|
||||
|
||||
## Scope
|
||||
|
||||
- Preserve direct Python call sites or migrate them explicitly
|
||||
- Inventory imports, state writers, side effects and error ordering before migrating a feature
|
||||
- Preserve configuration sync and serialization, and state which legacy reads remain
|
||||
- Do not migrate every route in a large module just to establish the pattern
|
||||
- Do not invent providers for resources without consumers
|
||||
|
||||
## Framework references
|
||||
|
||||
- [Lifespan events](https://fastapi.tiangolo.com/advanced/events/) and [Starlette lifespan state](https://starlette.dev/lifespan/)
|
||||
- [`SessionDep`-style `Annotated` aliases](https://fastapi.tiangolo.com/tutorial/sql-databases/#create-a-session-dependency), which `DatabaseDep` and `UserBannerRepositoryDep` follow
|
||||
- [Dependency composition and caching](https://fastapi.tiangolo.com/tutorial/dependencies/sub-dependencies/)
|
||||
- [Background tasks and dependencies with `yield`](https://fastapi.tiangolo.com/advanced/advanced-dependencies/#background-tasks-and-dependencies-with-yield-technical-details)
|
||||
- [Dependency overrides](https://fastapi.tiangolo.com/advanced/testing-dependencies/)
|
||||
- [Testing lifespan](https://fastapi.tiangolo.com/advanced/testing-events/)
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
# Python repositories
|
||||
|
||||
## Adding a query
|
||||
|
||||
Use this when a feature needs a read or write that no repository offers yet, including one more operation on an existing repository
|
||||
|
||||
### Before starting
|
||||
|
||||
- Search `litellm/repositories` for a repository that already owns the table, and extend it instead of adding a second one
|
||||
- Check `litellm/models/<feature>.py` for an existing Protocol and domain types
|
||||
- Decide the route's behavior when no database is configured, and where that check sits relative to authorization and validation
|
||||
|
||||
### Steps
|
||||
|
||||
1. Domain types and Protocol in `litellm/models/<feature>.py`
|
||||
- The Protocol takes and returns domain types, never Prisma models, raw rows or serialized payloads
|
||||
- Add the Protocol together with the provider, not for repositories nothing injects
|
||||
2. Repository in `litellm/repositories/<feature>_repository.py`
|
||||
- Implement the operation with Prisma, a Rust port is not required
|
||||
- Compose the Prisma client and type the tables it touches with `TableActions` from `prisma_protocols.py`, never inherit `PrismaTableRepository`
|
||||
- Parse and serialize stored values here, so callers only see domain types
|
||||
- Never import FastAPI, connect, disconnect or choose replicas per call
|
||||
3. Provider and `Annotated` alias in `litellm/proxy/deps/repositories.py`, see [dependency injection](python-fastapi.md#dependency-injection)
|
||||
4. The route takes `<Feature>RepositoryDep`, handles `None` with the behavior decided above, and calls the operation
|
||||
5. Jobs and helpers that need the operation receive the repository as an explicit argument, built by their caller from the handle it already holds
|
||||
|
||||
### Done when
|
||||
|
||||
- Handler tests drive the real route with `dependency_overrides` and the real repository code, see [verification](python-fastapi.md#verification)
|
||||
- Tests cover the no-database path, error precedence, and what was actually stored
|
||||
- Persistence changes are checked against real PostgreSQL through a live proxy
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Porting the query to Rust, see [porting a repository to SQLx](#porting-a-repository-to-sqlx)
|
||||
- Moving unrelated direct Prisma calls in the same module
|
||||
|
||||
## Moving existing access into a repository
|
||||
|
||||
Use this when code calls Prisma directly, through `prisma_client.db.<table>` or a helper, and the calls should live behind a repository. The backend stays Prisma
|
||||
|
||||
### Before starting
|
||||
|
||||
- List every call being moved, its callers, and the transaction or batch it runs in
|
||||
- List the module's imports, global state writers, side effects such as audit and config sync, and error ordering
|
||||
- Record current return values, raised errors and response bodies, these are what the change must preserve
|
||||
|
||||
### Steps
|
||||
|
||||
1. Move the existing Prisma code into the relevant repository in `litellm/repositories` without rewriting the queries
|
||||
2. Add domain types and a Protocol in `litellm/models/<feature>.py` when a route will inject the repository
|
||||
3. Replace each direct call with the repository operation, through a provider for routes and explicit arguments for jobs and helpers, see [Python and FastAPI](python-fastapi.md)
|
||||
4. Calls sharing a transaction move into the same repository or unit of work in one change
|
||||
5. State which legacy reads remain and why, when the change does not move all of them
|
||||
|
||||
### Done when
|
||||
|
||||
- Focused regression tests pin the preserved return values, errors and transaction behavior
|
||||
- Response bodies, status codes and OpenAPI match the base
|
||||
- Configuration sync and serialization behave the same, including invalid stored data
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Switching the backend, even when an equivalent Rust query exists
|
||||
- Converting other routes in the module just to establish the pattern
|
||||
- Cleanups unrelated to the moved calls
|
||||
|
||||
## Porting a repository to SQLx
|
||||
|
||||
Status: TBD. No repository has been ported yet. `database.native` is a `NativeDatabase` from `crates/python-bridge/src/db.rs` whose `connect` and `close` are noops that open no pool, and the PyO3 repository surface does not exist. A real pool takes typed writer/reader settings and a token provider, never the Prisma `DATABASE_URL`, which carries Prisma-only params and rotating IAM tokens. The constraints below are settled, the steps are a sketch to revise after the first port lands
|
||||
|
||||
Use this only when a port is explicitly requested. Repository extraction does not require one
|
||||
|
||||
### When not to port
|
||||
|
||||
- An equivalent Rust or SQLx query existing is not a reason to switch a Prisma caller
|
||||
- Leave existing backend selection unchanged unless integration is explicitly requested
|
||||
- Never port part of a repository, it is either all Prisma or all native
|
||||
|
||||
### Before starting
|
||||
|
||||
- Identify the repository's inputs, outputs, errors and transaction boundary
|
||||
- List repositories that share a transaction with it, they move together
|
||||
- Check `litellm/rust/crates/db` for an equivalent query and reuse it when its behavior matches
|
||||
|
||||
### Steps
|
||||
|
||||
1. Rust queries, following `litellm/rust/crates/db/AGENTS.md`
|
||||
- Static SQL in `queries/`, compile-checked with SQLx, typed inputs and row projections in `src/`
|
||||
- Dynamic filters and sort keys become closed Rust enums that select static SQL, every value is bound
|
||||
- Refresh SQLx query metadata after SQL changes
|
||||
2. PyO3 surface exposing typed operations, never SQL text, table names or column names from Python. TBD: where it lives and how the handle reaches it
|
||||
3. Native repository implementing the same Protocol in `litellm/repositories`
|
||||
- Converts typed PyO3 results to domain types, preserving defaults, invalid-data handling and storage errors
|
||||
- Python conversion stays in the bridge, not in `crates/db`
|
||||
4. Switch the provider in `litellm/proxy/deps/repositories.py`, the Protocol and routes stay unchanged
|
||||
|
||||
```diff
|
||||
async def get_user_banner_repository(database: DatabaseDep) -> UserBannerStore | None:
|
||||
- return None if database is None else UserBannerRepository(database.prisma)
|
||||
+ return None if database is None else NativeUserBannerRepository(database.native)
|
||||
```
|
||||
|
||||
5. Delete the Prisma repository in the same change and switch every caller, including jobs and helpers
|
||||
|
||||
### Done when
|
||||
|
||||
- Behavior is verified against real PostgreSQL, covering null handling, conflicts, ordering, errors and transaction semantics
|
||||
- `DATABASE_URL=... cargo test -p litellm-db --locked -- --include-ignored` passes against a Prisma-migrated database
|
||||
- Live proxy requests show unchanged responses. TBD: a Python-side real-PostgreSQL test harness
|
||||
|
||||
### Rollback
|
||||
|
||||
- Roll back the release, never retry an uncertain native write through Prisma
|
||||
|
||||
### Out of scope
|
||||
|
||||
- SQLx owning schema migrations, and removing Prisma
|
||||
- Converting unrelated routes to dependency injection
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
# Rust and Axum
|
||||
|
||||
- This should use same `crates/db` what is exposed to Python world.
|
||||
- Details are TBD.
|
||||
|
|
@ -1,189 +0,0 @@
|
|||
---
|
||||
name: litellm-setup
|
||||
description: Set up a self-hosted LiteLLM proxy, the OpenAI-compatible LLM gateway, on this machine, from install through provider keys, config.yaml, a master key, optional Postgres and Admin UI, and a verified chat completion. Use when the user asks to set up LiteLLM, install the LiteLLM proxy, self-host an LLM gateway, or put one endpoint in front of OpenAI, Anthropic, Gemini, Azure, Bedrock, Ollama, or other providers. Not for using the LiteLLM Python SDK inside an app
|
||||
---
|
||||
|
||||
# Set up a LiteLLM proxy
|
||||
|
||||
Take the user from nothing to a running LiteLLM proxy that has answered one chat completion. Run every command yourself unless a step says the user does it
|
||||
|
||||
Never print a secret. Provider keys, the master key, the salt key, the database URL, and virtual keys live only in `.env`, and `config.yaml` reads them through `os.environ/NAME`. Check that a value is there with `grep -c '^NAME=.' .env`, never with `cat .env` or `echo "$NAME"`. Load `.env` into a shell with this line, which exports each line as it is, so a `&` or `$` in a value stays literal, and first drops any database settings the shell already had and sets `LITELLM_MODE=PRODUCTION`, which stops `litellm` from also loading a `.env` it finds above its install folder, such as `~/.env`, so only this `.env` picks the database. The steps that need it start with it:
|
||||
|
||||
```bash
|
||||
unset DATABASE_URL DIRECT_URL DATABASE_HOST; export LITELLM_MODE=PRODUCTION; while IFS= read -r line || [ -n "$line" ]; do case "$line" in [A-Za-z_]*=*) export "$line" ;; esac; done < .env
|
||||
```
|
||||
|
||||
## 1. Ask once
|
||||
|
||||
Ask everything in one message, with defaults the user can accept by saying "go":
|
||||
|
||||
- Providers and models. Default is OpenAI with its newest flagship model, using the `OPENAI_API_KEY` in your environment when `[ -n "$OPENAI_API_KEY" ] && echo set` says it is there. Run that check before asking, and say in the question that the default uses that variable, naming it but never its value, since a key in your environment can belong to another account
|
||||
- Whether they want virtual keys, teams, spend tracking, or the Admin UI. Those need Postgres, plain routing does not. Default is no
|
||||
- uv (default) or Docker. For Docker with Postgres, use the deployment guide in step 3. uv needs a Postgres server they already have, with a new, empty database for LiteLLM
|
||||
- Folder and port. Defaults are `./litellm-proxy` and 4000
|
||||
|
||||
Then follow the steps for their answers. Plain routing skips steps 3 and 6. uv with their own Postgres skips step 3
|
||||
|
||||
In the same message, say that someone who picks uv and would rather answer prompts can run `litellm --setup` in a separate terminal window once LiteLLM is installed, not through Claude Code. Never run it yourself, since it needs a real terminal and without one prints "Setup cancelled." and writes nothing. Give them the command as `LITELLM_MODE=PRODUCTION <bin>/litellm --setup`, with `<bin>` replaced by the folder `uv tool dir --bin` prints, to run from the folder they want the config in. It writes `litellm_config.yaml` there with the provider keys and master key in plain text and offers to start the proxy, which then listens on every network interface and stops with Ctrl+C in that window. When the user is done, run this in that folder. It makes the file readable only by them, keeps it out of git (the wizard does neither), and prints only the model names. Never print that file or run `grep`, `sed`, or `cat` on it:
|
||||
|
||||
```bash
|
||||
chmod 600 litellm_config.yaml; grep -qx 'litellm_config.yaml' .gitignore 2>/dev/null || echo 'litellm_config.yaml' >> .gitignore
|
||||
"$(uv tool dir)/litellm/bin/python" -I -c 'import yaml; print(*(m["model_name"] for m in yaml.safe_load(open("litellm_config.yaml"))["model_list"]), sep="\n")'
|
||||
```
|
||||
|
||||
Then skip to step 8 and run it against their proxy, starting that shell command with this line instead of the loader line, which reads the master key without printing it:
|
||||
|
||||
```bash
|
||||
export LITELLM_MASTER_KEY="$("$(uv tool dir)/litellm/bin/python" -I -c 'import yaml; print(yaml.safe_load(open("litellm_config.yaml"))["general_settings"]["master_key"])')"
|
||||
```
|
||||
|
||||
## 2. Check prerequisites
|
||||
|
||||
```bash
|
||||
uv --version; docker info --format '{{.ServerVersion}}'; docker compose version; curl --version | head -1; openssl version; command -v python3
|
||||
```
|
||||
|
||||
On native Windows, ask the user to run Claude Code inside WSL, since every command here is POSIX shell. Every path needs curl, openssl, and python3, so ask the user to install any that is missing. The uv path needs uv, which downloads a supported Python (LiteLLM supports 3.10 to 3.14) when the `--python` range in step 4 rules out the one on the machine. If uv is missing, offer to install it, and only after the user says yes run `curl -LsSf https://astral.sh/uv/install.sh | env UV_NO_MODIFY_PATH=1 sh`, which leaves their shell profile alone (https://docs.astral.sh/uv/getting-started/installation/). It puts uv in `~/.local/bin`, so start every later command that runs `uv` or `litellm` with `export PATH="$HOME/.local/bin:$PATH";`. The Docker path needs a running Docker daemon, and Docker with Postgres also needs Docker Compose v2. If `docker info` says `permission denied`, the user's account cannot use Docker without sudo. Offer the uv path, or ask them to add themselves to the `docker` group the way https://docs.docker.com/engine/install/linux-postinstall/ shows and start a new login session before you check again
|
||||
|
||||
Check the port is free with `curl -s -o /dev/null --max-time 2 http://127.0.0.1:4000; echo $?`, which prints `7` (could not connect) only when nothing listens there. Don't use `lsof` for this, since it misses ports that root or Docker hold. If it is taken, pick another port and use it everywhere below, because a proxy asked for a busy 4000 silently moves to a random port
|
||||
|
||||
## 3. Postgres with Docker
|
||||
|
||||
Follow [the container deployment guide](../../../litellm/deploy/docker/README.md) instead of the remaining steps in this skill. Deployment tooling is maintained separately from these local uv and standalone Docker instructions.
|
||||
|
||||
## 4. Install
|
||||
|
||||
uv:
|
||||
|
||||
```bash
|
||||
uv tool install --python '>=3.10,<3.15' 'litellm[proxy]'
|
||||
export PATH="$(uv tool dir --bin):$PATH"; litellm --version
|
||||
```
|
||||
|
||||
With their own Postgres, install `'litellm[proxy,extra_proxy]'` instead, with the same `--python` range, which adds the Prisma client the proxy uses for the database. Running `uv tool install` again with the new extras replaces the old install. Start every later command that runs `litellm` with `export PATH="$(uv tool dir --bin):$PATH";`, so this install runs even when the shell has no `litellm` or an older one from a venv or `pip install litellm`. `uv tool update-shell` puts it on the PATH of new shells by editing the user's shell profile, so run it only after the user says yes
|
||||
|
||||
Docker:
|
||||
|
||||
```bash
|
||||
docker pull docker.litellm.ai/berriai/litellm:main-stable
|
||||
```
|
||||
|
||||
## 5. Write `.env` and `config.yaml`
|
||||
|
||||
Run every command from here on in the folder from step 1. Create `.env` readable only by the user, keep it out of git, and add a master key unless it already has one. The proxy refuses to start without one, and `sk-` is the format of every key it issues:
|
||||
|
||||
```bash
|
||||
mkdir -p litellm-proxy && cd litellm-proxy
|
||||
touch .env && chmod 600 .env; grep -qx '.env' .gitignore 2>/dev/null || echo '.env' >> .gitignore
|
||||
grep -q '^LITELLM_MASTER_KEY=.' .env || printf 'LITELLM_MASTER_KEY=sk-%s\n' "$(openssl rand -hex 32)" >> .env
|
||||
```
|
||||
|
||||
Copy a provider key that is already in your environment only after the user agreed to that variable, in their step 1 answer or when you ask now, and copy it without printing it, as in `printf 'OPENAI_API_KEY=%s\n' "$OPENAI_API_KEY" >> .env`. Otherwise ask the user to add the line `OPENAI_API_KEY=<their key>` to `.env` themselves and tell you when it is there. If they paste a key into the chat anyway, write it to `.env` and never repeat it. Keep one `NAME=value` per line with no quotes, so `docker --env-file` and the loader line read the same value
|
||||
|
||||
Confirm each model's current LiteLLM name in the cost map instead of trusting memory, since model names go stale fast. This prints the chat models of the provider named last (`openai`, `anthropic`, `gemini`, `azure`, `bedrock`, `vertex_ai`, and so on) the way `config.yaml` takes them, with the provider in front. It matches by prefix, so `bedrock` also lists the newer Bedrock models and `vertex_ai` lists every Vertex family. For Ollama, use a model from `ollama list` instead:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | python3 -c '
|
||||
import json, sys
|
||||
for name, row in json.load(sys.stdin).items():
|
||||
provider = str(row.get("litellm_provider", "")) if isinstance(row, dict) else ""
|
||||
if provider.startswith(sys.argv[1]) and row.get("mode") == "chat":
|
||||
prefix = "bedrock" if provider == "bedrock_converse" else "vertex_ai" if provider.startswith("vertex_ai") else provider
|
||||
print(name if name.startswith(prefix + "/") else prefix + "/" + name)
|
||||
' openai
|
||||
```
|
||||
|
||||
In `config.yaml`, `model` is a name from that list, as in `openai/gpt-6.1-sol` or `bedrock/us.anthropic.<name>`, and `model_name` is what clients will send. If the folder already has a `config.yaml`, show the user its `model_name` lines and ask before replacing it. This example uses the newest GPT model in the cost map when this skill was written, so replace it with what the user picked:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-6.1-sol
|
||||
litellm_params:
|
||||
model: openai/gpt-6.1-sol
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
```
|
||||
|
||||
Add one entry per model. On Azure, ask the user for each deployment's name and the model it runs. Use `azure/<deployment name>` as `model`, since Azure routes by deployment and a name from the lookup works only when the deployment has that same name, and add `model_info:` with `base_model: azure/<model it runs>` to the entry, since LiteLLM prices each call by that name. Providers that need more than a key, like Azure or Bedrock, list their `litellm_params` on their page under https://docs.litellm.ai/docs/providers. Point every secret at `.env` through `os.environ/NAME` the same way
|
||||
|
||||
## 6. Their own Postgres, on the uv path
|
||||
|
||||
Ask the user to create a new, empty database for LiteLLM on their server (`CREATE DATABASE litellm;`) and add `DATABASE_URL=postgresql://...` for it to `.env` themselves. The proxy creates its tables there on first start and stops on a database that already holds other tables. If that URL goes through a transaction pooler such as PgBouncer, they also add `DIRECT_URL` for the same database without the pooler, which the migrations use. Then add a salt key. It encrypts the provider credentials the proxy stores in the database and falls back to the master key when unset, so set it once and never change it:
|
||||
|
||||
```bash
|
||||
grep -q '^LITELLM_SALT_KEY=.' .env || printf 'LITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" >> .env
|
||||
```
|
||||
|
||||
Generate the Prisma client once inside the tool's environment. A pip or uv install doesn't ship it generated (Docker images do):
|
||||
|
||||
```bash
|
||||
tool="$(uv tool dir)/litellm"
|
||||
schema=$("$tool/bin/python" -I -c 'import importlib.util, os; print(os.path.join(importlib.util.find_spec("litellm").submodule_search_locations[0], "proxy", "schema.prisma"))')
|
||||
PATH="$tool/bin:$PATH" prisma generate --schema "$schema"
|
||||
```
|
||||
|
||||
## 7. Start the proxy
|
||||
|
||||
Bind to `127.0.0.1` unless the user wants other machines to reach it
|
||||
|
||||
uv, in one shell command that starts with the loader line, since the installed `litellm` does not read `.env` by itself:
|
||||
|
||||
```bash
|
||||
export PATH="$HOME/.local/bin:$PATH"; export PATH="$(uv tool dir --bin):$PATH"
|
||||
nohup litellm --config config.yaml --host 127.0.0.1 --port 4000 > litellm.log 2>&1 &
|
||||
echo $! > litellm.pid
|
||||
```
|
||||
|
||||
Docker:
|
||||
|
||||
```bash
|
||||
docker run -d --name litellm --env-file .env -v "$PWD/config.yaml:/app/config.yaml:ro" -p 127.0.0.1:4000:4000 \
|
||||
docker.litellm.ai/berriai/litellm:main-stable --config /app/config.yaml --port 4000
|
||||
```
|
||||
|
||||
Wait on the health route, stopping early if the proxy exits. The first start with Postgres applies the database migrations and takes longer. Define `alive` for the path you took, `alive() { kill -0 "$(cat litellm.pid)" 2>/dev/null; }` for uv or `alive() { [ "$(docker inspect -f '{{.State.Running}}' litellm 2>/dev/null)" = true ]; }` for Docker, in the same shell command as the loop:
|
||||
|
||||
```bash
|
||||
for i in $(seq 1 180); do curl -fsS http://127.0.0.1:4000/health/liveliness >/dev/null 2>&1 && break; alive || break; sleep 1; done
|
||||
curl -sS http://127.0.0.1:4000/health/readiness
|
||||
```
|
||||
|
||||
Readiness reports `"db":"connected"` with Postgres and `"db":"Not connected"` without it. If the proxy never comes up, read the end of `litellm.log` (or `docker logs litellm`) and check the troubleshooting list
|
||||
|
||||
## 8. Verify with a chat completion
|
||||
|
||||
Use a `model_name` from `config.yaml`, in one shell command that starts with the loader line:
|
||||
|
||||
```bash
|
||||
curl -sS http://127.0.0.1:4000/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" -H "Content-Type: application/json" \
|
||||
-d '{"model": "gpt-6.1-sol", "messages": [{"role": "user", "content": "Say hello in five words."}]}'
|
||||
```
|
||||
|
||||
Show the user the reply in `choices[0].message.content`. An error here means the setup is not done, so fix it before handing off
|
||||
|
||||
## 9. Hand off
|
||||
|
||||
Tell the user, without printing any secret:
|
||||
|
||||
- The base URL is `http://127.0.0.1:4000`. Any OpenAI-compatible client works with it as the base URL and the master key or a virtual key as the API key
|
||||
- The master key is `LITELLM_MASTER_KEY` in the folder's `.env`, or `general_settings.master_key` in `litellm_config.yaml` when they used `litellm --setup`
|
||||
- To stop it, `kill "$(cat litellm.pid)"` (uv), `docker stop litellm` (Docker), or Ctrl+C in its window (`litellm --setup`). To start it again, `docker start litellm`, on uv the whole step 7 command as you ran it, loader line and PATH export included, wrapped in `( ... )` so their shell keeps its own settings, shown as a code block and never paraphrased, since a new shell has neither, or for `litellm --setup` `LITELLM_MODE=PRODUCTION <bin>/litellm --config litellm_config.yaml --host 127.0.0.1 --port <port>` in a terminal window in that folder, with `<bin>` and `<port>` filled in, since running the wizard again writes a new config with a new master key.
|
||||
- With Postgres, the Admin UI is at `http://127.0.0.1:4000/ui` (user `admin`, password the master key), and `POST /key/generate` with the master key creates virtual keys with their own budgets and model lists. Without Postgres, both need step 3 or 6 first
|
||||
- The Admin UI warns that the master-key login is a shared secret, which is fine while only this machine reaches the proxy. Before anyone else can reach it, invite an admin with their own password on the Internal Users page, then set `general_settings.disable_env_credential_login: true` in the proxy config
|
||||
- To manage the running proxy from a coding agent (keys, teams, users, models, MCP servers), install `npx skills add BerriAI/litellm-skills -g`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- `refused to start: no master key is set` means the loader line did not run in the shell command that started `litellm`, or `--env-file .env` is missing from `docker run`
|
||||
- `a database URL is set but the prisma CLI is neither on PATH nor importable` means the install lacks `extra_proxy`, so rerun step 4 with it, or, on plain routing, that the command that started `litellm` did not start with the loader line
|
||||
- `The Client hasn't been generated yet` or `Unable to find Prisma binaries` means the Prisma generate command in step 6 has not run
|
||||
- `Incorrect API key provided` or another provider `AuthenticationError` means a provider key is wrong. Fix it in `.env` and restart, since the proxy reads the environment only at startup. On uv, stop it and rerun step 7. On Docker, `docker rm -f litellm` and rerun step 7, since `docker restart` keeps the environment the container was created with.
|
||||
- `Cannot automatically baseline this database` means `DATABASE_URL` points at a database that already holds tables LiteLLM did not create. Point it at a new, empty database
|
||||
- `LLM Provider NOT provided` means a `model` in `config.yaml` lacks its provider in front. Use the name as the step 5 lookup prints it
|
||||
- `Invalid model name passed in model=...` means the request's `model` is not a `model_name` in `config.yaml`. `GET /v1/models` lists the ones the key can use
|
||||
- `No connected db.` on a proxy without Postgres means the bearer token is not the master key
|
||||
- `Conflict. The container name "/litellm" is already in use` means an earlier run left a container. `docker start litellm` reuses it, or with the user's OK `docker rm -f litellm` and rerun step 7
|
||||
- Inside Docker, `localhost` in `config.yaml` (an Ollama `api_base`, say) is the container itself. Use `host.docker.internal`, and on Linux also pass `--add-host=host.docker.internal:host-gateway`
|
||||
0
.gitguardian.yaml → .github/gitguardian.yaml
vendored
0
.gitguardian.yaml → .github/gitguardian.yaml
vendored
2
.github/workflows/test-linting.yml
vendored
2
.github/workflows/test-linting.yml
vendored
|
|
@ -258,7 +258,7 @@ jobs:
|
|||
if [ -n "$GITGUARDIAN_API_KEY" ]; then
|
||||
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
|
||||
retry git fetch --no-tags --unshallow origin
|
||||
uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo .
|
||||
uv tool run --from 'ggshield==1.48.0' ggshield --config-path .github/gitguardian.yaml secret scan repo .
|
||||
else
|
||||
echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -88,8 +88,6 @@ Do not put names of customers or customer company names in code, PR descriptions
|
|||
|
||||
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
|
||||
|
||||
For database operations, connection ownership and proxy dependency wiring, follow [database-access](.agents/skills/database-access/SKILL.md)
|
||||
|
||||
Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
|
||||
|
||||
Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ token = "test-api-key-with-high-entropy"
|
|||
|
||||
## Configured Ignore Patterns
|
||||
|
||||
These patterns are in `.gitguardian.yaml` for high-entropy test keys:
|
||||
These patterns are in `.github/gitguardian.yaml` for high-entropy test keys:
|
||||
- `sk-test-*` - OpenAI-style test keys
|
||||
- `sk-mock-*` - Mock API keys
|
||||
- `sk-fake-*` - Fake API keys
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
from collections.abc import Iterator
|
||||
from fnmatch import fnmatch
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
_REPO_ROOT: Final = Path(__file__).resolve().parents[2]
|
||||
_SKILLS_DIR: Final = _REPO_ROOT / ".agents" / "skills"
|
||||
_SKILL_DIRECTORIES: Final = tuple(sorted(skill_file.parent for skill_file in _SKILLS_DIR.glob("*/SKILL.md")))
|
||||
_FULL_DEPTH_MAX_DEPTH: Final = 5
|
||||
_FULL_DEPTH_SKIPPED_NAMES: Final = frozenset(("node_modules", ".git", "dist", "build", "__pycache__"))
|
||||
_GITIGNORED_NAME_PATTERNS: Final = tuple(
|
||||
line.strip().rstrip("/")
|
||||
for line in (_REPO_ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()
|
||||
if line.strip() and not line.startswith(("#", "!")) and "/" not in line.strip().rstrip("/")
|
||||
)
|
||||
|
||||
|
||||
def _absent_from_a_clone_or_skipped_by_the_cli(name: str) -> bool:
|
||||
return name in _FULL_DEPTH_SKIPPED_NAMES or any(fnmatch(name, pattern) for pattern in _GITIGNORED_NAME_PATTERNS)
|
||||
|
||||
|
||||
def _full_depth_skill_directories(directory: Path, depth: int) -> Iterator[Path]:
|
||||
if (directory / "SKILL.md").is_file():
|
||||
yield directory
|
||||
if depth == _FULL_DEPTH_MAX_DEPTH:
|
||||
return
|
||||
for child in sorted(directory.iterdir()):
|
||||
if child.is_dir() and not _absent_from_a_clone_or_skipped_by_the_cli(child.name):
|
||||
yield from _full_depth_skill_directories(child, depth + 1)
|
||||
|
||||
|
||||
_NON_END_USER_SKILL_DIRECTORIES: Final = tuple(
|
||||
directory for directory in _full_depth_skill_directories(_REPO_ROOT, 0) if directory.parent != _SKILLS_DIR
|
||||
)
|
||||
|
||||
|
||||
def _frontmatter(skill_directory: Path) -> object:
|
||||
opening, frontmatter, _body = (skill_directory / "SKILL.md").read_text(encoding="utf-8").split("---\n", 2)
|
||||
assert opening == "", f"{skill_directory.name}/SKILL.md must open with a --- frontmatter block"
|
||||
return yaml.safe_load(frontmatter)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("skill_directory", _SKILL_DIRECTORIES, ids=lambda path: path.name)
|
||||
def test_skill_frontmatter_has_string_name_matching_its_directory_and_a_description(skill_directory: Path) -> None:
|
||||
frontmatter: Final = _frontmatter(skill_directory)
|
||||
assert isinstance(frontmatter, dict), frontmatter
|
||||
assert frontmatter.get("name") == skill_directory.name, frontmatter
|
||||
description: Final = frontmatter.get("description")
|
||||
assert isinstance(description, str) and description.strip(), frontmatter
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"skill_directory",
|
||||
_NON_END_USER_SKILL_DIRECTORIES,
|
||||
ids=lambda path: str(path.relative_to(_REPO_ROOT)),
|
||||
)
|
||||
def test_skill_outside_skills_folder_is_hidden_from_end_user_skill_installs(skill_directory: Path) -> None:
|
||||
frontmatter: Final = _frontmatter(skill_directory)
|
||||
assert isinstance(frontmatter, dict), frontmatter
|
||||
metadata: Final = frontmatter.get("metadata")
|
||||
assert isinstance(metadata, dict) and metadata.get("internal") is True, (
|
||||
f"{skill_directory} needs `metadata: {{internal: true}}`, or `npx skills add BerriAI/litellm` installs it "
|
||||
"for end users, from a root dot-folder always and from anywhere with --full-depth (skills CLI 1.7.4, "
|
||||
"checked 2026-10-28)"
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue