mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-28 06:23:07 +00:00
d93353bcb6
Adds litellm-rust/loadtest: a mock LLM (OpenAI chat, Anthropic messages, Bedrock invoke with event-stream framing) that takes its timing from the prompt, an open-loop load generator with workloads covering 10k to 1M token contexts, 6 minute non-streaming calls and 30 minute streams, a config and virtual key generator, a summarizer and a /proc sampler, plus Terraform for an AWS rig with load generators, gateway and mocks on separate machines The gateway sets TCP_NODELAY on accepted sockets, removing a ~40 ms stall per streamed response, and its Docker build takes LTO and CODEGEN_UNITS build args for faster iteration builds
14 lines
638 B
Docker
14 lines
638 B
Docker
# One image with every load-test tool: mockllm, loadgen, genconfig, summarize, procmon.
|
|
# docker build -t litellm-loadtest litellm-rust/loadtest
|
|
# docker run litellm-loadtest mockllm -addr :9000
|
|
FROM golang:1.24.7-bookworm@sha256:b8bae5bd9ba9b1f89b635c91c24cc75cea335a16fb5076310f38566fc674b1ec AS build
|
|
WORKDIR /src
|
|
COPY go.mod ./
|
|
COPY cmd ./cmd
|
|
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/...
|
|
|
|
FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab
|
|
COPY --from=build /out/ /usr/local/bin/
|
|
COPY workloads /workloads
|
|
WORKDIR /work
|
|
ENTRYPOINT []
|