From e1a878b6d99c2a9dceebf207e2085d0b0de6483c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 7 May 2026 15:40:49 -0700 Subject: [PATCH] feat(managed agents): bundled-harness shortcut, build_platform, README - DockerfileConfig.path now optional; when omitted, resolves to bundled harnesses//Dockerfile next to the loader source. Lets yaml shrink to `dockerfiles: {opencode: {container_port: 4096}}`. - Add `build_platform` (default linux/amd64). Threaded through docker build + ECS task def runtimePlatform.cpuArchitecture so image arch always matches Fargate runtime. Lets users target Graviton. - Rename sample_harnesses/ -> harnesses/ to match prod intent. - Add README covering architecture, prereqs, config, env contract, endpoints, reconciler, gotchas. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../proxy/managed_agents_endpoints/README.md | 161 ++++++++++++++++++ .../managed_agents_endpoints/config_loader.py | 24 ++- .../managed_agents_endpoints/endpoints.py | 1 + .../managed_agents_endpoints/fargate/build.py | 21 ++- .../fargate/registry.py | 15 +- .../opencode/Dockerfile | 0 .../opencode/entrypoint.sh | 0 .../proxy/managed_agents_endpoints/types.py | 3 +- 8 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 litellm/proxy/managed_agents_endpoints/README.md rename litellm/proxy/managed_agents_endpoints/{sample_harnesses => harnesses}/opencode/Dockerfile (100%) rename litellm/proxy/managed_agents_endpoints/{sample_harnesses => harnesses}/opencode/entrypoint.sh (100%) diff --git a/litellm/proxy/managed_agents_endpoints/README.md b/litellm/proxy/managed_agents_endpoints/README.md new file mode 100644 index 00000000000..738bc8f42a5 --- /dev/null +++ b/litellm/proxy/managed_agents_endpoints/README.md @@ -0,0 +1,161 @@ +# Managed Agents + +Spin up sandboxed coding agents on AWS Fargate. Each agent is a containerized harness (e.g. `opencode`) that clones a git repo, talks to LiteLLM as its model provider, and exposes an HTTP API that the proxy proxies through. + +## Architecture + +``` +client ──► litellm proxy ──► /v1/managed_agents/* endpoints + │ + ├── Postgres (templates, agents, sessions) + └── AWS + ├── ECR (one repo per dockerfile_id) + ├── ECS (cluster: litellm-agents) + └── Fargate task per session + │ + └─► harness (e.g. opencode) on container_port + │ + └─► LiteLLM API (model calls) +``` + +Lifecycle: admin registers a **sandbox template** (dockerfile + repo). User creates an **agent** (template + model + prompt). User opens a **session** — proxy launches a Fargate task, waits for the harness HTTP server, then forwards messages to it. + +## Prerequisites + +- **Postgres** with the LiteLLM Prisma schema applied (the `LiteLLM_ManagedAgent*` tables). +- **AWS account** with permission to create ECR repos, ECS clusters, IAM roles, security groups, and run Fargate tasks. The proxy auto-bootstraps shared infra on first template build. +- **Default VPC** in your target region with at least one public subnet (`map-public-ip-on-launch=true`). Or pass overrides (see `aws.subnets` / `aws.security_group` below). +- **Docker** on the proxy host. The proxy shells out to `docker build` / `docker push` to publish harness images to ECR. +- **AWS CLI credentials** in the proxy's environment (env vars or `~/.aws/credentials`). The default boto3 chain is used. + +## Configuration + +Add a `managed_agents` block under `general_settings` in your proxy YAML: + +```yaml +general_settings: + master_key: sk-1234 + managed_agents: + enabled: true + aws_region: us-west-2 + dockerfiles: + # Bundled sample harness — `path` is omitted, resolved to + # litellm/proxy/managed_agents_endpoints/harnesses/opencode/Dockerfile + opencode: + container_port: 4096 + # Custom harness — `path` is required and resolved relative to the + # proxy's working directory. + # my-harness: + # path: ./managed_agents_endpoints/harnesses/my-harness/Dockerfile + # container_port: 4096 + aws: {} # leave empty to auto-discover default VPC + create resources + reconcile_interval_seconds: 60 +``` + +### `dockerfiles` + +Each entry registers a harness image the proxy can build and run. + +| Field | Required | Description | +|---|---|---| +| `path` | no | Path to the Dockerfile. Resolved relative to the proxy's working directory. The directory containing the Dockerfile is the build context. **Omit `path` to use a bundled harness** — the proxy will look for `harnesses//Dockerfile` next to its source files. | +| `container_port` | yes | Port the harness listens on inside the container. SG ingress is opened to `0.0.0.0/0:container_port`. | +| `build_platform` | no | Docker target platform. Default `linux/amd64`. Set `linux/arm64` to build + run on Graviton. The value is also mapped onto the ECS task def's `runtimePlatform.cpuArchitecture`, so image and Fargate task always agree on architecture. | + +### `aws` overrides + +Leave `aws: {}` to auto-discover and create everything. Override individual fields when you need to pin to existing infra: + +| Field | Description | +|---|---| +| `cluster` | ECS cluster name (default: `litellm-agents`). Created if missing. | +| `subnets` | List of subnet IDs. Must be in the same VPC and have public IPs. If unset, the proxy picks one public subnet from the default VPC. | +| `security_group` | SG ID with inbound `container_port` and outbound 443 + DNS. If unset, the proxy creates one. | +| `task_execution_role_arn` | IAM role for ECS to pull from ECR and write CloudWatch logs. If unset, the proxy creates `litellm-agents-task-exec`. | + +## Adding a harness + +A harness is any container that exposes the OpenCode-compatible HTTP API on `container_port`. Drop a folder under `harnesses//` containing a `Dockerfile` and any helper files (e.g. `entrypoint.sh`), then register it under `dockerfiles` in the YAML. + +When the proxy launches a Fargate task for a session, it injects the following env vars into the container as `containerOverrides.environment` on the ECS `RunTask` call. The values come from the per-session agent + template rows in the DB — you do NOT set them in the Dockerfile or yaml. **The harness only needs to read them at startup** (e.g. in `entrypoint.sh`). + +| Env | Set by proxy from | +|---|---| +| `REPO_URL` | Template `repo_url` | +| `BRANCH` | Agent `branch` (or template `default_branch`) | +| `LITELLM_API_KEY` | Agent `litellm_api_key` | +| `LITELLM_API_BASE` | Agent `litellm_api_base` | +| `LITELLM_DEFAULT_MODEL` | Agent `model` | +| `AGENT_PROMPT` | Agent `prompt` (optional) | +| `GIT_TOKEN` | Decrypted from template's `git_credential_id` (optional, only for `visibility=private`) | +| `PORT` | `container_port` | + +Concretely: a client calls `POST /v1/managed_agents/agents` with `litellm_api_key` + `litellm_api_base` + `model` in the body — those are stored on the agent row. Later, when that agent's session is opened, the proxy reads the row and writes the values into the container's environment block before `RunTask`. Each session gets its own, isolated set of env vars. + +The shipped `harnesses/opencode/` is the reference implementation. + +### Build platform + +The proxy passes `--platform ` to `docker build` and matches it to the ECS task def's `runtimePlatform.cpuArchitecture`. Default is `linux/amd64`. Set `build_platform: linux/arm64` per dockerfile entry to target Graviton. Build host architecture doesn't matter — Apple Silicon hosts can build amd64 images thanks to QEMU emulation. + +## End-to-end flow + +``` +# 1. List configured dockerfiles +curl -H "Authorization: Bearer $KEY" $PROXY/v1/managed_agents/dockerfiles + +# 2. Create a template (admin only — slow first time: docker build + ECR push + register task def) +curl -H "Authorization: Bearer $KEY" -H "content-type: application/json" \ + -X POST $PROXY/v1/managed_agents/sandbox-templates \ + -d '{"name":"my-template","dockerfile_id":"opencode","repo_url":"https://github.com/owner/repo","default_branch":"main","visibility":"public"}' + +# 3. Create an agent (any user) +curl -H "Authorization: Bearer $KEY" -H "content-type: application/json" \ + -X POST $PROXY/v1/managed_agents/agents \ + -d '{"name":"my-agent","model":"anthropic/claude-sonnet-4-6","prompt":"Concise coding agent.","tools":[],"litellm_api_key":"sk-...","litellm_api_base":"https://...","template_id":""}' + +# 4. Open a session (~50–120s cold boot) +curl -H "Authorization: Bearer $KEY" -H "content-type: application/json" \ + -X POST $PROXY/v1/managed_agents/agents/$AGENT_ID/session \ + -d '{"title":"smoke","initial_prompt":"What does this repo do?"}' + +# 5. Send a follow-up message +curl -H "Authorization: Bearer $KEY" -H "content-type: application/json" \ + -X POST $PROXY/v1/managed_agents/sessions/$SESSION_ID/message \ + -d '{"text":"List the top-level directories."}' + +# 6. Tear down +curl -H "Authorization: Bearer $KEY" -X DELETE $PROXY/v1/managed_agents/sessions/$SESSION_ID +``` + +## Endpoints + +| Method | Path | Auth | Description | +|---|---|---|---| +| GET | `/v1/managed_agents/dockerfiles` | any | List configured dockerfile entries. | +| POST | `/v1/managed_agents/sandbox-templates` | admin | Build + push image, register task def, persist template row. | +| GET | `/v1/managed_agents/sandbox-templates` | any | List templates. | +| GET | `/v1/managed_agents/sandbox-templates/{template_id}` | any | Fetch one template. | +| DELETE | `/v1/managed_agents/sandbox-templates/{template_id}` | admin | Delete template. Refused if any agents reference it. | +| POST | `/v1/managed_agents/agents` | any | Create an agent bound to a template. | +| GET | `/v1/managed_agents/agents/{agent_id}` | any | Fetch one agent. | +| POST | `/v1/managed_agents/agents/{agent_id}/session` | any | Launch a Fargate task, wait for ready, optionally send `initial_prompt`. | +| GET | `/v1/managed_agents/sessions/{session_id}` | any | Fetch session status. | +| GET | `/v1/managed_agents/sessions/{session_id}/events` | any | SSE stream of harness events. | +| POST | `/v1/managed_agents/sessions/{session_id}/message` | any | Send a follow-up `{"text": "..."}` (or `parts`). | +| DELETE | `/v1/managed_agents/sessions/{session_id}` | any | Stop the Fargate task and mark the session dead. | + +## Background reconciler + +When `enabled: true`, the proxy starts a background loop (`reconcile_interval_seconds`, default 60s) that: + +- Stops Fargate tasks whose session row was deleted in the DB. +- Marks sessions stuck in `creating` for too long as `failed`. +- Tags every task with `litellm_session_id` + `litellm_agent_id` so reconciliation is robust against orphans from crashed proxy processes. + +## Gotchas + +- **Don't double up the dockerfile path.** The path is resolved relative to the proxy's CWD. If the proxy is launched from `litellm/proxy/`, write `./managed_agents_endpoints/...`, not `./litellm/proxy/managed_agents_endpoints/...`. Or just omit `path` and rely on the bundled-harness lookup. +- **Session creation can take 1–3 minutes the first time.** Cold Fargate task + ECR pull + git clone + harness boot. The proxy waits up to 600s. If it consistently times out, check CloudWatch (`/ecs/litellm-agents` log group) for the offending task. +- **Default VPC required.** If your AWS account has no default VPC in the target region, set `aws.subnets` + `aws.security_group` explicitly. +- **AWS credentials.** boto3's default credential chain is used. Static IAM keys, SSO, instance profiles, all work as long as boto3 can resolve them when the proxy starts. diff --git a/litellm/proxy/managed_agents_endpoints/config_loader.py b/litellm/proxy/managed_agents_endpoints/config_loader.py index 7875e5cdbbe..901399f927f 100644 --- a/litellm/proxy/managed_agents_endpoints/config_loader.py +++ b/litellm/proxy/managed_agents_endpoints/config_loader.py @@ -23,6 +23,16 @@ from litellm.proxy.managed_agents_endpoints.types import ManagedAgentsConfig _DOCKERFILE_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$") +_BUILTIN_HARNESSES_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "harnesses" +) + + +def _builtin_dockerfile_path(dockerfile_id: str) -> Optional[str]: + """Return path to bundled harnesses//Dockerfile if it exists.""" + candidate = os.path.join(_BUILTIN_HARNESSES_DIR, dockerfile_id, "Dockerfile") + return candidate if os.path.isfile(candidate) else None + @dataclass(frozen=True) class DockerfileEntry: @@ -31,6 +41,7 @@ class DockerfileEntry: context_dir: str # absolute path to context dir (default: dockerfile dir) container_port: int content_hash: str # sha256 of dockerfile + context + build_platform: str # e.g. "linux/amd64", "linux/arm64" # Module-level state. Populated by ``initialize`` at proxy startup. @@ -91,7 +102,17 @@ def build_dockerfile_registry( for dockerfile_id, dockerfile_cfg in config.dockerfiles.items(): _validate_dockerfile_id(dockerfile_id) - abs_path = _resolve_path(dockerfile_cfg.path) + if dockerfile_cfg.path: + abs_path = _resolve_path(dockerfile_cfg.path) + else: + builtin = _builtin_dockerfile_path(dockerfile_id) + if builtin is None: + raise FileNotFoundError( + f"managed_agents: dockerfile id '{dockerfile_id}' has no " + f"path set and no bundled harnesses/{dockerfile_id}/" + f"Dockerfile exists" + ) + abs_path = builtin if not os.path.isfile(abs_path): raise FileNotFoundError( f"managed_agents: dockerfile path for id '{dockerfile_id}' " @@ -112,6 +133,7 @@ def build_dockerfile_registry( context_dir=context_dir, container_port=dockerfile_cfg.container_port, content_hash=content_hash, + build_platform=dockerfile_cfg.build_platform, ) registry[dockerfile_id] = entry diff --git a/litellm/proxy/managed_agents_endpoints/endpoints.py b/litellm/proxy/managed_agents_endpoints/endpoints.py index dac1f325519..4b8dce2cebb 100644 --- a/litellm/proxy/managed_agents_endpoints/endpoints.py +++ b/litellm/proxy/managed_agents_endpoints/endpoints.py @@ -149,6 +149,7 @@ async def create_sandbox_template( container_port=dockerfile_entry.container_port, region=region, aws_overrides=aws_overrides, + build_platform=dockerfile_entry.build_platform, ) except Exception as e: verbose_proxy_logger.exception( diff --git a/litellm/proxy/managed_agents_endpoints/fargate/build.py b/litellm/proxy/managed_agents_endpoints/fargate/build.py index 2a95f4aedbe..3ef18ed9885 100644 --- a/litellm/proxy/managed_agents_endpoints/fargate/build.py +++ b/litellm/proxy/managed_agents_endpoints/fargate/build.py @@ -47,6 +47,19 @@ def _sanitize_dockerfile_id(dockerfile_id: str) -> str: return _DOCKERFILE_ID_SANITIZE_RE.sub("-", dockerfile_id.lower()) +def _platform_to_cpu_arch(platform: str) -> str: + """Map docker `--platform` value to ECS task def `runtimePlatform.cpuArchitecture`.""" + p = platform.lower().strip() + if p in ("linux/amd64", "amd64", "linux/x86_64", "x86_64"): + return "X86_64" + if p in ("linux/arm64", "arm64", "linux/aarch64", "aarch64"): + return "ARM64" + raise ValueError( + f"unsupported build_platform '{platform}' for Fargate " + f"(expected linux/amd64 or linux/arm64)" + ) + + def _register_task_definition( *, region: str, @@ -54,6 +67,7 @@ def _register_task_definition( image_uri: str, container_port: int, shared_infra: SharedInfra, + cpu_architecture: str, ) -> str: ecs = _ecs_client(region) r = ecs.register_task_definition( @@ -64,7 +78,7 @@ def _register_task_definition( memory="1024", executionRoleArn=shared_infra.task_exec_role_arn, runtimePlatform={ - "cpuArchitecture": "X86_64", + "cpuArchitecture": cpu_architecture, "operatingSystemFamily": "LINUX", }, containerDefinitions=[ @@ -95,6 +109,7 @@ async def provision_template( container_port: int, region: str, aws_overrides: AwsOverrides, + build_platform: str = "linux/amd64", log_callback: Optional[Callable[[str], None]] = None, ) -> ProvisionedTemplate: """Bootstrap shared infra → build/push image → register task def. Idempotent.""" @@ -129,6 +144,8 @@ async def provision_template( f"provision_template build start dockerfile_id={dockerfile_id} " f"hash={image_hash[:12]} repo={repo_name}" ) + cpu_arch = _platform_to_cpu_arch(build_platform) + image_uri = await asyncio.to_thread( build_and_push, region=region, @@ -136,6 +153,7 @@ async def provision_template( dockerfile_path=dockerfile_path, context_dir=ctx_dir, content_hash=image_hash, + platform=build_platform, log_callback=log_callback, ) @@ -146,6 +164,7 @@ async def provision_template( image_uri=image_uri, container_port=container_port, shared_infra=shared_infra, + cpu_architecture=cpu_arch, ) verbose_proxy_logger.info( f"provision_template register-task-def complete family={family} " diff --git a/litellm/proxy/managed_agents_endpoints/fargate/registry.py b/litellm/proxy/managed_agents_endpoints/fargate/registry.py index 95104a28d02..a84c2f6ce4f 100644 --- a/litellm/proxy/managed_agents_endpoints/fargate/registry.py +++ b/litellm/proxy/managed_agents_endpoints/fargate/registry.py @@ -183,6 +183,7 @@ def docker_build( context_dir: str, image_uri: str, *, + platform: str = "linux/amd64", log_callback: Optional[Callable[[str], None]] = None, ) -> None: if not os.path.isfile(dockerfile_path): @@ -199,7 +200,7 @@ def docker_build( "docker", "build", "--platform", - "linux/amd64", + platform, "-f", dockerfile_path, "-t", @@ -235,6 +236,7 @@ def build_and_push( dockerfile_path: str, context_dir: str, content_hash: str, + platform: str = "linux/amd64", log_callback: Optional[Callable[[str], None]] = None, ) -> str: if not os.path.isfile(dockerfile_path): @@ -258,10 +260,17 @@ def build_and_push( return image_uri verbose_proxy_logger.info( - f"Building image {repo_name}:{tag} from {dockerfile_path} (context={ctx})" + f"Building image {repo_name}:{tag} from {dockerfile_path} " + f"(context={ctx}, platform={platform})" ) docker_login(region) - docker_build(dockerfile_path, ctx, image_uri, log_callback=log_callback) + docker_build( + dockerfile_path, + ctx, + image_uri, + platform=platform, + log_callback=log_callback, + ) docker_push(image_uri, log_callback=log_callback) verbose_proxy_logger.info(f"Pushed image {image_uri}") return image_uri diff --git a/litellm/proxy/managed_agents_endpoints/sample_harnesses/opencode/Dockerfile b/litellm/proxy/managed_agents_endpoints/harnesses/opencode/Dockerfile similarity index 100% rename from litellm/proxy/managed_agents_endpoints/sample_harnesses/opencode/Dockerfile rename to litellm/proxy/managed_agents_endpoints/harnesses/opencode/Dockerfile diff --git a/litellm/proxy/managed_agents_endpoints/sample_harnesses/opencode/entrypoint.sh b/litellm/proxy/managed_agents_endpoints/harnesses/opencode/entrypoint.sh similarity index 100% rename from litellm/proxy/managed_agents_endpoints/sample_harnesses/opencode/entrypoint.sh rename to litellm/proxy/managed_agents_endpoints/harnesses/opencode/entrypoint.sh diff --git a/litellm/proxy/managed_agents_endpoints/types.py b/litellm/proxy/managed_agents_endpoints/types.py index 46b2027c4b7..ce14d29cc1d 100644 --- a/litellm/proxy/managed_agents_endpoints/types.py +++ b/litellm/proxy/managed_agents_endpoints/types.py @@ -8,8 +8,9 @@ from pydantic import BaseModel, ConfigDict, Field class DockerfileConfig(BaseModel): model_config = ConfigDict(extra="forbid") - path: str + path: Optional[str] = None container_port: int = 4096 + build_platform: str = "linux/amd64" class AwsOverrides(BaseModel):