From ccd8c4112c2092d680fcfc60a2b20da1b722d5e9 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 17:38:48 -0700 Subject: [PATCH 01/29] docs(readme): add Deploy on AWS/GCP with Terraform section Adds a quickstart for the two published Terraform modules on the public registry (BerriAI/litellm/aws and BerriAI/litellm/google). Copy-paste main.tf for each cloud, the one-time GCP Artifact Registry remote-repo command, and pointers to the registry pages for the full input surface. Sits inside the Get Started section, between the gateway/SDK table and Run in Developer Mode -- where someone scanning the README for "how do I deploy this" will land. Co-Authored-By: Claude Opus 4.7 --- README.md | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/README.md b/README.md index d600f3952c6..faf79c3729d 100644 --- a/README.md +++ b/README.md @@ -404,6 +404,124 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+). +### Deploy on AWS or GCP with Terraform + +Run the LiteLLM proxy as a production-ready componentized stack (gateway, backend, UI on separate services; managed Postgres + Redis + object store) using the published Terraform modules. Both modules are on the [public Terraform Registry](https://registry.terraform.io/namespaces/BerriAI) — no auth needed. + +#### AWS — ECS Fargate + Aurora + ElastiCache + ALB + +[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest) + +```hcl +# main.tf +terraform { + required_version = ">= 1.6.0" + required_providers { + aws = { source = "hashicorp/aws", version = "~> 5.60" } + } +} + +provider "aws" { + region = "us-west-2" +} + +module "litellm" { + source = "BerriAI/litellm/aws" + version = "~> 1.89" + + region = "us-west-2" + azs = ["us-west-2a", "us-west-2b"] + tenant = "acme" + env = "prod" + + # Production: provide an ACM cert. Without one, set allow_plaintext_alb = true + # (dev/trial only). + # acm_certificate_arn = "arn:aws:acm:us-west-2:111122223333:certificate/..." + allow_plaintext_alb = true +} + +output "litellm_url" { + value = module.litellm.alb_dns_name +} +``` + +```bash +terraform init +terraform apply +``` + +Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest?tab=inputs). + +#### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB + +[Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest) + +Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR: + +```bash +gcloud artifacts repositories create litellm \ + --location=us-central1 \ + --repository-format=docker \ + --mode=remote-repository \ + --remote-docker-repo=https://ghcr.io \ + --project=my-gcp-project +``` + +Then: + +```hcl +# main.tf +terraform { + required_version = ">= 1.6.0" + required_providers { + google = { source = "hashicorp/google", version = "~> 6.10" } + google-beta = { source = "hashicorp/google-beta", version = "~> 6.10" } + } +} + +provider "google" { project = "my-gcp-project"; region = "us-central1" } +provider "google-beta" { project = "my-gcp-project"; region = "us-central1" } + +module "litellm" { + source = "BerriAI/litellm/google" + version = "~> 1.89" + + project_id = "my-gcp-project" + region = "us-central1" + tenant = "acme" + env = "prod" + + image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai" + + # Production: provide DNS already pointing at the LB IP for Google-managed certs. + # Without one, set allow_plaintext_lb = true (dev/trial only). + # lb_domains = ["proxy.example.com"] + allow_plaintext_lb = true +} + +output "litellm_url" { + value = module.litellm.load_balancer_url +} +``` + +```bash +terraform init +terraform apply +``` + +Provider API keys live in Secret Manager; reference resource IDs (e.g. `projects/my-gcp-project/secrets/openai-api-key`) via `gateway_extra_secrets`. Full input list and architecture diagram on the [registry page](https://registry.terraform.io/modules/BerriAI/litellm/google/latest?tab=inputs). + +#### Both stacks include + +- The full componentized split (gateway / backend / UI as independent services) +- Managed Postgres (writer + reader) and Redis +- Versioned object store for proxy state + file uploads +- An auto-generated `LITELLM_MASTER_KEY` in your cloud's secret manager +- A one-off migration job that runs `prisma migrate deploy` before the proxy starts +- The same `proxy_config` surface as the [Helm chart](./helm/litellm/) — pass YAML as a typed map + +The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws/) and [`terraform/litellm/gcp/`](./terraform/litellm/gcp/) in this repo; the registry entries are read-only mirrors updated on each release. + ### Run in Developer Mode #### Services 1. Setup .env file in root From 260f5f1371e3308810cdaab61daa4da231cc1e76 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 17:44:49 -0700 Subject: [PATCH 02/29] docs(readme): add 1-click deploy buttons for AWS + GCP GCP gets the real 1-click: Open in Cloud Shell badge that clones the repo and walks through `terraform apply` via the existing DeployStack tutorial (already shipped at terraform/litellm/gcp/examples/default/ TUTORIAL.md). User just picks a project. AWS gets a soft 1-click: a Launch in AWS CloudShell badge that opens an in-browser, already-authenticated shell. User runs four commands (clone + cd + cp tfvars + terraform apply) once inside. There's no native AWS deeplink that pre-clones a repo + runs a tutorial -- CFN "Launch Stack" + CodeBuild would be needed for that, and that's a separate piece of work. Co-Authored-By: Claude Opus 4.7 --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index faf79c3729d..d304d15f169 100644 --- a/README.md +++ b/README.md @@ -410,8 +410,19 @@ Run the LiteLLM proxy as a production-ready componentized stack (gateway, backen #### AWS — ECS Fargate + Aurora + ElastiCache + ALB +[![Launch in AWS CloudShell](https://img.shields.io/badge/Launch-AWS_CloudShell-FF9900?logo=amazon-aws&logoColor=white)](https://console.aws.amazon.com/cloudshell/home) — opens an in-browser shell, already authenticated to your AWS account. Once inside, run: + +```bash +git clone https://github.com/BerriAI/litellm.git +cd litellm/terraform/litellm/aws/examples/default +cp terraform.tfvars.example terraform.tfvars # edit region/tenant/env +terraform init && terraform apply +``` + [Module page →](https://registry.terraform.io/modules/BerriAI/litellm/aws/latest) +Or call the module from your own root config: + ```hcl # main.tf terraform { @@ -454,9 +465,13 @@ Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra #### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB +[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true) + +Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply. + [Module page →](https://registry.terraform.io/modules/BerriAI/litellm/google/latest) -Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR: +To call the module from your own config instead, Cloud Run can't pull from `ghcr.io` directly, so first set up a one-time Artifact Registry remote repo backed by GHCR: ```bash gcloud artifacts repositories create litellm \ From a505734a2cddcece2d27ae7dea2e192f0f8b7a13 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 17:50:53 -0700 Subject: [PATCH 03/29] docs(readme): move AWS + GCP deploy buttons next to Render button --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d304d15f169..7802e33baee 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Deploy on Railway + Launch in AWS CloudShell + Open in Cloud Shell

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From d0ff1753973ee16b2c9ef88d34d2647e3d7cbec4 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 17:54:38 -0700 Subject: [PATCH 04/29] docs(readme): unify deploy button sizes and badge styles --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7802e33baee..2509b761563 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

- Deploy to Render + Deploy to Render - Deploy on Railway + Deploy on Railway - Launch in AWS CloudShell - Open in Cloud Shell + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From daba177a5205f1be70a658a5dd063b23fd67e21f Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:15:36 -0700 Subject: [PATCH 05/29] docs(readme): bump deploy button height to 48 to match Render/Railway --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2509b761563..693aee14683 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

- Deploy to Render + Deploy to Render - Deploy on Railway + Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 1a18184f0eae4b331946ae564fd51dc7dbf1a0a2 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:25:49 -0700 Subject: [PATCH 06/29] docs(readme): bump AWS/GCP badge height to compensate for SVG padding --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 693aee14683..d97f07a1b8b 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 8ba452c0ecfaf6b81268eb596ba2fb0d463bee06 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:26:21 -0700 Subject: [PATCH 07/29] docs(readme): bump AWS/GCP badge height to 72 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d97f07a1b8b..91246629ad6 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 2c56b4d3f64ae941322b78b35df4bf0dfea51f34 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:27:10 -0700 Subject: [PATCH 08/29] docs(readme): bump AWS/GCP badge height to 84 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 91246629ad6..07a74296fd1 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 65890b732e1e67af0d3c09a318197534d8966ced Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 01:33:43 +0000 Subject: [PATCH 09/29] fix(readme): make deploy buttons same height (48px) https://claude.ai/code/session_01MxQRMHSDXbqJh74rF86UBc --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 07a74296fd1..693aee14683 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

From 97f067bc70d7006f7aeb8dc117412c4d0f67e514 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 6 Jun 2026 18:37:28 -0700 Subject: [PATCH 10/29] docs(readme): flag GCP project ID substitution in image_registry --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 693aee14683..529e1de3354 100644 --- a/README.md +++ b/README.md @@ -508,6 +508,7 @@ module "litellm" { tenant = "acme" env = "prod" + # Replace my-gcp-project with your GCP project ID (same value as project_id above). image_registry = "us-central1-docker.pkg.dev/my-gcp-project/litellm/berriai" # Production: provide DNS already pointing at the LB IP for Google-managed certs. From 99e1a7612860fe469af44a5ef08434867359fd41 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 02:10:22 +0000 Subject: [PATCH 11/29] docs(readme): equalize deploy button heights and fix Cloud Shell button font GitHub rewrites an image's height attribute to "height: auto; max-height: Npx", which only caps and never stretches, so each image renders at its intrinsic height. The AWS/GCP shields badges are intrinsically 28px while the Render/Railway buttons are 40px, leaving the row uneven regardless of the height="48" we set. Replace the two shields badges with committed 40px PNGs so all four header buttons render at the same 40px. Also swap the Cloud Shell button from open-btn.svg to open-btn.png. The SVG renders its label as live text with font-family "Roboto, Sans" and no generic fallback; since neither font exists in GitHub's render environment, the text fell back to a serif (Times New Roman). The PNG bakes in the correct typeface. --- .github/deploy-on-aws.png | Bin 0 -> 4113 bytes .github/deploy-on-gcp.png | Bin 0 -> 4922 bytes README.md | 10 +++++----- 3 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .github/deploy-on-aws.png create mode 100644 .github/deploy-on-gcp.png diff --git a/.github/deploy-on-aws.png b/.github/deploy-on-aws.png new file mode 100644 index 0000000000000000000000000000000000000000..06d41f2a5e06cd60aca118d53ea50f0beb234ce8 GIT binary patch literal 4113 zcmc)N)msyQy9aR4fiwf8hor>lPDgi1cS?-zZV)D&0wN;yqoh+A-5^Y20|!!~B9msL zk>h#JA8_u@xp|-O#rNX9df(@h`rJU1oP>!44-b!ATT9It4-Y^7Uw%&X;NN_|FByb~ zM>(ObrfeEsy8kY$fEL9vIS3bM#rAf(dhqKUl{=gEzT{xz0c&JDbyg^RA|A z>#0{c@dF@A;t4P<;5Jr^d|kPC(;i&d-MJopMA7#qYIAn2XMZj1J#2HgPtolV?pOTo z(^>yuD6HVNhm9!FG1-MEkyM2d7^DW&lgEk0P)T22RXMD6j5YVQ!!0YJT^zXB;tdTZ8* zD4<49vO2_+ndh7%j55ppVMME1G(lY;cVu6S^v@lvQ7K}$9#pFE$^_=`oE9UcrOv$O^Mlrb^^4B4RF*)+t?cj`jDyI!nJnnat_Z7h1kWFl&f}(kNUp~O8dRgU+ zO6w@AjOdeH3&L z5)BbCU{O2ad!451r>dcd#AKY}I$Hvz*rFg(*^`B-Ml20S?wFErtCL6GZyZB)k1`R$ zz3{#mabK0$y>p5NsU>RzmI?Y8k}ETK?L4#<4v!Hlgt(u3MDklhwhmA2qeGj}wR*JJ zw>4dQhp_w`Dz&=-=!Hx}kA`Q-GNLWm8^*K-GVuCa^Y^6;wF5rbjH0Vo4V7ekyk*H;9%y#v?mkN4Cx8rVD|c(VAcX&=xbF~r_}VEbT?{f{9-Z9SBJKb zv3?fslP_|o5ocv9O?b!6SzoXgc(7KCT&i`i&7HceUv9tL`&wss_h&waYzB3DU$_iZ z)Q5d|(S|EM!am+y#E ziJY)RwJy}c<;#*aCS>K>f0e?R6>RW6dtKCfsLixK_u2(J-Ywi3C;MHiav~lqt-P`_ ze-Tt5lu%&;nn7vQ?ut4R=IL0%(b2+4#NOp!2WwLO4!1!GI0~9A zl?@e)DssF!GKyxh&xX)EKuwbBrzUh5H9buuNsiZWzV!$#%D&F2Q560GlXr*0QP$9u zTGdN{x%BJE_nw z<*G&HB>dR*ZwSoPnxQp4Hr$Y)i3|a<@b%d6=y*5b zy{TOTqa|l4GZJzs{q%$G{+#fR^o`_R$dmQML++Dz76-TYJpxq(JzH&h1C=e?3g_G4pCIt8O}h2Ia#1?O>Q9RYVymRCHWOLHF+ z2!rt+HqniBVHp}Rc)CrF0FCa$7vdV^?P{0DP=1c9)?oW{>1 z=Jab1i1dm2^2(HA)h}8Dkz3u*DHh{8!o6Dn)5wJhvL* zXgCqA__uXA3#T2Ku%hPYGF>h6p&s!;bl7$;y6Elca5R>xP$6A)Goqv}K4w1r6-n&` z%i04q+=&LOY)EGxr4Osgr#xV=DARhp;7*l-OXv11blSvTRI#+D<+5XKc8Fwh4k1hx zbSc1iCaXEyo_i>?^PJyd;?PfIi(fy9f%25)VutFb=U0RzQzEyb$%|YeLpVZDdE6&Qq(~)dDT*TK$Ydjc2>eR=ioswRaII`1{XT__;Vfx22 z>1v967<=)3njZ-nG1J3U8FSarzWG&&Kv-(Lv@bP3CPdss%Ij}4 zh|Xc)F8u}|hol6uQ8cz3p{##Tl9&0Fobdl;sN&LUXT*Iw<)diA9l@Ohvb>kAM;fp>Jf-ET;a zb!Y_-ffGN9Sb%sc$|`8xqwIo{w<3lnm4p>NhNw^!3x?&uPxHr390O30qG;Wzd*3FZ zcr-kJ_T8OP$fERVjEhfSiqYAhpD^)RelKa50Ie&4E&L*Cu6Y)${HdSX$|OVQm@ zkVv1AwArVHGGOfq*;)o<=k*|KSyDP@Dp={!ptudlP{!9k%}e6rmN>xYiRvI9#Dgy@ zyLePDXZWO1>n?~A3eVM#zoOi^6uNII&QE)NrVg%tD`-#owA8jg{%Kbvl2hD}Mt|ZH zSW-FFxQ!hXnIT`3BlQ@$d{|tw&X-chSeF&=)bY}Jw(FT2g{rzl?1d1d$L3S2PEM}1 zMFV&YXv&+im}rS($~rpts1=@Qi}}gVP?r4ff~G2Q1`zNn5L$IJeJAEggV2ybb^^vf zLy|o1;4{ZrvKznQqtmLn*Ub3i$M2&m(26|eF5GSi3;c{JhSIE$KfYtBVifsAOmhod z&TXZwX_Mtegx2^pG;D!L89yhU4q(OU;9O+cn3f_VgA<#zgn_G-R_brtBVOL%a~(5{ zgjzR1{O|gok7^gqn6N@|y9R8+auKxV+h&YtYme$Cg~heQ$#(brP2* zXA5UCIdRerZU~U7OLvTKO7phMfV~nKaS^j$^RrU_(F0R8w4^1r06w?>*2oNKL$>L; zbpOrA56nVHtY!yF8W{uHBl%MQ2YbF;5;p3A>k8u?E!jPi z>)x07MgnDyh3i^IO_7olx;CRM&1txQ`+9o*UMW?G|6S3r_$WYSMZ!>SFdQM6COfNY zcK$KkJ@ZFP3T6sZp7Px6;`!i@vB(Yv8C@@JzLo>AR%$LKuBZWSB|Yy5-6$0! zPl~~q&j|H*vZ5<77(x+%Bb0jZoMj5B@tFHT!kWCC}*2>>Kr z2cZ$xyxk6-EUpY=Tm0ofo%79u>KEZ*NwV5ja8}7iKWPbXRcTbFH5Y7_d9Ii$sYU(i z4fE?m;@7*KkW^GpZ4!@}c48z2am)h=4RM zEU{HAKZY=*!h_$QX*P*6bSv9z+SIdu*ZMvlU$R+6$IsXp^N^kxq-TTeLUvyDyZW&+ z`1uA_EsqH3WqcS)AC8RiVR{cTd5l=v_l;596rg{VH~!_LnbFg>+h=Dr9IS;>j+cv@ zK)9QStF4}Q@~IaB+ysO_Gbj{gAn7vysyH&@QF5jk98=xKs!6RH4>fkfnxr=o&W)n{ zO6oxyWD{S6o)l(E#8j5;6bMCW|4C#|pPSv9DI%p*nIzYk$(T2LtBIV$HH%f;PXYY< zeXV=uXdbMXX{1O65+}!kNCXe~FfHL7=X-A^bCx{EUWXx+V$0Bz39+^=zg^b*C=T8) z&%9C*X>bi^zS+TeehnSI*5j>ySkT~XxrP5)nYo>+Iufh+q^FCH)d8F8MlB7n5dmWTv1S`V~^p^yps`DjgqhQJ2^8ej?xMn9{kjmn9UKJAcqM7? zTeOSSiiG4unxV!g7I%>!z2_age)xMl|1zjZ6D=B%pyBQ0U*Ky*AH&Yv&rqU}g4!8^ z{wakAG%`%jaK1ghTf;QriNdUMUA~M4f;Ib7MWOcIcRA(@h|(81cmb=&&(@AVirF>! z^~WYzrh94E_wf94 z7riyUDOdRF8LM)%HXeQ@PveH+fPu&D1SRr^q7TtTwbMBZCKF9BPWpaqlEpM|Qo3E$ zXB7@`*r>K|V~Y5cVkea~`lzU$*WOp@DLrd<^56tau(h_;UqhmF$~>LYnY4;$;scX( zMmjQXok8%#uu52bi(Iphdz)~erSY=Y8f|3T#@JK$CW9uz_xMkxEG%Y&c?rgG$n|rO zfPGuroOm5vR3~XZ?TU&NssXqE8au*Cw^ds6|9opY`+PeHH%RGi0nyTKgi=L2Rc0=X z>gXeNiWX^Kqo&pHs}NN7t_~yr1JtW*VuJrKmj3TL-nswh`TKh^iR+Sof$+aY{}2yI XH7&3dhx>acd4x^h`m+psv0$-R&i6K zu}5sGXdd^U@Vt1QH|ILnS=Twg>-XY&eG-fewCL{KyF)@kLZ_px4k95TZMgA`s3~u3 zX#4?ygoKeqM_t7fp0}HSJDp{^v!D1=VN&6Fq>M796DI}B3qHPk>_-0-GFiVZBG1H> zX`vtfv6W2Ncw~rA639~5M;hNnzEmH7NDv~Ae#-%oZomSpAXH9qb8VP*-|l;YCw@y6 zHH592zKigUsg*qY(#k(SsxPl+tRD3Wa%tZI#4!GP?!5FvF)|`rl^PK9o8u1WP0UUW zz<-nR5Q4-os)o|VA{kZVYd@lxRH+YNut?k#b|F+XsR3VSj0XQ}y7zzCRH_cdt;A8T z{w@G)&B+Q)Z*R`Y@@e_q)(z8x1nPqn=Rc$~NcTC7=V-pF7r)tE^6=MKHW_< zx>wFd^Uk={~e5?&eTCKVSyGe7%Ks0f${$W_-y6+C-U$!$NH7Z!lO_?dCmJ zX^<|!2I$GtG~VPyZGZ}v2{3fqbM7v7h? zr<%setjtq^vZlv2pq{|F3Z64jcYt-f0)TjBVeO+5n9^Mv|4n$GO^i2q+ZezZ;F)l; z!z9u1$1i|OduT`$%aAPLtAYO75E|Q~g;@Q!HuiaVm-hAbJKt*PYS5D>Fv91Nkh7t7 z@zc(vTr#l}nPT7k#PmFGb>}k3Vufaqb62>vTzPu$Sbuu+?^o9P!^CWBK?73b?t@|w zzKbUetWEzIdO^JHuh2d<>uGWv+K0e)G>VB0ORvhvd|)>1O&Od*bv>G}$ggzN7$y$L zk*HYJi;F=cxH6q~8b1tpE||C(nRv|eP&=tg5%em?yhL(j*zeB7?!M%{?Xj=okpa`-cpOD?=xedvYS!#OV_8@J_L71sJ5~K#vtAb(X}waa zjbYP)Tj}_~3%G}BSZY>>({aHjuHG-q{sF$u50Kv31-TBw+F{P=IH}gBp4z(Av8)Cv zB@9oyGi}bG*EY@+fSw{gyk0+Q8HHZoi*QzdG9l@GOeh}tTL|~Xg7GN&Cn*2pdJD5z zMonrbIi_Y%2~q!x3T8Do`Xyvx%YHbB{&1_P;04z++*4lAlgo6R?U|9j!PpK8ttRoe zD&DVFQi)x?*>I*1hBW`x22wDT{`<{xYP4S=KWdsPVx#feKFCc+&o)?4z?fm?dusuI z`{8c9pV08%kVYlX^f+hDwf1di8^wBav9LOGTz9ReS2jqn_TRdp?g+21aE=aDMKXp; z&jrqsqh+(T{vy0fiY!nl5Siktz$Vz^a44T)IQj{Wi8dOu8B&vJO>Lf7dR}eojuwqT zP%vodI0qh3Z8#@#uG#>1Qm}10#6KeT?)r{t?blx~=GvlCTgDzqa!u55dNosKc>XB)(5-sl7MdV&3e39Ai{z||i`Ra{ zWspV(0(|r6+7{*KXSJ|6tH)R}jJ$mTYdxk2G2gtG5qPmO>3{Tnnib)?h20dkw_41# z`n;2ePMI(=nGE#LjO3aTEk5l+=Jm7;t;wg(IMZ2e3MjB)^;J~U(b6kOulrjsqR6`Cn$4xS|$Si>53GsfhqfI zclnyjIM(-DUoJs4SLai6;`|HjxGsSkQ%6w1FKD%Ax}l=SJr!`bH0@Yk2N=tf&7I+} zG9@z$C{|>a9&#p+a z8mJ3_sF3G^>EbaRR>zI>c+!^*B{3BAy-Z>weWALT~`xn++pU-e(kQMg7on1@%y>@NS@?QhH+We=I6~1APR`moJfK4^Y+K$}TJR2AB_(;LjkNiX#mf zxpWE%U>&+mZa+;$61STvd%dD^tsGlvaxD3b9HrO2l_2+qgN{dH{pf0X9iPC6@0~H{ zpfP&R_}gkM^-8b>HW?q^WSH9~b6!jQfwD+T<<8@wl2<+nvFL;R#1PClmlct6#!p4M zv&Z=+Gq-2ZqV2}tS)>c9b)f1QGO=%h44pNvCyyNv@Zt#abux9=zWy?xf>H(Y#? ztMKVQuxItlc`H7s(?#x2aUGUBe%#=}yQB$Dron47c`gGvt=I$A92(;A+pq%_J5kCU1$nt{qdb*9u}g9#rd`3FCWq>BlbJ8rrmg&gxgv4TwQY9o;WnA? z<@IGc0*V|=GuuC0*rLnhOQK;pK;?3dpS9@M0KCAMF|m;= zTWtXb!rJJ!lHT2r%$PjZhN$Um$4`zC5*}`sf4FsCl^x2Yv>ZmFbxkbF;~9VGySNx= zr^ARabC{Z^7c!b)fWG(KY{bBLJZs9x^ZQDMZf1)HaiRFZLMkBa)KLmn8nR;6xFNYZ z{$`!KyUJ=T38G(MNxDzl$z@&zs;M-Dj{#Fv#R?AB$Xbys8gj;{VW#Sra;iT9N?!IHQfIf~H^*WBnpVIeM zR4?ic%jEFOThZYT!mE{>TIpQqe|Wki`ONW~sj`_LC`>NY2jBV>0Y^0pB!z99;ewwB zY*w%}e*9pSbMOhJq~aJLt^eu}k&jE{s9F9VkUbM6;3vOZIZ5O>(w_^k26v;&wfU?l zW|0r#SfXQC$bg69w z+OMdP>~`qxY^1Fm_n^Lhj#((#a5L=L1vLFVx=i>qnSC%r)Dpiy;3k==rTI9WMSZ37 zgjUw4@WNWltt$iNFEqx5a)AG87dEFsAVOQ5iubW@_@1~X+VTEP;0A!)zQ!`tl3a;6`$6_u?hZ|dqOXr!NLe!CW5!B z0QSzSrvI`O=`u55`%BN3ce{g|RptYyxIk4fKBMI`nN4`8!PKlkIj2YyKNo@c6}zcf zI^_H~Om6P4?)g=~7HX+Ak>X!E0o;l6mV&n@0wO%2yrjL&270$4wfEPXn6aj*M5eym zs&O_-`Y;vt`j9v-bX=;5{VtR>yHPcs$sRVPRxMm2O>-#u4$AvVTv|Yb=UDcU!)Ctk zJtS4m*6`#l@&L6aGwzeKr2?)mD%9ddhIgkE%lZ89rZJysZiGn3xZ8fe=@}_)O_f=D?j*jXKWQ%u%O-m=X7#`!O{BmwZJ`2j{iY z+QpK5PepKJm{{Sfr1H}VV*eeJm&{>$Li6Wx{QA1N^|xzT`w0A#`P@*#Vk>cZ3c6R? z6YD%#6En{IPIswStsm?p?6+C_I774i6v5rP#&9=>S=Ppx5DU}ZC~racTSM9IvZNxp zHa%C#gXT{<*!yqO;owowC+7yxXY`Q>;s+$ITAM<1)FQY=()HQj&HY@n&z&#)`*;ot zq)S)JTHU-03X7+oSj)U|+krmxqW=S{;i!nf3$L$HRh@6v#{dy-$ZUDydXt*xF_La z1?7Yx!|C?9Vh~sEFO=jAY%*UzL2Ru1T^9{VYnxE2rNe0-Eq||LLK#zr-tO3t8f5s^ zEINl5q_8d~YGI-a-hV)fO$M7o%lZ)eaYNiEI$OXLU&>ccvj$JM3B5b}A(Au&6w>`Y zxB%Sv`4=vV+GbaCV+{6n}1QUMgx87pb4UVN*LW{CbAmM&`b}$U=lh z5F#^$2Wy2(80^cbj{MGb)#AhxJ`hV`%S{y@Z&qSguM+-yx+HsJpz&hRpS9mhB-Xlh z|Ru&Acapd_PSpRn=xr`RHQh(AHtn6=8 zU@U8PvRD1{J&KTjt3QdwzrAN!aY3nfNvz&tV5RSoc<=Errl*aOPcj9zg`VRJy2`e9 z)Ov4d8t#!VLAhO|^O%7OIT~@i@+PAe!4%oTTMVjDB!uwu_y4eyH}&$$%o&@NtF&_Z z4(|pZm(tIl{b})OWt3S@+iRG0!&lh%?JdX6D>T(!kk2;^P|c07pbGmIlU@SbF6!+* z=c%i8)JRj(P@k!M&COZi-eH4|xUxLyY#FyZ7UZr?AyX~z(&SoZHOR7rni94m0})_y z^Gyj0{BKEsYuVDFKeC|s6Xq>ZZoRd-bIwD|^mQRuE95L~zLH^e7j#9EC>nEV-K*EuA= zUIQdnXoIg|-|=<7eJEke9XIn{x~+{y@qxySlDS8d?kX8F+G%I+S~L;TrBs58k6g_m zl)Z7C=gD^bsNolD_u4p!JPDsVV@YEm{yF-WW^?vjQ#9C~iuB}s*8*S&juvvU#r)Gl zVk@O3YAy9^3_{zA_MXA+~u-6(lve$4kwGi^MFX1IT5p`ZDNJLbrH{#*dkc$BBK zcrOq_#qsp{<)_lGHE2~te!XBS6caXa$(*aUO0-SX{E)eq{$MVz

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

- Deploy to Render + Deploy to Render - Deploy on Railway + Deploy on Railway - Deploy on AWS - Deploy on GCP + Deploy on AWS + Deploy on GCP

LiteLLM Proxy Server (AI Gateway) | Hosted Proxy | Enterprise Tier | Website

@@ -467,7 +467,7 @@ Provider API keys live in AWS Secrets Manager; reference ARNs via `gateway_extra #### GCP — Cloud Run + Cloud SQL + Memorystore + HTTPS LB -[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true) +[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.png)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FBerriAI%2Flitellm&cloudshell_workspace=terraform%2Flitellm%2Fgcp%2Fexamples%2Fdefault&cloudshell_tutorial=TUTORIAL.md&cloudshell_image=gcr.io/ds-artifacts-cloudshell/deploystack_custom_image&shellonly=true) Real 1-click. Opens Cloud Shell, clones this repo, and walks you through `terraform apply` via a built-in [DeployStack tutorial](./terraform/litellm/gcp/examples/default/TUTORIAL.md) — pick the project, the tutorial sets up the Artifact Registry remote repo, writes `terraform.tfvars` from your answers, and runs apply. From 7cf1b263356fa6960da275f360db0031182fb1e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 02:10:22 +0000 Subject: [PATCH 12/29] docs(readme): collapse Railway deploy anchor to a single line The Railway button wrapped its img across indented lines, so the anchor contained leading and trailing whitespace. GitHub underlines link content, rendering that whitespace as a small blue underline beside the button. Put the anchor on one line like the other three buttons so there is no inner whitespace to underline. --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 9e6b4030966..719f74e5924 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,7 @@

Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.

Deploy to Render - - Deploy on Railway - + Deploy on Railway Deploy on AWS Deploy on GCP

From 7f57a7a068d125dbe390e665391fe63f8a26dd8e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 10 Jun 2026 03:58:08 +0000 Subject: [PATCH 13/29] Add Claude Fable 5 cost map entries as a data-only hotfix Backports only the model map changes from #30064 so deployments on released litellm versions pick up Fable 5 pricing, context window, and the adaptive thinking flag through the hosted cost map fetch without upgrading. Includes the supports_sampling_params flag on the 28 Fable 5 / Opus 4.7 / Opus 4.8 entries (ignored by released code, read by the gating that ships with the next release) and the matching one-line schema declaration so the map validation test passes. https://claude.ai/code/session_01MZarYYT3aS7DxaNjoax6Gm --- ...odel_prices_and_context_window_backup.json | 276 ++++++++++++++++++ model_prices_and_context_window.json | 276 ++++++++++++++++++ tests/test_litellm/test_utils.py | 1 + 3 files changed, 553 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 397f96fdb1e..757aacf1caf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1156,6 +1156,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1202,6 +1203,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1233,6 +1235,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1264,6 +1267,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1295,6 +1299,139 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "global.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "us.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 2.2e-05, + "cache_read_input_token_cost": 1.1e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "eu.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 2.2e-05, + "cache_read_input_token_cost": 1.1e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1327,6 +1464,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1359,6 +1497,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1391,6 +1530,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1423,6 +1563,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1455,6 +1596,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1485,6 +1627,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -2208,6 +2351,37 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-fable-5": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -2237,6 +2411,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10133,6 +10308,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10167,6 +10343,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10177,6 +10354,40 @@ }, "supports_output_config": true }, + "claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true + }, "claude-opus-4-8": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -10201,6 +10412,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -33967,6 +34179,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -33995,6 +34208,67 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-fable-5@default": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34024,6 +34298,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34053,6 +34328,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b2836a096b7..ddd7d51d76b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1156,6 +1156,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1202,6 +1203,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1233,6 +1235,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1264,6 +1267,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1295,6 +1299,139 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "global.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "us.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 2.2e-05, + "cache_read_input_token_cost": 1.1e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "bedrock_output_config_effort_ceiling": "xhigh" + }, + "eu.anthropic.claude-fable-5": { + "cache_creation_input_token_cost": 1.375e-05, + "cache_creation_input_token_cost_above_1hr": 2.2e-05, + "cache_read_input_token_cost": 1.1e-06, + "input_cost_per_token": 1.1e-05, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1327,6 +1464,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1359,6 +1497,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1391,6 +1530,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1423,6 +1563,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1455,6 +1596,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -1485,6 +1627,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -2208,6 +2351,37 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "azure_ai/claude-fable-5": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -2237,6 +2411,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10133,6 +10308,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10167,6 +10343,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -10177,6 +10354,40 @@ }, "supports_output_config": true }, + "claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "us": 1.1 + }, + "supports_output_config": true + }, "claude-opus-4-8": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -10201,6 +10412,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34007,6 +34219,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34035,6 +34248,67 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-fable-5": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true + }, + "vertex_ai/claude-fable-5@default": { + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 1e-05, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34064,6 +34338,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, @@ -34093,6 +34368,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index f179e9c8f93..4c4d9e1133b 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -858,6 +858,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, + "supports_sampling_params": {"type": "boolean"}, "supports_service_tier": {"type": "boolean"}, "supports_preset": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, From 725deeed19281d1ba34f573e7b76f7611284d191 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 15:36:21 -0700 Subject: [PATCH 14/29] feat: make rust OCR async-first --- litellm-rust/Cargo.lock | 118 +++- litellm-rust/Cargo.toml | 3 +- litellm-rust/crates/core/src/error.rs | 2 + litellm-rust/crates/core/src/lib.rs | 2 + .../crates/core/src/ocr/transformation.rs | 2 +- litellm-rust/crates/core/src/providers/mod.rs | 3 + .../crates/core/src/providers/provider.rs | 659 ++++++++++++++++++ litellm-rust/crates/providers/src/ocr.rs | 100 ++- litellm-rust/crates/python-bridge/Cargo.toml | 2 + litellm-rust/crates/python-bridge/src/lib.rs | 153 +++- litellm/ocr/main.py | 384 ++++++---- litellm/ocr/rust_bridge.py | 47 +- tests/test_litellm/ocr/test_rust_bridge.py | 118 +++- 13 files changed, 1398 insertions(+), 195 deletions(-) create mode 100644 litellm-rust/crates/core/src/providers/mod.rs create mode 100644 litellm-rust/crates/core/src/providers/provider.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 2ec86fc00e7..5f9c3a0b270 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -137,12 +137,24 @@ dependencies = [ "syn", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -152,6 +164,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -168,12 +195,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -192,8 +241,10 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -238,6 +289,31 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -293,6 +369,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -445,6 +522,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indoc" version = "2.0.7" @@ -511,7 +598,9 @@ dependencies = [ "litellm-core", "litellm-providers", "pyo3", + "pyo3-async-runtimes", "serde_json", + "tokio", ] [[package]] @@ -633,6 +722,19 @@ dependencies = [ "unindent", ] +[[package]] +name = "pyo3-async-runtimes" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "977dc837525cfd22919ba6a831413854beb7c99a256c03bf8624ad707e45810e" +dependencies = [ + "futures", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + [[package]] name = "pyo3-build-config" version = "0.23.5" @@ -815,9 +917,8 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", - "futures-channel", "futures-core", - "futures-util", + "h2", "http", "http-body", "http-body-util", @@ -1218,6 +1319,19 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 0b1528672a7..63c7d6e8b9d 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -15,7 +15,8 @@ repository = "https://github.com/BerriAI/litellm" litellm-core = { path = "crates/core" } litellm-providers = { path = "crates/providers" } pyo3 = "0.23.5" -reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } +pyo3-async-runtimes = { version = "0.23.0", features = ["tokio-runtime"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "http2"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "2.0" diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 645e261f76d..b57fb2f5de8 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -13,6 +13,8 @@ pub enum CoreError { MissingField(&'static str), #[error("invalid response: {0}")] InvalidResponse(String), + #[error("invalid provider: {0}")] + InvalidProvider(String), #[error("{0}")] Auth(String), #[error("OCR request failed with status {status}: {body}")] diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index 5d4d5bfd142..6142cece16f 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -1,5 +1,7 @@ pub mod error; pub mod ocr; +pub mod providers; pub mod realtime; pub use error::{CoreError, CoreResult}; +pub use providers::LlmProvider; diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index 7353d9d22c4..c24f3519354 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -4,7 +4,7 @@ use crate::CoreResult; use super::types::{OcrRequestData, OcrResponseData}; -pub trait OcrProviderConfig { +pub trait OcrProviderConfig: Sync { fn supported_ocr_params(&self) -> &'static [&'static str]; fn map_ocr_params(&self, non_default_params: &Map) -> Map { diff --git a/litellm-rust/crates/core/src/providers/mod.rs b/litellm-rust/crates/core/src/providers/mod.rs new file mode 100644 index 00000000000..3d6f90356a1 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/mod.rs @@ -0,0 +1,3 @@ +pub mod provider; + +pub use provider::LlmProvider; diff --git a/litellm-rust/crates/core/src/providers/provider.rs b/litellm-rust/crates/core/src/providers/provider.rs new file mode 100644 index 00000000000..f14152aae18 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/provider.rs @@ -0,0 +1,659 @@ +use std::fmt; +use std::str::FromStr; + +use crate::error::CoreError; + +#[allow(clippy::enum_variant_names)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LlmProvider { + Openai, + Chatgpt, + OpenaiLike, + JinaAi, + Xai, + Zai, + CustomOpenai, + TextCompletionOpenai, + Cohere, + CohereChat, + Clarifai, + Anthropic, + AnthropicText, + Bytez, + Replicate, + Reducto, + Runwayml, + AwsPolly, + Huggingface, + TogetherAi, + Openrouter, + Datarobot, + VertexAi, + VertexAiBeta, + Gemini, + Ai21, + Baseten, + BlackForestLabs, + Azure, + AzureText, + AzureAi, + Sagemaker, + SagemakerChat, + SagemakerNova, + Bedrock, + Vllm, + NlpCloud, + Petals, + Oobabooga, + Ollama, + OllamaChat, + Deepinfra, + Perplexity, + Mistral, + Milvus, + Groq, + A2a, + Gigachat, + NvidiaNim, + NvidiaRiva, + Soniox, + Cerebras, + Ai21Chat, + Volcengine, + Codestral, + TextCompletionCodestral, + Dashscope, + Modelscope, + Moonshot, + Publicai, + V0, + Morph, + LambdaAi, + Inception, + TextCompletionInception, + Deepseek, + Sambanova, + Maritalk, + Voyage, + Cloudflare, + Xinference, + FireworksAi, + Friendliai, + FeatherlessAi, + Watsonx, + WatsonxText, + Triton, + Predibase, + Databricks, + Empower, + Github, + Ragflow, + Compactifai, + DockerModelRunner, + Custom, + LitellmProxy, + HostedVllm, + Llamafile, + LmStudio, + Galadriel, + Nebius, + Infinity, + Deepgram, + Elevenlabs, + Novita, + AiohttpOpenai, + Langfuse, + Humanloop, + Topaz, + Sap, + Assemblyai, + CharityEngine, + GithubCopilot, + Snowflake, + GradientAi, + MetaLlama, + Nscale, + PgVector, + S3Vectors, + Helicone, + Hyperbolic, + Recraft, + FalAi, + Stability, + Heroku, + Aiml, + Cometapi, + Oci, + AutoRouter, + VercelAiGateway, + Dotprompt, + Manus, + Wandb, + Ovhcloud, + Scaleway, + Lemonade, + AmazonNova, + A2aAgent, + Langgraph, + Langflow, + Minimax, + Synthetic, + Apertis, + NanoGpt, + Poe, + Chutes, + Neosantara, + Parasail, + XiaomiMimo, + Tensormesh, + Libertai, + Pinstripes, + Darkbloom, + LitellmAgent, + Cursor, + BedrockMantle, +} + +impl LlmProvider { + #[allow(clippy::too_many_lines)] + pub fn as_str(self) -> &'static str { + match self { + LlmProvider::Openai => "openai", + LlmProvider::Chatgpt => "chatgpt", + LlmProvider::OpenaiLike => "openai_like", + LlmProvider::JinaAi => "jina_ai", + LlmProvider::Xai => "xai", + LlmProvider::Zai => "zai", + LlmProvider::CustomOpenai => "custom_openai", + LlmProvider::TextCompletionOpenai => "text-completion-openai", + LlmProvider::Cohere => "cohere", + LlmProvider::CohereChat => "cohere_chat", + LlmProvider::Clarifai => "clarifai", + LlmProvider::Anthropic => "anthropic", + LlmProvider::AnthropicText => "anthropic_text", + LlmProvider::Bytez => "bytez", + LlmProvider::Replicate => "replicate", + LlmProvider::Reducto => "reducto", + LlmProvider::Runwayml => "runwayml", + LlmProvider::AwsPolly => "aws_polly", + LlmProvider::Huggingface => "huggingface", + LlmProvider::TogetherAi => "together_ai", + LlmProvider::Openrouter => "openrouter", + LlmProvider::Datarobot => "datarobot", + LlmProvider::VertexAi => "vertex_ai", + LlmProvider::VertexAiBeta => "vertex_ai_beta", + LlmProvider::Gemini => "gemini", + LlmProvider::Ai21 => "ai21", + LlmProvider::Baseten => "baseten", + LlmProvider::BlackForestLabs => "black_forest_labs", + LlmProvider::Azure => "azure", + LlmProvider::AzureText => "azure_text", + LlmProvider::AzureAi => "azure_ai", + LlmProvider::Sagemaker => "sagemaker", + LlmProvider::SagemakerChat => "sagemaker_chat", + LlmProvider::SagemakerNova => "sagemaker_nova", + LlmProvider::Bedrock => "bedrock", + LlmProvider::Vllm => "vllm", + LlmProvider::NlpCloud => "nlp_cloud", + LlmProvider::Petals => "petals", + LlmProvider::Oobabooga => "oobabooga", + LlmProvider::Ollama => "ollama", + LlmProvider::OllamaChat => "ollama_chat", + LlmProvider::Deepinfra => "deepinfra", + LlmProvider::Perplexity => "perplexity", + LlmProvider::Mistral => "mistral", + LlmProvider::Milvus => "milvus", + LlmProvider::Groq => "groq", + LlmProvider::A2a => "a2a", + LlmProvider::Gigachat => "gigachat", + LlmProvider::NvidiaNim => "nvidia_nim", + LlmProvider::NvidiaRiva => "nvidia_riva", + LlmProvider::Soniox => "soniox", + LlmProvider::Cerebras => "cerebras", + LlmProvider::Ai21Chat => "ai21_chat", + LlmProvider::Volcengine => "volcengine", + LlmProvider::Codestral => "codestral", + LlmProvider::TextCompletionCodestral => "text-completion-codestral", + LlmProvider::Dashscope => "dashscope", + LlmProvider::Modelscope => "modelscope", + LlmProvider::Moonshot => "moonshot", + LlmProvider::Publicai => "publicai", + LlmProvider::V0 => "v0", + LlmProvider::Morph => "morph", + LlmProvider::LambdaAi => "lambda_ai", + LlmProvider::Inception => "inception", + LlmProvider::TextCompletionInception => "text-completion-inception", + LlmProvider::Deepseek => "deepseek", + LlmProvider::Sambanova => "sambanova", + LlmProvider::Maritalk => "maritalk", + LlmProvider::Voyage => "voyage", + LlmProvider::Cloudflare => "cloudflare", + LlmProvider::Xinference => "xinference", + LlmProvider::FireworksAi => "fireworks_ai", + LlmProvider::Friendliai => "friendliai", + LlmProvider::FeatherlessAi => "featherless_ai", + LlmProvider::Watsonx => "watsonx", + LlmProvider::WatsonxText => "watsonx_text", + LlmProvider::Triton => "triton", + LlmProvider::Predibase => "predibase", + LlmProvider::Databricks => "databricks", + LlmProvider::Empower => "empower", + LlmProvider::Github => "github", + LlmProvider::Ragflow => "ragflow", + LlmProvider::Compactifai => "compactifai", + LlmProvider::DockerModelRunner => "docker_model_runner", + LlmProvider::Custom => "custom", + LlmProvider::LitellmProxy => "litellm_proxy", + LlmProvider::HostedVllm => "hosted_vllm", + LlmProvider::Llamafile => "llamafile", + LlmProvider::LmStudio => "lm_studio", + LlmProvider::Galadriel => "galadriel", + LlmProvider::Nebius => "nebius", + LlmProvider::Infinity => "infinity", + LlmProvider::Deepgram => "deepgram", + LlmProvider::Elevenlabs => "elevenlabs", + LlmProvider::Novita => "novita", + LlmProvider::AiohttpOpenai => "aiohttp_openai", + LlmProvider::Langfuse => "langfuse", + LlmProvider::Humanloop => "humanloop", + LlmProvider::Topaz => "topaz", + LlmProvider::Sap => "sap", + LlmProvider::Assemblyai => "assemblyai", + LlmProvider::CharityEngine => "charity_engine", + LlmProvider::GithubCopilot => "github_copilot", + LlmProvider::Snowflake => "snowflake", + LlmProvider::GradientAi => "gradient_ai", + LlmProvider::MetaLlama => "meta_llama", + LlmProvider::Nscale => "nscale", + LlmProvider::PgVector => "pg_vector", + LlmProvider::S3Vectors => "s3_vectors", + LlmProvider::Helicone => "helicone", + LlmProvider::Hyperbolic => "hyperbolic", + LlmProvider::Recraft => "recraft", + LlmProvider::FalAi => "fal_ai", + LlmProvider::Stability => "stability", + LlmProvider::Heroku => "heroku", + LlmProvider::Aiml => "aiml", + LlmProvider::Cometapi => "cometapi", + LlmProvider::Oci => "oci", + LlmProvider::AutoRouter => "auto_router", + LlmProvider::VercelAiGateway => "vercel_ai_gateway", + LlmProvider::Dotprompt => "dotprompt", + LlmProvider::Manus => "manus", + LlmProvider::Wandb => "wandb", + LlmProvider::Ovhcloud => "ovhcloud", + LlmProvider::Scaleway => "scaleway", + LlmProvider::Lemonade => "lemonade", + LlmProvider::AmazonNova => "amazon_nova", + LlmProvider::A2aAgent => "a2a_agent", + LlmProvider::Langgraph => "langgraph", + LlmProvider::Langflow => "langflow", + LlmProvider::Minimax => "minimax", + LlmProvider::Synthetic => "synthetic", + LlmProvider::Apertis => "apertis", + LlmProvider::NanoGpt => "nano-gpt", + LlmProvider::Poe => "poe", + LlmProvider::Chutes => "chutes", + LlmProvider::Neosantara => "neosantara", + LlmProvider::Parasail => "parasail", + LlmProvider::XiaomiMimo => "xiaomi_mimo", + LlmProvider::Tensormesh => "tensormesh", + LlmProvider::Libertai => "libertai", + LlmProvider::Pinstripes => "pinstripes", + LlmProvider::Darkbloom => "darkbloom", + LlmProvider::LitellmAgent => "litellm_agent", + LlmProvider::Cursor => "cursor", + LlmProvider::BedrockMantle => "bedrock_mantle", + } + } + + #[allow(clippy::too_many_lines)] + pub const ALL: [LlmProvider; 146] = [ + LlmProvider::Openai, + LlmProvider::Chatgpt, + LlmProvider::OpenaiLike, + LlmProvider::JinaAi, + LlmProvider::Xai, + LlmProvider::Zai, + LlmProvider::CustomOpenai, + LlmProvider::TextCompletionOpenai, + LlmProvider::Cohere, + LlmProvider::CohereChat, + LlmProvider::Clarifai, + LlmProvider::Anthropic, + LlmProvider::AnthropicText, + LlmProvider::Bytez, + LlmProvider::Replicate, + LlmProvider::Reducto, + LlmProvider::Runwayml, + LlmProvider::AwsPolly, + LlmProvider::Huggingface, + LlmProvider::TogetherAi, + LlmProvider::Openrouter, + LlmProvider::Datarobot, + LlmProvider::VertexAi, + LlmProvider::VertexAiBeta, + LlmProvider::Gemini, + LlmProvider::Ai21, + LlmProvider::Baseten, + LlmProvider::BlackForestLabs, + LlmProvider::Azure, + LlmProvider::AzureText, + LlmProvider::AzureAi, + LlmProvider::Sagemaker, + LlmProvider::SagemakerChat, + LlmProvider::SagemakerNova, + LlmProvider::Bedrock, + LlmProvider::Vllm, + LlmProvider::NlpCloud, + LlmProvider::Petals, + LlmProvider::Oobabooga, + LlmProvider::Ollama, + LlmProvider::OllamaChat, + LlmProvider::Deepinfra, + LlmProvider::Perplexity, + LlmProvider::Mistral, + LlmProvider::Milvus, + LlmProvider::Groq, + LlmProvider::A2a, + LlmProvider::Gigachat, + LlmProvider::NvidiaNim, + LlmProvider::NvidiaRiva, + LlmProvider::Soniox, + LlmProvider::Cerebras, + LlmProvider::Ai21Chat, + LlmProvider::Volcengine, + LlmProvider::Codestral, + LlmProvider::TextCompletionCodestral, + LlmProvider::Dashscope, + LlmProvider::Modelscope, + LlmProvider::Moonshot, + LlmProvider::Publicai, + LlmProvider::V0, + LlmProvider::Morph, + LlmProvider::LambdaAi, + LlmProvider::Inception, + LlmProvider::TextCompletionInception, + LlmProvider::Deepseek, + LlmProvider::Sambanova, + LlmProvider::Maritalk, + LlmProvider::Voyage, + LlmProvider::Cloudflare, + LlmProvider::Xinference, + LlmProvider::FireworksAi, + LlmProvider::Friendliai, + LlmProvider::FeatherlessAi, + LlmProvider::Watsonx, + LlmProvider::WatsonxText, + LlmProvider::Triton, + LlmProvider::Predibase, + LlmProvider::Databricks, + LlmProvider::Empower, + LlmProvider::Github, + LlmProvider::Ragflow, + LlmProvider::Compactifai, + LlmProvider::DockerModelRunner, + LlmProvider::Custom, + LlmProvider::LitellmProxy, + LlmProvider::HostedVllm, + LlmProvider::Llamafile, + LlmProvider::LmStudio, + LlmProvider::Galadriel, + LlmProvider::Nebius, + LlmProvider::Infinity, + LlmProvider::Deepgram, + LlmProvider::Elevenlabs, + LlmProvider::Novita, + LlmProvider::AiohttpOpenai, + LlmProvider::Langfuse, + LlmProvider::Humanloop, + LlmProvider::Topaz, + LlmProvider::Sap, + LlmProvider::Assemblyai, + LlmProvider::CharityEngine, + LlmProvider::GithubCopilot, + LlmProvider::Snowflake, + LlmProvider::GradientAi, + LlmProvider::MetaLlama, + LlmProvider::Nscale, + LlmProvider::PgVector, + LlmProvider::S3Vectors, + LlmProvider::Helicone, + LlmProvider::Hyperbolic, + LlmProvider::Recraft, + LlmProvider::FalAi, + LlmProvider::Stability, + LlmProvider::Heroku, + LlmProvider::Aiml, + LlmProvider::Cometapi, + LlmProvider::Oci, + LlmProvider::AutoRouter, + LlmProvider::VercelAiGateway, + LlmProvider::Dotprompt, + LlmProvider::Manus, + LlmProvider::Wandb, + LlmProvider::Ovhcloud, + LlmProvider::Scaleway, + LlmProvider::Lemonade, + LlmProvider::AmazonNova, + LlmProvider::A2aAgent, + LlmProvider::Langgraph, + LlmProvider::Langflow, + LlmProvider::Minimax, + LlmProvider::Synthetic, + LlmProvider::Apertis, + LlmProvider::NanoGpt, + LlmProvider::Poe, + LlmProvider::Chutes, + LlmProvider::Neosantara, + LlmProvider::Parasail, + LlmProvider::XiaomiMimo, + LlmProvider::Tensormesh, + LlmProvider::Libertai, + LlmProvider::Pinstripes, + LlmProvider::Darkbloom, + LlmProvider::LitellmAgent, + LlmProvider::Cursor, + LlmProvider::BedrockMantle, + ]; +} + +impl fmt::Display for LlmProvider { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for LlmProvider { + type Err = CoreError; + + fn from_str(value: &str) -> Result { + LlmProvider::ALL + .iter() + .copied() + .find(|provider| provider.as_str() == value) + .ok_or_else(|| CoreError::InvalidProvider(value.to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PYTHON_LLM_PROVIDER_VALUES: &[&str] = &[ + "openai", + "chatgpt", + "openai_like", + "jina_ai", + "xai", + "zai", + "custom_openai", + "text-completion-openai", + "cohere", + "cohere_chat", + "clarifai", + "anthropic", + "anthropic_text", + "bytez", + "replicate", + "reducto", + "runwayml", + "aws_polly", + "huggingface", + "together_ai", + "openrouter", + "datarobot", + "vertex_ai", + "vertex_ai_beta", + "gemini", + "ai21", + "baseten", + "black_forest_labs", + "azure", + "azure_text", + "azure_ai", + "sagemaker", + "sagemaker_chat", + "sagemaker_nova", + "bedrock", + "vllm", + "nlp_cloud", + "petals", + "oobabooga", + "ollama", + "ollama_chat", + "deepinfra", + "perplexity", + "mistral", + "milvus", + "groq", + "a2a", + "gigachat", + "nvidia_nim", + "nvidia_riva", + "soniox", + "cerebras", + "ai21_chat", + "volcengine", + "codestral", + "text-completion-codestral", + "dashscope", + "modelscope", + "moonshot", + "publicai", + "v0", + "morph", + "lambda_ai", + "inception", + "text-completion-inception", + "deepseek", + "sambanova", + "maritalk", + "voyage", + "cloudflare", + "xinference", + "fireworks_ai", + "friendliai", + "featherless_ai", + "watsonx", + "watsonx_text", + "triton", + "predibase", + "databricks", + "empower", + "github", + "ragflow", + "compactifai", + "docker_model_runner", + "custom", + "litellm_proxy", + "hosted_vllm", + "llamafile", + "lm_studio", + "galadriel", + "nebius", + "infinity", + "deepgram", + "elevenlabs", + "novita", + "aiohttp_openai", + "langfuse", + "humanloop", + "topaz", + "sap", + "assemblyai", + "charity_engine", + "github_copilot", + "snowflake", + "gradient_ai", + "meta_llama", + "nscale", + "pg_vector", + "s3_vectors", + "helicone", + "hyperbolic", + "recraft", + "fal_ai", + "stability", + "heroku", + "aiml", + "cometapi", + "oci", + "auto_router", + "vercel_ai_gateway", + "dotprompt", + "manus", + "wandb", + "ovhcloud", + "scaleway", + "lemonade", + "amazon_nova", + "a2a_agent", + "langgraph", + "langflow", + "minimax", + "synthetic", + "apertis", + "nano-gpt", + "poe", + "chutes", + "neosantara", + "parasail", + "xiaomi_mimo", + "tensormesh", + "libertai", + "pinstripes", + "darkbloom", + "litellm_agent", + "cursor", + "bedrock_mantle", + ]; + + #[test] + fn provider_values_match_python_enum_order() { + assert_eq!(LlmProvider::ALL.len(), 146); + assert_eq!( + LlmProvider::ALL + .iter() + .map(|provider| provider.as_str()) + .collect::>(), + PYTHON_LLM_PROVIDER_VALUES + ); + } + + #[test] + fn from_str_round_trips_all_providers() { + for provider in LlmProvider::ALL { + assert_eq!(LlmProvider::from_str(provider.as_str()), Ok(provider)); + assert_eq!(provider.to_string(), provider.as_str()); + } + } + + #[test] + fn from_str_rejects_unknown_provider() { + assert_eq!( + LlmProvider::from_str("not-a-provider"), + Err(CoreError::InvalidProvider("not-a-provider".to_string())) + ); + } +} diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs index dcd56a5f0b4..a52d5532afc 100644 --- a/litellm-rust/crates/providers/src/ocr.rs +++ b/litellm-rust/crates/providers/src/ocr.rs @@ -4,12 +4,14 @@ //! resolve the API key, build the URL + body via the pure transforms, POST it, //! and normalize the response. The HTTP client is built once and reused. +use std::str::FromStr; use std::sync::OnceLock; use std::time::Duration; use litellm_core::error::CoreError; use litellm_core::ocr::transformation::OcrProviderConfig; use litellm_core::CoreResult; +use litellm_core::LlmProvider; use serde_json::{Map, Value}; use crate::mistral::ocr::transformation as mistral; @@ -25,11 +27,11 @@ const OCR_TIMEOUT_SECS: u64 = 600; /// forwarding sensitive payloads across the host boundary. const ERROR_BODY_MAX_CHARS: usize = 256; -/// Process-wide blocking HTTP client (connection pool + TLS reused across calls). -fn http_client() -> &'static reqwest::blocking::Client { - static CLIENT: OnceLock = OnceLock::new(); +/// Process-wide async HTTP client (connection pool + TLS reused across calls). +fn http_client() -> &'static reqwest::Client { + static CLIENT: OnceLock = OnceLock::new(); CLIENT.get_or_init(|| { - reqwest::blocking::Client::builder() + reqwest::Client::builder() .timeout(Duration::from_secs(OCR_TIMEOUT_SECS)) .build() .expect("failed to build reqwest client") @@ -44,39 +46,69 @@ fn truncate_error_body(body: &str) -> String { format!("{truncated}... (truncated)") } +fn ocr_config_for(provider: LlmProvider) -> Option<&'static dyn OcrProviderConfig> { + match provider { + LlmProvider::Mistral => Some(&MISTRAL_OCR_CONFIG), + _ => None, + } +} + +fn string_headers(extra_headers: Option>) -> Vec<(String, String)> { + extra_headers + .unwrap_or_default() + .into_iter() + .filter_map(|(key, value)| value.as_str().map(|value| (key, value.to_string()))) + .collect() +} + +pub struct OcrRequest<'a> { + pub model: &'a str, + pub document: Value, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub custom_llm_provider: &'a str, + pub extra_headers: Option>, + pub optional_params: Map, + pub timeout: Option, +} + /// Perform a Mistral OCR call end to end and return the normalized response as /// JSON (the shape the Python `OCRResponse` model expects). /// -/// Blocking: intended to be called with the GIL released from the Python bridge. -pub fn run_ocr( - model: &str, - document: Value, - api_key: Option<&str>, - api_base: Option<&str>, - optional_params: Map, - timeout: Option, -) -> CoreResult { - let config = &MISTRAL_OCR_CONFIG; +/// Async: intended to be awaited directly by the Python bridge's async entrypoint. +pub async fn ocr(request: OcrRequest<'_>) -> CoreResult { + let model = request.model; + let provider = LlmProvider::from_str(request.custom_llm_provider)?; + let config = + ocr_config_for(provider).ok_or_else(|| CoreError::InvalidProvider(provider.to_string()))?; - let api_key = mistral::resolve_api_key(api_key, &|key| std::env::var(key).ok())?; - let url = mistral::complete_url(api_base); - let filtered_params = config.map_ocr_params(&optional_params); + // TODO: key and URL resolution are still Mistral-specific while Mistral is + // the only Rust OCR provider. Move these onto the trait when another OCR + // provider is added here. + let api_key = mistral::resolve_api_key(request.api_key, &|key| std::env::var(key).ok())?; + let url = mistral::complete_url(request.api_base); + let filtered_params = config.map_ocr_params(&request.optional_params); let body = config - .transform_ocr_request(model, document, filtered_params)? + .transform_ocr_request(model, request.document, filtered_params)? .data; - let mut request = http_client().post(&url).bearer_auth(&api_key).json(&body); - if let Some(duration) = timeout { - request = request.timeout(duration); + let mut request_builder = http_client().post(&url).bearer_auth(&api_key).json(&body); + for (key, value) in string_headers(request.extra_headers) { + request_builder = request_builder.header(&key, value); + } + if let Some(duration) = request.timeout { + request_builder = request_builder.timeout(duration); } - let response = request + let response = request_builder .send() + .await .map_err(|err| CoreError::Network(err.to_string()))?; let status = response.status(); let text = response .text() + .await .map_err(|err| CoreError::Network(err.to_string()))?; if !status.is_success() { @@ -97,6 +129,7 @@ pub fn run_ocr( #[cfg(test)] mod tests { use super::*; + use serde_json::json; #[test] fn truncate_error_body_passes_short_strings_through() { @@ -124,4 +157,27 @@ mod tests { let truncated = truncate_error_body(&body); assert!(truncated.is_char_boundary(truncated.len())); } + + #[test] + fn ocr_registry_supports_only_mistral() { + assert!(ocr_config_for(LlmProvider::Mistral).is_some()); + assert!(ocr_config_for(LlmProvider::Openai).is_none()); + } + + #[test] + fn string_headers_keeps_only_string_values() { + let headers = json!({ + "x-trace-id": "trace-1", + "x-number": 42, + "x-bool": true + }) + .as_object() + .unwrap() + .clone(); + + assert_eq!( + string_headers(Some(headers)), + vec![("x-trace-id".to_string(), "trace-1".to_string())] + ); + } } diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 80b6478daac..d0a7cd56cc6 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -13,4 +13,6 @@ crate-type = ["cdylib"] litellm-core.workspace = true litellm-providers.workspace = true pyo3 = { workspace = true, features = ["extension-module"] } +pyo3-async-runtimes.workspace = true serde_json.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 15e93f7b00c..46ff700f456 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,7 +1,6 @@ use std::time::Duration; use litellm_core::error::CoreError; -use litellm_providers::ocr::run_ocr; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyAny, PyDict}; @@ -9,6 +8,13 @@ use serde_json::{Map, Value}; mod gil; +type MarshaledOcrInputs = ( + Value, + Option>, + Map, + Option, +); + fn py_to_json(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { let json = py.import("json")?; let encoded: String = json.call_method1("dumps", (value,))?.extract()?; @@ -28,53 +34,93 @@ fn json_to_py(py: Python<'_>, value: Value) -> PyResult> { fn core_error_to_pyerr(err: CoreError) -> PyErr { match err { CoreError::Auth(message) => PyValueError::new_err(message), - CoreError::InvalidType { .. } | CoreError::MissingField(_) => { - PyValueError::new_err(err.to_string()) - } + CoreError::InvalidProvider(_) + | CoreError::InvalidType { .. } + | CoreError::MissingField(_) => PyValueError::new_err(err.to_string()), other => PyRuntimeError::new_err(other.to_string()), } } +fn optional_object_to_map( + py: Python<'_>, + name: &'static str, + value: Option>, +) -> PyResult> { + match value { + Some(value) => match py_to_json(py, value.bind(py))? { + Value::Object(map) => Ok(map), + _ => Err(PyValueError::new_err(format!("{name} must be a dict"))), + }, + None => Ok(Map::new()), + } +} + +fn optional_timeout(timeout_seconds: Option) -> Option { + timeout_seconds.and_then(|secs| { + if secs.is_finite() && secs > 0.0 { + Some(Duration::from_secs_f64(secs)) + } else { + None + } + }) +} + +fn marshal_inputs( + py: Python<'_>, + document: Py, + extra_headers: Option>, + optional_params: Option>, + timeout_seconds: Option, +) -> PyResult { + let document = py_to_json(py, document.bind(py))?; + let extra_headers = match extra_headers { + Some(headers) => Some(optional_object_to_map(py, "extra_headers", Some(headers))?), + None => None, + }; + let optional_params = optional_object_to_map(py, "optional_params", optional_params)?; + let timeout = optional_timeout(timeout_seconds); + + Ok((document, extra_headers, optional_params, timeout)) +} + /// Perform a Mistral OCR call end to end and return the response as a dict. #[pyfunction] -#[pyo3(signature = (model, document, api_key=None, api_base=None, optional_params=None, timeout_seconds=None))] +#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] fn ocr( py: Python<'_>, model: String, document: Py, api_key: Option, api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, optional_params: Option>, timeout_seconds: Option, ) -> PyResult> { - let document = py_to_json(py, document.bind(py))?; + let custom_llm_provider = custom_llm_provider.unwrap_or_else(|| "mistral".to_string()); + let (document, extra_headers, optional_params, timeout) = marshal_inputs( + py, + document, + extra_headers, + optional_params, + timeout_seconds, + )?; - let optional_params = match optional_params { - Some(params) => match py_to_json(py, params.bind(py))? { - Value::Object(map) => map, - _ => return Err(PyValueError::new_err("optional_params must be a dict")), - }, - None => Map::new(), - }; - - let timeout = timeout_seconds.and_then(|secs| { - if secs.is_finite() && secs > 0.0 { - Some(Duration::from_secs_f64(secs)) - } else { - None - } - }); - - // Release the GIL during the blocking HTTP call (counted for observability). + // Release the GIL while the sync API waits on async Rust work. let result = gil::release_gil(py, || { - run_ocr( - &model, - document, - api_key.as_deref(), - api_base.as_deref(), - optional_params, - timeout, - ) + pyo3_async_runtimes::tokio::get_runtime().block_on(litellm_providers::ocr::ocr( + litellm_providers::ocr::OcrRequest { + model: &model, + document, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: &custom_llm_provider, + extra_headers, + optional_params, + timeout, + }, + )) }); match result { @@ -83,8 +129,50 @@ fn ocr( } } +/// Perform an OCR call end to end and return an asyncio awaitable. +#[pyfunction] +#[pyo3(signature = (model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None))] +#[allow(clippy::too_many_arguments)] +fn aocr( + py: Python<'_>, + model: String, + document: Py, + api_key: Option, + api_base: Option, + custom_llm_provider: Option, + extra_headers: Option>, + optional_params: Option>, + timeout_seconds: Option, +) -> PyResult> { + let custom_llm_provider = custom_llm_provider.unwrap_or_else(|| "mistral".to_string()); + let (document, extra_headers, optional_params, timeout) = marshal_inputs( + py, + document, + extra_headers, + optional_params, + timeout_seconds, + )?; + + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let value = litellm_providers::ocr::ocr(litellm_providers::ocr::OcrRequest { + model: &model, + document, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: &custom_llm_provider, + extra_headers, + optional_params, + timeout, + }) + .await + .map_err(core_error_to_pyerr)?; + + Python::with_gil(|py| json_to_py(py, value)) + }) +} + /// Bridge GIL accounting, e.g. `{"releases": 12}`. Lets the Python side observe -/// how often the bridge has dropped the GIL for blocking work. +/// how often the sync bridge has dropped the GIL while awaiting Rust work. #[pyfunction] fn gil_stats(py: Python<'_>) -> PyResult> { let stats = PyDict::new(py); @@ -95,6 +183,7 @@ fn gil_stats(py: Python<'_>) -> PyResult> { #[pymodule] fn litellm_python_bridge(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(ocr, module)?)?; + module.add_function(wrap_pyfunction!(aocr, module)?)?; module.add_function(wrap_pyfunction!(gil_stats, module)?)?; Ok(()) } diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 3a9ef8db804..98f6c6af8f2 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -4,11 +4,10 @@ Main OCR function for LiteLLM. import asyncio import base64 -import contextvars import mimetypes import os import re -from functools import partial +from dataclasses import dataclass from io import IOBase from typing import Any, Callable, Coroutine, Dict, Optional, Union, cast @@ -20,7 +19,13 @@ from litellm.constants import request_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.ocr.rust_bridge import RustOcr, load_rust_ocr, rust_ocr_enabled +from litellm.ocr.rust_bridge import ( + RustAocr, + RustOcr, + load_rust_aocr, + load_rust_ocr, + rust_ocr_enabled, +) from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -29,6 +34,21 @@ base_llm_http_handler = BaseLLMHTTPHandler() ################################################# +@dataclass +class _PreparedOCRRequest: + model: str + document: dict[str, object] + api_key: Optional[str] + api_base: Optional[str] + custom_llm_provider: str + extra_headers: Optional[dict[str, object]] + provider_config: BaseOCRConfig + optional_params: dict[str, object] + litellm_params: dict[str, object] + effective_timeout: Union[float, httpx.Timeout] + litellm_logging_obj: LiteLLMLoggingObj + + def _timeout_to_seconds( timeout: Optional[Union[float, httpx.Timeout]], ) -> Optional[float]: @@ -45,6 +65,107 @@ def _timeout_to_seconds( return float(timeout) +def _prepare_ocr_request( + model: str, + document: Dict[str, Any], + api_key: Optional[str], + api_base: Optional[str], + timeout: Optional[Union[float, httpx.Timeout]], + custom_llm_provider: Optional[str], + extra_headers: Optional[Dict[str, Any]], + kwargs: dict[str, object], +) -> _PreparedOCRRequest: + litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj")) + litellm_call_id = cast(Optional[str], kwargs.get("litellm_call_id", None)) + + if not isinstance(document, dict): + raise ValueError( + f"document must be a dict with 'type' and URL/file field, got {type(document)}" + ) + + doc_type = document.get("type") + + if doc_type == "file": + document = convert_file_document_to_url_document(document) + doc_type = document.get("type") + + if doc_type not in ["document_url", "image_url"]: + raise ValueError( + f"Invalid document type: {doc_type}. " + "Must be 'document_url', 'image_url', or 'file'" + ) + + ( + model, + custom_llm_provider, + dynamic_api_key, + dynamic_api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + + if dynamic_api_key: + api_key = dynamic_api_key + if dynamic_api_base: + api_base = dynamic_api_base + + ocr_provider_config = ProviderConfigManager.get_provider_ocr_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), + ) + + if ocr_provider_config is None: + raise ValueError(f"OCR is not supported for provider: {custom_llm_provider}") + + verbose_logger.debug(f"OCR call - model: {model}, provider: {custom_llm_provider}") + + litellm_params = GenericLiteLLMParams(**kwargs) + + supported_params = ocr_provider_config.get_supported_ocr_params(model=model) + non_default_params = {} + for param in supported_params: + if param in kwargs: + non_default_params[param] = kwargs.pop(param) + + optional_params = ocr_provider_config.map_ocr_params( + non_default_params=non_default_params, + optional_params={}, + model=model, + ) + + verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") + + effective_timeout = timeout or request_timeout + + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model, + optional_params=optional_params, + litellm_params={ + "litellm_call_id": litellm_call_id, + "api_base": api_base, + }, + custom_llm_provider=custom_llm_provider, + ) + + return _PreparedOCRRequest( + model=model, + document=cast(dict[str, object], document), + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=cast(Optional[dict[str, object]], extra_headers), + provider_config=ocr_provider_config, + optional_params=cast(dict[str, object], optional_params), + litellm_params=dict(litellm_params), + effective_timeout=effective_timeout, + litellm_logging_obj=litellm_logging_obj, + ) + + def _run_rust_ocr( rust_ocr: RustOcr, logging_obj: LiteLLMLoggingObj, @@ -54,6 +175,8 @@ def _run_rust_ocr( document: dict[str, object], api_key: Optional[str], api_base: Optional[str], + custom_llm_provider: str, + extra_headers: Optional[dict[str, object]], optional_params: dict[str, object], litellm_params: dict[str, object], timeout_seconds: Optional[float], @@ -68,7 +191,7 @@ def _run_rust_ocr( """ resolved_api_key = api_key or resolve_api_key("MISTRAL_API_KEY") resolved_headers = provider_config.validate_environment( - headers={}, + headers=extra_headers or {}, model=model, api_key=resolved_api_key, api_base=api_base, @@ -99,6 +222,64 @@ def _run_rust_ocr( document=document, api_key=resolved_api_key, api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=timeout_seconds, + ) + ) + + +async def _run_rust_aocr( + rust_aocr: RustAocr, + logging_obj: LiteLLMLoggingObj, + provider_config: BaseOCRConfig, + resolve_api_key: Callable[[str], Optional[str]], + model: str, + document: dict[str, object], + api_key: Optional[str], + api_base: Optional[str], + custom_llm_provider: str, + extra_headers: Optional[dict[str, object]], + optional_params: dict[str, object], + litellm_params: dict[str, object], + timeout_seconds: Optional[float], +) -> OCRResponse: + resolved_api_key = api_key or resolve_api_key("MISTRAL_API_KEY") + resolved_headers = provider_config.validate_environment( + headers=extra_headers or {}, + model=model, + api_key=resolved_api_key, + api_base=api_base, + litellm_params=litellm_params, + ) + resolved_complete_url = provider_config.get_complete_url( + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + logging_obj.pre_call( + input="OCR document processing", + api_key=resolved_api_key, + additional_args={ + "complete_input_dict": { + "model": model, + "document": document, + **optional_params, + }, + "api_base": resolved_complete_url, + "headers": resolved_headers, + }, + ) + return OCRResponse.model_validate( + await rust_aocr( + model=model, + document=document, + api_key=resolved_api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, optional_params=optional_params, timeout_seconds=timeout_seconds, ) @@ -176,17 +357,7 @@ async def aocr( """ local_vars = locals() try: - loop = asyncio.get_event_loop() - kwargs["aocr"] = True - - # Get custom llm provider - if custom_llm_provider is None: - _, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, api_base=api_base - ) - - func = partial( - ocr, + prepared = _prepare_ocr_request( model=model, document=document, api_key=api_key, @@ -194,17 +365,56 @@ async def aocr( timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, - **kwargs, + kwargs=kwargs, ) - ctx = contextvars.copy_context() - func_with_context = partial(ctx.run, func) - init_response = await loop.run_in_executor(None, func_with_context) + if prepared.custom_llm_provider == "mistral" and rust_ocr_enabled(): + rust_aocr = load_rust_aocr() + if rust_aocr is None: + verbose_logger.debug( + "Async Rust OCR bridge unavailable; falling back to Python path" + ) + else: + from litellm.secret_managers.main import get_secret_str - if asyncio.iscoroutine(init_response): - response = await init_response - else: - response = init_response + response = await _run_rust_aocr( + rust_aocr=rust_aocr, + logging_obj=prepared.litellm_logging_obj, + provider_config=prepared.provider_config, + resolve_api_key=get_secret_str, + model=prepared.model, + document=prepared.document, + api_key=prepared.api_key, + api_base=prepared.api_base, + custom_llm_provider=prepared.custom_llm_provider, + extra_headers=prepared.extra_headers, + optional_params=prepared.optional_params, + litellm_params=prepared.litellm_params, + timeout_seconds=_timeout_to_seconds(prepared.effective_timeout), + ) + if response is None: + raise ValueError( + f"Got an unexpected None response from the OCR API: {response}" + ) + return response + + response = base_llm_http_handler.ocr( + model=prepared.model, + document=prepared.document, + optional_params=prepared.optional_params, + timeout=prepared.effective_timeout, + logging_obj=prepared.litellm_logging_obj, + api_key=prepared.api_key, + api_base=prepared.api_base, + custom_llm_provider=prepared.custom_llm_provider, + aocr=True, + headers=prepared.extra_headers, + provider_config=prepared.provider_config, + litellm_params=prepared.litellm_params, + ) + + if asyncio.iscoroutine(response): + response = await response if response is None: raise ValueError( @@ -297,94 +507,20 @@ def ocr( """ local_vars = locals() try: - litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj")) - litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("aocr", False) is True - - # Validate document parameter format - if not isinstance(document, dict): - raise ValueError( - f"document must be a dict with 'type' and URL/file field, got {type(document)}" - ) - - doc_type = document.get("type") - - # Handle file type: convert to document_url/image_url with base64 data URI - if doc_type == "file": - document = convert_file_document_to_url_document(document) - doc_type = document.get("type") - - if doc_type not in ["document_url", "image_url"]: - raise ValueError( - f"Invalid document type: {doc_type}. " - "Must be 'document_url', 'image_url', or 'file'" - ) - - ( - model, - custom_llm_provider, - dynamic_api_key, - dynamic_api_base, - ) = litellm.get_llm_provider( + prepared = _prepare_ocr_request( model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, + document=document, api_key=api_key, - ) - - # Update with dynamic values if available - if dynamic_api_key: - api_key = dynamic_api_key - if dynamic_api_base: - api_base = dynamic_api_base - - ocr_provider_config: Optional[BaseOCRConfig] = ( - ProviderConfigManager.get_provider_ocr_config( - model=model, - provider=litellm.LlmProviders(custom_llm_provider), - ) - ) - - if ocr_provider_config is None: - raise ValueError( - f"OCR is not supported for provider: {custom_llm_provider}" - ) - - verbose_logger.debug( - f"OCR call - model: {model}, provider: {custom_llm_provider}" - ) - - litellm_params = GenericLiteLLMParams(**kwargs) - - supported_params = ocr_provider_config.get_supported_ocr_params(model=model) - non_default_params = {} - for param in supported_params: - if param in kwargs: - non_default_params[param] = kwargs.pop(param) - - optional_params = ocr_provider_config.map_ocr_params( - non_default_params=non_default_params, - optional_params={}, - model=model, - ) - - verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") - - effective_timeout = timeout or request_timeout - - litellm_logging_obj.update_from_kwargs( + api_base=api_base, kwargs=kwargs, - model=model, - optional_params=optional_params, - litellm_params={ - "litellm_call_id": litellm_call_id, - "api_base": api_base, - }, custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + timeout=timeout, ) # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. - if custom_llm_provider == "mistral" and rust_ocr_enabled(): + if prepared.custom_llm_provider == "mistral" and rust_ocr_enabled(): rust_ocr = load_rust_ocr() if rust_ocr is None: verbose_logger.debug( @@ -395,31 +531,33 @@ def ocr( return _run_rust_ocr( rust_ocr=rust_ocr, - logging_obj=litellm_logging_obj, - provider_config=ocr_provider_config, + logging_obj=prepared.litellm_logging_obj, + provider_config=prepared.provider_config, resolve_api_key=get_secret_str, - model=model, - document=document, - api_key=api_key, - api_base=api_base, - optional_params=optional_params, - litellm_params=dict(litellm_params), - timeout_seconds=_timeout_to_seconds(effective_timeout), + model=prepared.model, + document=prepared.document, + api_key=prepared.api_key, + api_base=prepared.api_base, + custom_llm_provider=prepared.custom_llm_provider, + extra_headers=prepared.extra_headers, + optional_params=prepared.optional_params, + litellm_params=prepared.litellm_params, + timeout_seconds=_timeout_to_seconds(prepared.effective_timeout), ) response = base_llm_http_handler.ocr( - model=model, - document=document, - optional_params=optional_params, - timeout=effective_timeout, - logging_obj=litellm_logging_obj, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, + model=prepared.model, + document=prepared.document, + optional_params=prepared.optional_params, + timeout=prepared.effective_timeout, + logging_obj=prepared.litellm_logging_obj, + api_key=prepared.api_key, + api_base=prepared.api_base, + custom_llm_provider=prepared.custom_llm_provider, aocr=_is_async, - headers=extra_headers, - provider_config=ocr_provider_config, - litellm_params=dict(litellm_params), + headers=prepared.extra_headers, + provider_config=prepared.provider_config, + litellm_params=prepared.litellm_params, ) return response diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index 61f9e8ca69a..0dec57d9168 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -11,7 +11,7 @@ can import it statically without forming an import cycle. from __future__ import annotations -from typing import Final, Protocol, cast +from typing import Awaitable, Final, Protocol, cast class RustOcr(Protocol): @@ -23,11 +23,29 @@ class RustOcr(Protocol): document: dict[str, object], api_key: str | None, api_base: str | None, + custom_llm_provider: str, + extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout_seconds: float | None, ) -> dict[str, object]: ... +class RustAocr(Protocol): + """Signature of the compiled ``litellm_python_bridge.aocr`` entrypoint.""" + + def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> Awaitable[dict[str, object]]: ... + + class _Unset: """Sentinel type so ``ocr=None`` can clear a prior injection while omission preserves it.""" @@ -36,21 +54,27 @@ _UNSET: Final[_Unset] = _Unset() _rust_ocr_enabled = False _rust_ocr_impl: RustOcr | None = None +_rust_aocr_impl: RustAocr | None = None def use_litellm_rust( - enabled: bool = True, *, ocr: RustOcr | None | _Unset = _UNSET + enabled: bool = True, + *, + ocr: RustOcr | None | _Unset = _UNSET, + aocr: RustAocr | None | _Unset = _UNSET, ) -> None: """Route supported OCR calls through the Rust ``litellm_python_bridge`` extension. - ``ocr`` injects the bridge callable; when omitted the compiled extension is - loaded on demand and any previously injected bridge is preserved. Pass - ``ocr=None`` explicitly to clear a prior injection. + ``ocr`` and ``aocr`` inject bridge callables; when omitted the compiled + extension is loaded on demand and any previously injected bridge is + preserved. Pass ``None`` explicitly to clear a prior injection. """ - global _rust_ocr_enabled, _rust_ocr_impl + global _rust_ocr_enabled, _rust_ocr_impl, _rust_aocr_impl _rust_ocr_enabled = enabled if not isinstance(ocr, _Unset): _rust_ocr_impl = ocr + if not isinstance(aocr, _Unset): + _rust_aocr_impl = aocr def rust_ocr_enabled() -> bool: @@ -72,3 +96,14 @@ def load_rust_ocr() -> RustOcr | None: except ImportError: return None return cast(RustOcr, litellm_python_bridge.ocr) + + +def load_rust_aocr() -> RustAocr | None: + """Return the async Rust OCR callable, or ``None`` when unavailable.""" + if _rust_aocr_impl is not None: + return _rust_aocr_impl + try: + import litellm_python_bridge + except ImportError: + return None + return cast(RustAocr, getattr(litellm_python_bridge, "aocr", None)) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 7e028064e4c..aee8506b84b 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -35,7 +35,15 @@ class RecordingBridge: self.calls = [] def __call__( - self, model, document, api_key, api_base, optional_params, timeout_seconds + self, + model, + document, + api_key, + api_base, + custom_llm_provider, + extra_headers, + optional_params, + timeout_seconds, ): self.calls.append( { @@ -43,6 +51,40 @@ class RecordingBridge: "document": document, "api_key": api_key, "api_base": api_base, + "custom_llm_provider": custom_llm_provider, + "extra_headers": extra_headers, + "optional_params": optional_params, + "timeout_seconds": timeout_seconds, + } + ) + return dict(FAKE_OCR_RESPONSE) + + +class RecordingAsyncBridge: + """A fake async ``RustAocr`` callable that records the args it was handed.""" + + def __init__(self): + self.calls = [] + + async def __call__( + self, + model, + document, + api_key, + api_base, + custom_llm_provider, + extra_headers, + optional_params, + timeout_seconds, + ): + self.calls.append( + { + "model": model, + "document": document, + "api_key": api_key, + "api_base": api_base, + "custom_llm_provider": custom_llm_provider, + "extra_headers": extra_headers, "optional_params": optional_params, "timeout_seconds": timeout_seconds, } @@ -70,7 +112,7 @@ class FakeOCRConfig: def validate_environment( self, *, headers, model, api_key, api_base, litellm_params ): - return {"authorization": f"Bearer {api_key}"} + return {"Authorization": f"Bearer {api_key}", **headers} def get_complete_url(self, *, api_base, model, optional_params, litellm_params): return f"{api_base or 'https://api.mistral.ai/v1'}/ocr" @@ -79,9 +121,9 @@ class FakeOCRConfig: @pytest.fixture(autouse=True) def _reset_rust_flag(): """Keep the global toggle isolated between tests.""" - rust_bridge.use_litellm_rust(False, ocr=None) + rust_bridge.use_litellm_rust(False, ocr=None, aocr=None) yield - rust_bridge.use_litellm_rust(False, ocr=None) + rust_bridge.use_litellm_rust(False, ocr=None, aocr=None) @pytest.fixture @@ -92,6 +134,14 @@ def fake_bridge(): return bridge +@pytest.fixture +def fake_async_bridge(): + """Enable the async Rust path with an injected recording bridge.""" + bridge = RecordingAsyncBridge() + litellm.use_litellm_rust(True, aocr=bridge) + return bridge + + def test_use_litellm_rust_toggles_flag(): assert rust_bridge.rust_ocr_enabled() is False litellm.use_litellm_rust() @@ -106,6 +156,12 @@ def test_load_rust_ocr_returns_injected_impl(): assert rust_bridge.load_rust_ocr() is bridge +def test_load_rust_aocr_returns_injected_impl(): + bridge = RecordingAsyncBridge() + litellm.use_litellm_rust(True, aocr=bridge) + assert rust_bridge.load_rust_aocr() is bridge + + def test_toggle_without_ocr_arg_preserves_injected_impl(): """Regression: routine enable/disable calls must not clobber a prior injection. @@ -114,20 +170,25 @@ def test_toggle_without_ocr_arg_preserves_injected_impl(): a caller toggled the flag without re-passing ``ocr=``. """ bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + async_bridge = RecordingAsyncBridge() + litellm.use_litellm_rust(True, ocr=bridge, aocr=async_bridge) litellm.use_litellm_rust(False) assert rust_bridge.load_rust_ocr() is bridge + assert rust_bridge.load_rust_aocr() is async_bridge litellm.use_litellm_rust(True) assert rust_bridge.load_rust_ocr() is bridge + assert rust_bridge.load_rust_aocr() is async_bridge def test_explicit_ocr_none_clears_injected_impl(): bridge = RecordingBridge() - litellm.use_litellm_rust(True, ocr=bridge) + async_bridge = RecordingAsyncBridge() + litellm.use_litellm_rust(True, ocr=bridge, aocr=async_bridge) - litellm.use_litellm_rust(True, ocr=None) + litellm.use_litellm_rust(True, ocr=None, aocr=None) assert rust_bridge.load_rust_ocr() is None + assert rust_bridge.load_rust_aocr() is None def test_load_rust_ocr_none_when_extension_absent(): @@ -135,6 +196,7 @@ def test_load_rust_ocr_none_when_extension_absent(): caller degrades to the Python path instead of raising ImportError.""" litellm.use_litellm_rust(True) # no impl injected; extension isn't built in CI assert rust_bridge.load_rust_ocr() is None + assert rust_bridge.load_rust_aocr() is None def test_load_rust_ocr_uses_compiled_extension(monkeypatch): @@ -143,10 +205,12 @@ def test_load_rust_ocr_uses_compiled_extension(monkeypatch): built in CI, so stand in a fake module via ``sys.modules``.""" fake_module = types.ModuleType("litellm_python_bridge") fake_module.ocr = lambda **kwargs: dict(FAKE_OCR_RESPONSE) # type: ignore[attr-defined] + fake_module.aocr = lambda **kwargs: dict(FAKE_OCR_RESPONSE) # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "litellm_python_bridge", fake_module) litellm.use_litellm_rust(True) # enabled, no impl injected -> import the extension assert rust_bridge.load_rust_ocr() is fake_module.ocr + assert rust_bridge.load_rust_aocr() is fake_module.aocr def test_timeout_to_seconds_handles_float_timeout_and_none(): @@ -168,6 +232,8 @@ def test_run_rust_ocr_forwards_args_and_wraps_response(): document=DOCUMENT, api_key="sk-test", api_base="https://proxy.internal", + custom_llm_provider="mistral", + extra_headers={"x-trace-id": "trace-1"}, optional_params={"include_image_base64": True}, litellm_params={}, timeout_seconds=12.5, @@ -181,6 +247,8 @@ def test_run_rust_ocr_forwards_args_and_wraps_response(): "document": DOCUMENT, "api_key": "sk-test", "api_base": "https://proxy.internal", + "custom_llm_provider": "mistral", + "extra_headers": {"x-trace-id": "trace-1"}, "optional_params": {"include_image_base64": True}, "timeout_seconds": 12.5, } @@ -202,6 +270,8 @@ def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): document=DOCUMENT, api_key=None, api_base=None, + custom_llm_provider="mistral", + extra_headers=None, optional_params={}, litellm_params={}, timeout_seconds=None, @@ -227,6 +297,8 @@ def test_run_rust_ocr_prefers_explicit_key_over_resolver(): document=DOCUMENT, api_key="sk-explicit", api_base=None, + custom_llm_provider="mistral", + extra_headers=None, optional_params={}, litellm_params={}, timeout_seconds=None, @@ -249,6 +321,8 @@ def test_run_rust_ocr_runs_pre_call_logging(): document=DOCUMENT, api_key="sk-test", api_base="https://api.mistral.ai/v1", + custom_llm_provider="mistral", + extra_headers={"x-trace-id": "trace-1"}, optional_params={"include_image_base64": True}, litellm_params={}, timeout_seconds=None, @@ -262,7 +336,10 @@ def test_run_rust_ocr_runs_pre_call_logging(): assert complete_input["include_image_base64"] is True # The logged request mirrors what Rust sends: resolved URL + headers. assert additional_args["api_base"] == "https://api.mistral.ai/v1/ocr" - assert additional_args["headers"] == {"authorization": "Bearer sk-test"} + assert additional_args["headers"] == { + "Authorization": "Bearer sk-test", + "x-trace-id": "trace-1", + } def test_ocr_routes_to_rust_when_enabled(fake_bridge): @@ -270,6 +347,7 @@ def test_ocr_routes_to_rust_when_enabled(fake_bridge): model=MODEL, document=DOCUMENT, api_key="sk-test", + extra_headers={"x-trace-id": "trace-1"}, include_image_base64=True, ) @@ -281,10 +359,34 @@ def test_ocr_routes_to_rust_when_enabled(fake_bridge): assert call["model"] == "mistral-ocr-latest" assert call["document"] == DOCUMENT assert call["api_key"] == "sk-test" + assert call["custom_llm_provider"] == "mistral" + assert call["extra_headers"] == {"x-trace-id": "trace-1"} # Raw OCR params ride along in optional_params; Rust filters to supported keys. assert call["optional_params"].get("include_image_base64") is True +@pytest.mark.asyncio +async def test_aocr_routes_to_async_rust_when_enabled(fake_async_bridge): + response = await litellm.aocr( + model=MODEL, + document=DOCUMENT, + api_key="sk-test", + extra_headers={"x-trace-id": "trace-1"}, + include_image_base64=True, + ) + + assert isinstance(response, OCRResponse) + assert response.pages[0].markdown == "hello world" + assert len(fake_async_bridge.calls) == 1 + call = fake_async_bridge.calls[0] + assert call["model"] == "mistral-ocr-latest" + assert call["document"] == DOCUMENT + assert call["api_key"] == "sk-test" + assert call["custom_llm_provider"] == "mistral" + assert call["extra_headers"] == {"x-trace-id": "trace-1"} + assert call["optional_params"].get("include_image_base64") is True + + def test_ocr_forwards_timeout_to_rust(fake_bridge): """Caller-supplied timeout must flow into the Rust bridge so the fixed 600s client ceiling doesn't silently override shorter deadlines.""" From a9a1af607d82b9a01e2b196e46e6fb064d7d7e5a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 15:38:09 -0700 Subject: [PATCH 15/29] docs: clarify rust provider call flow --- litellm-rust/ADDING_A_PROVIDER.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index 2fa81798605..dc4b95ea173 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -1,9 +1,8 @@ # Adding a provider / route to litellm-rust -Three layers, same for every route (see `ocr` and `realtime` as references): - -1. **Transform contract (pure)** — `crates/core/src//transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) + types in `types.rs`. No network, env, or auth. -2. **Provider config (pure)** — `crates/providers/src///transformation.rs`: implement that trait as a `const __CONFIG`, mirroring the Python provider tree. Add parity unit tests. -3. **HTTP / transport (the host)** — `crates/providers/src/.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O. - -**Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`. +- Keep the route contract pure in `crates/core/src//`: define the typed request/response structs and a provider config trait with no network, env, auth, or logging. +- Put provider-specific transforms in `crates/providers/src///transformation.rs`, mirroring the Python provider tree and exposing a `const __CONFIG`. +- The provider config owns three pure steps: map LiteLLM params, transform the LiteLLM request into the provider request, and transform the provider response back into the LiteLLM response. +- If the provider has a reverse or normalization step, keep it pure and explicit next to the transforms; do not hide reverse mapping inside the HTTP transport. +- Route host functions in `crates/providers/src/.rs` must be async: resolve auth/base URL, call the transforms, send with async transport, then call the response transform. +- Register modules in `lib.rs` / `mod.rs`, add parity tests for params/request/response behavior, then run `cargo fmt && cargo clippy --workspace --all-targets --locked -- -D warnings && cargo test --workspace --locked`. From 463148d6a96e4b80c553d6a75baef40033eb28a3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 15:55:33 -0700 Subject: [PATCH 16/29] docs: clarify OCR provider transform contract --- litellm-rust/crates/core/src/ocr/transformation.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index c24f3519354..049959c7bc1 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -4,7 +4,12 @@ use crate::CoreResult; use super::types::{OcrRequestData, OcrResponseData}; -pub trait OcrProviderConfig: Sync { +/// Provider-specific OCR transforms. +/// +/// Implementations should stay pure and non-blocking: map supported params, +/// build the provider request body, and normalize the provider response. The +/// route layer owns async HTTP I/O. +pub trait OcrProviderConfig: Send + Sync { fn supported_ocr_params(&self) -> &'static [&'static str]; fn map_ocr_params(&self, non_default_params: &Map) -> Map { From e55b9f981a4f382ea684a781f2f3710d96ac1af8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 15:58:24 -0700 Subject: [PATCH 17/29] docs: note Tokio route contract --- litellm-rust/ADDING_A_PROVIDER.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index dc4b95ea173..4a85af61905 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -4,5 +4,5 @@ - Put provider-specific transforms in `crates/providers/src///transformation.rs`, mirroring the Python provider tree and exposing a `const __CONFIG`. - The provider config owns three pure steps: map LiteLLM params, transform the LiteLLM request into the provider request, and transform the provider response back into the LiteLLM response. - If the provider has a reverse or normalization step, keep it pure and explicit next to the transforms; do not hide reverse mapping inside the HTTP transport. -- Route host functions in `crates/providers/src/.rs` must be async: resolve auth/base URL, call the transforms, send with async transport, then call the response transform. +- Route host functions in `crates/providers/src/.rs` must be async: resolve auth/base URL, call the transforms, send with async transport, then call the response transform. Use Tokio/async all the way through Rust route I/O; only the PyO3 sync compatibility wrapper should `block_on` the async route, and it must release the GIL while waiting. - Register modules in `lib.rs` / `mod.rs`, add parity tests for params/request/response behavior, then run `cargo fmt && cargo clippy --workspace --all-targets --locked -- -D warnings && cargo test --workspace --locked`. From 82ec9aeb70d729c736f2fc9d383d6b0f72312843 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 16:04:31 -0700 Subject: [PATCH 18/29] fix: address OCR bridge review comments --- litellm-rust/crates/core/src/error.rs | 2 + litellm-rust/crates/providers/src/ocr.rs | 42 ++++++-- litellm-rust/crates/python-bridge/src/lib.rs | 1 + litellm/llms/base_llm/ocr/transformation.py | 6 ++ litellm/llms/mistral/ocr/transformation.py | 7 +- litellm/ocr/main.py | 102 ++++++++++++------- litellm/ocr/rust_bridge.py | 6 +- tests/test_litellm/ocr/test_rust_bridge.py | 34 +++++++ 8 files changed, 151 insertions(+), 49 deletions(-) diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index b57fb2f5de8..c05bd6103e9 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -13,6 +13,8 @@ pub enum CoreError { MissingField(&'static str), #[error("invalid response: {0}")] InvalidResponse(String), + #[error("invalid request: {0}")] + InvalidRequest(String), #[error("invalid provider: {0}")] InvalidProvider(String), #[error("{0}")] diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs index a52d5532afc..17d5dc1ac83 100644 --- a/litellm-rust/crates/providers/src/ocr.rs +++ b/litellm-rust/crates/providers/src/ocr.rs @@ -53,11 +53,21 @@ fn ocr_config_for(provider: LlmProvider) -> Option<&'static dyn OcrProviderConfi } } -fn string_headers(extra_headers: Option>) -> Vec<(String, String)> { +fn string_headers(extra_headers: Option>) -> CoreResult> { extra_headers .unwrap_or_default() .into_iter() - .filter_map(|(key, value)| value.as_str().map(|value| (key, value.to_string()))) + .map(|(key, value)| { + value + .as_str() + .map(|value| (key.clone(), value.to_string())) + .ok_or_else(|| { + CoreError::InvalidRequest(format!( + "OCR extra_headers.{key} must be a string, got {}", + litellm_core::error::json_type_name(&value) + )) + }) + }) .collect() } @@ -93,7 +103,7 @@ pub async fn ocr(request: OcrRequest<'_>) -> CoreResult { .data; let mut request_builder = http_client().post(&url).bearer_auth(&api_key).json(&body); - for (key, value) in string_headers(request.extra_headers) { + for (key, value) in string_headers(request.extra_headers)? { request_builder = request_builder.header(&key, value); } if let Some(duration) = request.timeout { @@ -165,19 +175,35 @@ mod tests { } #[test] - fn string_headers_keeps_only_string_values() { + fn string_headers_accepts_string_values() { let headers = json!({ - "x-trace-id": "trace-1", - "x-number": 42, - "x-bool": true + "x-trace-id": "trace-1" }) .as_object() .unwrap() .clone(); assert_eq!( - string_headers(Some(headers)), + string_headers(Some(headers)).expect("string headers accepted"), vec![("x-trace-id".to_string(), "trace-1".to_string())] ); } + + #[test] + fn string_headers_rejects_non_string_values() { + let headers = json!({ + "x-retry-count": 3 + }) + .as_object() + .unwrap() + .clone(); + + let err = string_headers(Some(headers)).expect_err("non-string header rejected"); + assert_eq!( + err, + CoreError::InvalidRequest( + "OCR extra_headers.x-retry-count must be a string, got number".to_string() + ) + ); + } } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 46ff700f456..aa1fb2e0127 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -36,6 +36,7 @@ fn core_error_to_pyerr(err: CoreError) -> PyErr { CoreError::Auth(message) => PyValueError::new_err(message), CoreError::InvalidProvider(_) | CoreError::InvalidType { .. } + | CoreError::InvalidRequest(_) | CoreError::MissingField(_) => PyValueError::new_err(err.to_string()), other => PyRuntimeError::new_err(other.to_string()), } diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index 263e0c094ce..de6bc2471ec 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -101,6 +101,12 @@ class BaseOCRConfig: """ return [] + def get_api_key_env_var(self) -> Optional[str]: + """ + Return the provider-specific API key environment variable name, if any. + """ + return None + def map_ocr_params( self, non_default_params: dict, diff --git a/litellm/llms/mistral/ocr/transformation.py b/litellm/llms/mistral/ocr/transformation.py index 21e0e27a314..49555baaf70 100644 --- a/litellm/llms/mistral/ocr/transformation.py +++ b/litellm/llms/mistral/ocr/transformation.py @@ -15,6 +15,8 @@ from litellm.llms.base_llm.ocr.transformation import ( ) from litellm.secret_managers.main import get_secret_str +MISTRAL_OCR_API_KEY_ENV_VAR = "MISTRAL_API_KEY" + class MistralOCRConfig(BaseOCRConfig): """ @@ -59,6 +61,9 @@ class MistralOCRConfig(BaseOCRConfig): "id", ] + def get_api_key_env_var(self) -> Optional[str]: + return MISTRAL_OCR_API_KEY_ENV_VAR + def map_ocr_params( self, non_default_params: dict, @@ -95,7 +100,7 @@ class MistralOCRConfig(BaseOCRConfig): """ # Get API key from environment if not provided if api_key is None: - api_key = get_secret_str("MISTRAL_API_KEY") + api_key = get_secret_str(MISTRAL_OCR_API_KEY_ENV_VAR) if api_key is None: raise ValueError( diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 98f6c6af8f2..f79c17256ca 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -49,6 +49,13 @@ class _PreparedOCRRequest: litellm_logging_obj: LiteLLMLoggingObj +@dataclass +class _PreparedRustOCRCall: + api_key: Optional[str] + headers: dict[str, object] + complete_url: str + + def _timeout_to_seconds( timeout: Optional[Union[float, httpx.Timeout]], ) -> Optional[float]: @@ -166,8 +173,7 @@ def _prepare_ocr_request( ) -def _run_rust_ocr( - rust_ocr: RustOcr, +def _prepare_rust_ocr_call( logging_obj: LiteLLMLoggingObj, provider_config: BaseOCRConfig, resolve_api_key: Callable[[str], Optional[str]], @@ -175,21 +181,14 @@ def _run_rust_ocr( document: dict[str, object], api_key: Optional[str], api_base: Optional[str], - custom_llm_provider: str, extra_headers: Optional[dict[str, object]], optional_params: dict[str, object], litellm_params: dict[str, object], - timeout_seconds: Optional[float], -) -> OCRResponse: - """Run the Mistral OCR call through the Rust bridge and wrap the result. - - Resolves the key the same way the Python path does so secret-manager backends - (AWS/Azure/GCP/Vault) work; the Rust bridge's own fallback only reads the - process environment. The request that Rust actually sends (resolved URL and - headers) is mirrored into pre_call so logs match the wire. Dependencies are - injected so this stays unit-testable without patching module globals. - """ - resolved_api_key = api_key or resolve_api_key("MISTRAL_API_KEY") +) -> _PreparedRustOCRCall: + api_key_env_var = provider_config.get_api_key_env_var() + resolved_api_key = api_key or ( + resolve_api_key(api_key_env_var) if api_key_env_var is not None else None + ) resolved_headers = provider_config.validate_environment( headers=extra_headers or {}, model=model, @@ -216,11 +215,53 @@ def _run_rust_ocr( "headers": resolved_headers, }, ) + return _PreparedRustOCRCall( + api_key=resolved_api_key, + headers=cast(dict[str, object], resolved_headers), + complete_url=resolved_complete_url, + ) + + +def _run_rust_ocr( + rust_ocr: RustOcr, + logging_obj: LiteLLMLoggingObj, + provider_config: BaseOCRConfig, + resolve_api_key: Callable[[str], Optional[str]], + model: str, + document: dict[str, object], + api_key: Optional[str], + api_base: Optional[str], + custom_llm_provider: str, + extra_headers: Optional[dict[str, object]], + optional_params: dict[str, object], + litellm_params: dict[str, object], + timeout_seconds: Optional[float], +) -> OCRResponse: + """Run the Mistral OCR call through the Rust bridge and wrap the result. + + Resolves the key the same way the Python path does so secret-manager backends + (AWS/Azure/GCP/Vault) work; the Rust bridge's own fallback only reads the + process environment. The request that Rust actually sends (resolved URL and + headers) is mirrored into pre_call so logs match the wire. Dependencies are + injected so this stays unit-testable without patching module globals. + """ + prepared = _prepare_rust_ocr_call( + logging_obj=logging_obj, + provider_config=provider_config, + resolve_api_key=resolve_api_key, + model=model, + document=document, + api_key=api_key, + api_base=api_base, + extra_headers=extra_headers, + optional_params=optional_params, + litellm_params=litellm_params, + ) return OCRResponse.model_validate( rust_ocr( model=model, document=document, - api_key=resolved_api_key, + api_key=prepared.api_key, api_base=api_base, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, @@ -245,38 +286,23 @@ async def _run_rust_aocr( litellm_params: dict[str, object], timeout_seconds: Optional[float], ) -> OCRResponse: - resolved_api_key = api_key or resolve_api_key("MISTRAL_API_KEY") - resolved_headers = provider_config.validate_environment( - headers=extra_headers or {}, + prepared = _prepare_rust_ocr_call( + logging_obj=logging_obj, + provider_config=provider_config, + resolve_api_key=resolve_api_key, model=model, - api_key=resolved_api_key, + document=document, + api_key=api_key, api_base=api_base, - litellm_params=litellm_params, - ) - resolved_complete_url = provider_config.get_complete_url( - api_base=api_base, - model=model, + extra_headers=extra_headers, optional_params=optional_params, litellm_params=litellm_params, ) - logging_obj.pre_call( - input="OCR document processing", - api_key=resolved_api_key, - additional_args={ - "complete_input_dict": { - "model": model, - "document": document, - **optional_params, - }, - "api_base": resolved_complete_url, - "headers": resolved_headers, - }, - ) return OCRResponse.model_validate( await rust_aocr( model=model, document=document, - api_key=resolved_api_key, + api_key=prepared.api_key, api_base=api_base, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, diff --git a/litellm/ocr/rust_bridge.py b/litellm/ocr/rust_bridge.py index 0dec57d9168..1e3312c1473 100644 --- a/litellm/ocr/rust_bridge.py +++ b/litellm/ocr/rust_bridge.py @@ -27,7 +27,8 @@ class RustOcr(Protocol): extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout_seconds: float | None, - ) -> dict[str, object]: ... + ) -> dict[str, object]: + raise NotImplementedError class RustAocr(Protocol): @@ -43,7 +44,8 @@ class RustAocr(Protocol): extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout_seconds: float | None, - ) -> Awaitable[dict[str, object]]: ... + ) -> Awaitable[dict[str, object]]: + raise NotImplementedError class _Unset: diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index aee8506b84b..d51b56330d2 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -109,6 +109,12 @@ class RecordingLogging: class FakeOCRConfig: """A stand-in ``BaseOCRConfig`` that echoes the request it would build.""" + def __init__(self, api_key_env_var="MISTRAL_API_KEY"): + self.api_key_env_var = api_key_env_var + + def get_api_key_env_var(self): + return self.api_key_env_var + def validate_environment( self, *, headers, model, api_key, api_base, litellm_params ): @@ -280,6 +286,34 @@ def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): assert bridge.calls[0]["api_key"] == "sk-from-vault" +def test_run_rust_ocr_uses_provider_api_key_env_var(): + bridge = RecordingBridge() + resolver_calls = [] + + def _resolver(name): + resolver_calls.append(name) + return "sk-provider-env" + + ocr_main._run_rust_ocr( + rust_ocr=bridge, + logging_obj=RecordingLogging(), + provider_config=FakeOCRConfig(api_key_env_var="PROVIDER_OCR_API_KEY"), + resolve_api_key=_resolver, + model="provider-ocr-model", + document=DOCUMENT, + api_key=None, + api_base=None, + custom_llm_provider="mistral", + extra_headers=None, + optional_params={}, + litellm_params={}, + timeout_seconds=None, + ) + + assert resolver_calls == ["PROVIDER_OCR_API_KEY"] + assert bridge.calls[0]["api_key"] == "sk-provider-env" + + def test_run_rust_ocr_prefers_explicit_key_over_resolver(): bridge = RecordingBridge() resolver_calls = [] From 5d5dd898298202ff9e1d4481462f15a1cd3a304f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 16:12:04 -0700 Subject: [PATCH 19/29] docs: bound rust OCR HTTP exception --- litellm-rust/ADDING_A_PROVIDER.md | 1 + litellm-rust/crates/providers/src/ocr.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index 4a85af61905..f9a725c833a 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -5,4 +5,5 @@ - The provider config owns three pure steps: map LiteLLM params, transform the LiteLLM request into the provider request, and transform the provider response back into the LiteLLM response. - If the provider has a reverse or normalization step, keep it pure and explicit next to the transforms; do not hide reverse mapping inside the HTTP transport. - Route host functions in `crates/providers/src/.rs` must be async: resolve auth/base URL, call the transforms, send with async transport, then call the response transform. Use Tokio/async all the way through Rust route I/O; only the PyO3 sync compatibility wrapper should `block_on` the async route, and it must release the GIL while waiting. +- Do not add per-provider HTTP clients casually. Today Rust cannot call Python's `BaseLLMHTTPHandler`; if a route needs end-to-end Rust I/O, keep the async transport route-scoped, opt-in from Python, and do not broaden it to more providers until there is a shared Rust HTTP abstraction. - Register modules in `lib.rs` / `mod.rs`, add parity tests for params/request/response behavior, then run `cargo fmt && cargo clippy --workspace --all-targets --locked -- -D warnings && cargo test --workspace --locked`. diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs index 17d5dc1ac83..03d7ce508c8 100644 --- a/litellm-rust/crates/providers/src/ocr.rs +++ b/litellm-rust/crates/providers/src/ocr.rs @@ -28,6 +28,11 @@ const OCR_TIMEOUT_SECS: u64 = 600; const ERROR_BODY_MAX_CHARS: usize = 256; /// Process-wide async HTTP client (connection pool + TLS reused across calls). +/// +/// The Python fallback path uses LiteLLM's standard `BaseLLMHTTPHandler`. This +/// Rust path is opt-in and owns end-to-end OCR I/O, so it cannot call the +/// Python handler directly; keep this route-scoped until litellm-rust has a +/// shared HTTP abstraction. fn http_client() -> &'static reqwest::Client { static CLIENT: OnceLock = OnceLock::new(); CLIENT.get_or_init(|| { From 9ba5a1522b445d2cf206c7c86d6c081f5cecd97c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 16:32:21 -0700 Subject: [PATCH 20/29] feat: generate rust providers from registry --- litellm-rust/ADDING_A_PROVIDER.md | 1 + litellm-rust/crates/core/Cargo.toml | 4 + litellm-rust/crates/core/build.rs | 335 +++++++ .../crates/core/src/providers/provider.rs | 660 +------------ litellm-rust/providers/README.md | 21 + litellm-rust/providers/providers.json | 878 ++++++++++++++++++ 6 files changed, 1240 insertions(+), 659 deletions(-) create mode 100644 litellm-rust/crates/core/build.rs create mode 100644 litellm-rust/providers/README.md create mode 100644 litellm-rust/providers/providers.json diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index f9a725c833a..ac454727c8a 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -1,6 +1,7 @@ # Adding a provider / route to litellm-rust - Keep the route contract pure in `crates/core/src//`: define the typed request/response structs and a provider config trait with no network, env, auth, or logging. +- Add provider identity to `providers/providers.json`: `routing_name` must match Python `LlmProviders.value`; `display_name` is human-facing; `default_api_base` and `api_key_env_var` are optional provider defaults. - Put provider-specific transforms in `crates/providers/src///transformation.rs`, mirroring the Python provider tree and exposing a `const __CONFIG`. - The provider config owns three pure steps: map LiteLLM params, transform the LiteLLM request into the provider request, and transform the provider response back into the LiteLLM response. - If the provider has a reverse or normalization step, keep it pure and explicit next to the transforms; do not hide reverse mapping inside the HTTP transport. diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index e54002fe5e8..c3b16431b36 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -9,3 +9,7 @@ repository.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true + +[build-dependencies] +serde.workspace = true +serde_json.workspace = true diff --git a/litellm-rust/crates/core/build.rs b/litellm-rust/crates/core/build.rs new file mode 100644 index 00000000000..e31152c2035 --- /dev/null +++ b/litellm-rust/crates/core/build.rs @@ -0,0 +1,335 @@ +use std::collections::HashSet; +use std::env; +use std::fs; +use std::path::PathBuf; + +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +struct ProviderMetadataInput { + routing_name: String, + display_name: String, + default_api_base: Option, + api_key_env_var: Option, +} + +fn rust_string(value: &str) -> String { + format!("{value:?}") +} + +fn rust_option(value: Option<&str>) -> String { + value + .map(|value| format!("Some({})", rust_string(value))) + .unwrap_or_else(|| "None".to_string()) +} + +fn variant_name(routing_name: &str) -> String { + routing_name + .split(['_', '-', '.']) + .filter(|part| !part.is_empty()) + .map(|part| { + let mut chars = part.chars(); + match chars.next() { + Some(first) => { + let mut out = String::new(); + out.extend(first.to_uppercase()); + out.push_str(chars.as_str()); + out + } + None => String::new(), + } + }) + .collect() +} + +fn validate_providers(providers: &[ProviderMetadataInput]) { + let mut routing_names = HashSet::new(); + let mut variants = HashSet::new(); + + for provider in providers { + if provider.routing_name.trim().is_empty() { + panic!("provider routing_name cannot be empty"); + } + if provider.display_name.trim().is_empty() { + panic!( + "provider {} has an empty display_name", + provider.routing_name + ); + } + if !routing_names.insert(provider.routing_name.as_str()) { + panic!("duplicate provider routing_name: {}", provider.routing_name); + } + + let variant = variant_name(&provider.routing_name); + if variant.is_empty() { + panic!( + "provider {} generated an empty Rust variant", + provider.routing_name + ); + } + if !variants.insert(variant.clone()) { + panic!( + "provider {} generated duplicate Rust variant {variant}", + provider.routing_name + ); + } + } +} + +fn generate_provider_code(providers: &[ProviderMetadataInput]) -> String { + let variants: Vec = providers + .iter() + .map(|provider| variant_name(&provider.routing_name)) + .collect(); + + let enum_variants = variants + .iter() + .map(|variant| format!(" {variant},")) + .collect::>() + .join("\n"); + + let all_values = variants + .iter() + .map(|variant| format!(" LlmProvider::{variant},")) + .collect::>() + .join("\n"); + + let metadata_values = providers + .iter() + .zip(variants.iter()) + .map(|(provider, variant)| { + format!( + " ProviderMetadata {{ provider: LlmProvider::{variant}, routing_name: {}, display_name: {}, default_api_base: {}, api_key_env_var: {} }},", + rust_string(&provider.routing_name), + rust_string(&provider.display_name), + rust_option(provider.default_api_base.as_deref()), + rust_option(provider.api_key_env_var.as_deref()), + ) + }) + .collect::>() + .join("\n"); + + let metadata_match_arms = variants + .iter() + .enumerate() + .map(|(index, variant)| { + format!(" LlmProvider::{variant} => &Self::METADATA[{index}],") + }) + .collect::>() + .join("\n"); + + format!( + r#"// @generated by crates/core/build.rs from litellm-rust/providers/providers.json. +// Do not edit this file by hand. + +use std::fmt; +use std::str::FromStr; + +use crate::error::CoreError; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProviderMetadata {{ + pub provider: LlmProvider, + pub routing_name: &'static str, + pub display_name: &'static str, + pub default_api_base: Option<&'static str>, + pub api_key_env_var: Option<&'static str>, +}} + +#[allow(clippy::enum_variant_names)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LlmProvider {{ +{enum_variants} +}} + +impl LlmProvider {{ + pub const ALL: [LlmProvider; {provider_count}] = [ +{all_values} + ]; + + pub const METADATA: [ProviderMetadata; {provider_count}] = [ +{metadata_values} + ]; + + pub fn metadata(self) -> &'static ProviderMetadata {{ + match self {{ +{metadata_match_arms} + }} + }} + + pub fn as_str(self) -> &'static str {{ + self.metadata().routing_name + }} + + pub fn display_name(self) -> &'static str {{ + self.metadata().display_name + }} + + pub fn default_api_base(self) -> Option<&'static str> {{ + self.metadata().default_api_base + }} + + pub fn api_key_env_var(self) -> Option<&'static str> {{ + self.metadata().api_key_env_var + }} +}} + +impl fmt::Display for LlmProvider {{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {{ + f.write_str(self.as_str()) + }} +}} + +impl FromStr for LlmProvider {{ + type Err = CoreError; + + fn from_str(value: &str) -> Result {{ + LlmProvider::ALL + .iter() + .copied() + .find(|provider| provider.as_str() == value) + .ok_or_else(|| CoreError::InvalidProvider(value.to_string())) + }} +}} + +#[cfg(test)] +mod tests {{ + use super::*; + + #[derive(serde::Deserialize)] + struct RegistryProvider {{ + routing_name: String, + }} + + fn registry_provider_values() -> Vec {{ + let raw = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../providers/providers.json" + )); + serde_json::from_str::>(raw) + .expect("providers.json parses") + .into_iter() + .map(|provider| provider.routing_name) + .collect() + }} + + fn python_provider_values() -> Vec {{ + let source = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/types/utils.py" + )); + let mut values = Vec::new(); + let mut in_llm_providers = false; + + for line in source.lines() {{ + if line.starts_with("class LlmProviders(") {{ + in_llm_providers = true; + continue; + }} + if in_llm_providers && line.starts_with("class ") {{ + break; + }} + if !in_llm_providers {{ + continue; + }} + + let trimmed = line.trim(); + let Some((name, value_expr)) = trimmed.split_once('=') else {{ + continue; + }}; + if !name + .trim() + .chars() + .all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_') + {{ + continue; + }} + let Some(value) = value_expr.split('"').nth(1) else {{ + continue; + }}; + values.push(value.to_string()); + }} + + assert!(!values.is_empty(), "parsed Python LlmProviders values"); + values + }} + + #[test] + fn provider_values_match_registry_order() {{ + let registry_values = registry_provider_values(); + assert_eq!(LlmProvider::ALL.len(), registry_values.len()); + assert_eq!( + LlmProvider::ALL + .iter() + .map(|provider| provider.as_str().to_string()) + .collect::>(), + registry_values + ); + }} + + #[test] + fn provider_registry_matches_python_llm_providers() {{ + assert_eq!(registry_provider_values(), python_provider_values()); + }} + + #[test] + fn from_str_round_trips_all_providers() {{ + for provider in LlmProvider::ALL {{ + assert_eq!(LlmProvider::from_str(provider.as_str()), Ok(provider)); + assert_eq!(provider.to_string(), provider.as_str()); + }} + }} + + #[test] + fn from_str_rejects_unknown_provider() {{ + assert_eq!( + LlmProvider::from_str("not-a-provider"), + Err(CoreError::InvalidProvider("not-a-provider".to_string())) + ); + }} + + #[test] + fn provider_metadata_exposes_optional_defaults() {{ + assert_eq!(LlmProvider::Mistral.display_name(), "Mistral"); + assert_eq!( + LlmProvider::Mistral.default_api_base(), + Some("https://api.mistral.ai/v1") + ); + assert_eq!(LlmProvider::Mistral.api_key_env_var(), Some("MISTRAL_API_KEY")); + }} +}} +"#, + enum_variants = enum_variants, + provider_count = providers.len(), + all_values = all_values, + metadata_values = metadata_values, + metadata_match_arms = metadata_match_arms, + ) +} + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let providers_path = manifest_dir.join("../../providers/providers.json"); + println!("cargo:rerun-if-changed={}", providers_path.display()); + println!( + "cargo:rerun-if-changed={}", + manifest_dir.join("build.rs").display() + ); + + let raw = fs::read_to_string(&providers_path).unwrap_or_else(|err| { + panic!( + "failed to read provider registry {}: {err}", + providers_path.display() + ) + }); + let providers: Vec = serde_json::from_str(&raw) + .unwrap_or_else(|err| panic!("failed to parse {}: {err}", providers_path.display())); + validate_providers(&providers); + + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + fs::write( + out_dir.join("provider_generated.rs"), + generate_provider_code(&providers), + ) + .expect("failed to write generated provider code"); +} diff --git a/litellm-rust/crates/core/src/providers/provider.rs b/litellm-rust/crates/core/src/providers/provider.rs index f14152aae18..c0a511d10bb 100644 --- a/litellm-rust/crates/core/src/providers/provider.rs +++ b/litellm-rust/crates/core/src/providers/provider.rs @@ -1,659 +1 @@ -use std::fmt; -use std::str::FromStr; - -use crate::error::CoreError; - -#[allow(clippy::enum_variant_names)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum LlmProvider { - Openai, - Chatgpt, - OpenaiLike, - JinaAi, - Xai, - Zai, - CustomOpenai, - TextCompletionOpenai, - Cohere, - CohereChat, - Clarifai, - Anthropic, - AnthropicText, - Bytez, - Replicate, - Reducto, - Runwayml, - AwsPolly, - Huggingface, - TogetherAi, - Openrouter, - Datarobot, - VertexAi, - VertexAiBeta, - Gemini, - Ai21, - Baseten, - BlackForestLabs, - Azure, - AzureText, - AzureAi, - Sagemaker, - SagemakerChat, - SagemakerNova, - Bedrock, - Vllm, - NlpCloud, - Petals, - Oobabooga, - Ollama, - OllamaChat, - Deepinfra, - Perplexity, - Mistral, - Milvus, - Groq, - A2a, - Gigachat, - NvidiaNim, - NvidiaRiva, - Soniox, - Cerebras, - Ai21Chat, - Volcengine, - Codestral, - TextCompletionCodestral, - Dashscope, - Modelscope, - Moonshot, - Publicai, - V0, - Morph, - LambdaAi, - Inception, - TextCompletionInception, - Deepseek, - Sambanova, - Maritalk, - Voyage, - Cloudflare, - Xinference, - FireworksAi, - Friendliai, - FeatherlessAi, - Watsonx, - WatsonxText, - Triton, - Predibase, - Databricks, - Empower, - Github, - Ragflow, - Compactifai, - DockerModelRunner, - Custom, - LitellmProxy, - HostedVllm, - Llamafile, - LmStudio, - Galadriel, - Nebius, - Infinity, - Deepgram, - Elevenlabs, - Novita, - AiohttpOpenai, - Langfuse, - Humanloop, - Topaz, - Sap, - Assemblyai, - CharityEngine, - GithubCopilot, - Snowflake, - GradientAi, - MetaLlama, - Nscale, - PgVector, - S3Vectors, - Helicone, - Hyperbolic, - Recraft, - FalAi, - Stability, - Heroku, - Aiml, - Cometapi, - Oci, - AutoRouter, - VercelAiGateway, - Dotprompt, - Manus, - Wandb, - Ovhcloud, - Scaleway, - Lemonade, - AmazonNova, - A2aAgent, - Langgraph, - Langflow, - Minimax, - Synthetic, - Apertis, - NanoGpt, - Poe, - Chutes, - Neosantara, - Parasail, - XiaomiMimo, - Tensormesh, - Libertai, - Pinstripes, - Darkbloom, - LitellmAgent, - Cursor, - BedrockMantle, -} - -impl LlmProvider { - #[allow(clippy::too_many_lines)] - pub fn as_str(self) -> &'static str { - match self { - LlmProvider::Openai => "openai", - LlmProvider::Chatgpt => "chatgpt", - LlmProvider::OpenaiLike => "openai_like", - LlmProvider::JinaAi => "jina_ai", - LlmProvider::Xai => "xai", - LlmProvider::Zai => "zai", - LlmProvider::CustomOpenai => "custom_openai", - LlmProvider::TextCompletionOpenai => "text-completion-openai", - LlmProvider::Cohere => "cohere", - LlmProvider::CohereChat => "cohere_chat", - LlmProvider::Clarifai => "clarifai", - LlmProvider::Anthropic => "anthropic", - LlmProvider::AnthropicText => "anthropic_text", - LlmProvider::Bytez => "bytez", - LlmProvider::Replicate => "replicate", - LlmProvider::Reducto => "reducto", - LlmProvider::Runwayml => "runwayml", - LlmProvider::AwsPolly => "aws_polly", - LlmProvider::Huggingface => "huggingface", - LlmProvider::TogetherAi => "together_ai", - LlmProvider::Openrouter => "openrouter", - LlmProvider::Datarobot => "datarobot", - LlmProvider::VertexAi => "vertex_ai", - LlmProvider::VertexAiBeta => "vertex_ai_beta", - LlmProvider::Gemini => "gemini", - LlmProvider::Ai21 => "ai21", - LlmProvider::Baseten => "baseten", - LlmProvider::BlackForestLabs => "black_forest_labs", - LlmProvider::Azure => "azure", - LlmProvider::AzureText => "azure_text", - LlmProvider::AzureAi => "azure_ai", - LlmProvider::Sagemaker => "sagemaker", - LlmProvider::SagemakerChat => "sagemaker_chat", - LlmProvider::SagemakerNova => "sagemaker_nova", - LlmProvider::Bedrock => "bedrock", - LlmProvider::Vllm => "vllm", - LlmProvider::NlpCloud => "nlp_cloud", - LlmProvider::Petals => "petals", - LlmProvider::Oobabooga => "oobabooga", - LlmProvider::Ollama => "ollama", - LlmProvider::OllamaChat => "ollama_chat", - LlmProvider::Deepinfra => "deepinfra", - LlmProvider::Perplexity => "perplexity", - LlmProvider::Mistral => "mistral", - LlmProvider::Milvus => "milvus", - LlmProvider::Groq => "groq", - LlmProvider::A2a => "a2a", - LlmProvider::Gigachat => "gigachat", - LlmProvider::NvidiaNim => "nvidia_nim", - LlmProvider::NvidiaRiva => "nvidia_riva", - LlmProvider::Soniox => "soniox", - LlmProvider::Cerebras => "cerebras", - LlmProvider::Ai21Chat => "ai21_chat", - LlmProvider::Volcengine => "volcengine", - LlmProvider::Codestral => "codestral", - LlmProvider::TextCompletionCodestral => "text-completion-codestral", - LlmProvider::Dashscope => "dashscope", - LlmProvider::Modelscope => "modelscope", - LlmProvider::Moonshot => "moonshot", - LlmProvider::Publicai => "publicai", - LlmProvider::V0 => "v0", - LlmProvider::Morph => "morph", - LlmProvider::LambdaAi => "lambda_ai", - LlmProvider::Inception => "inception", - LlmProvider::TextCompletionInception => "text-completion-inception", - LlmProvider::Deepseek => "deepseek", - LlmProvider::Sambanova => "sambanova", - LlmProvider::Maritalk => "maritalk", - LlmProvider::Voyage => "voyage", - LlmProvider::Cloudflare => "cloudflare", - LlmProvider::Xinference => "xinference", - LlmProvider::FireworksAi => "fireworks_ai", - LlmProvider::Friendliai => "friendliai", - LlmProvider::FeatherlessAi => "featherless_ai", - LlmProvider::Watsonx => "watsonx", - LlmProvider::WatsonxText => "watsonx_text", - LlmProvider::Triton => "triton", - LlmProvider::Predibase => "predibase", - LlmProvider::Databricks => "databricks", - LlmProvider::Empower => "empower", - LlmProvider::Github => "github", - LlmProvider::Ragflow => "ragflow", - LlmProvider::Compactifai => "compactifai", - LlmProvider::DockerModelRunner => "docker_model_runner", - LlmProvider::Custom => "custom", - LlmProvider::LitellmProxy => "litellm_proxy", - LlmProvider::HostedVllm => "hosted_vllm", - LlmProvider::Llamafile => "llamafile", - LlmProvider::LmStudio => "lm_studio", - LlmProvider::Galadriel => "galadriel", - LlmProvider::Nebius => "nebius", - LlmProvider::Infinity => "infinity", - LlmProvider::Deepgram => "deepgram", - LlmProvider::Elevenlabs => "elevenlabs", - LlmProvider::Novita => "novita", - LlmProvider::AiohttpOpenai => "aiohttp_openai", - LlmProvider::Langfuse => "langfuse", - LlmProvider::Humanloop => "humanloop", - LlmProvider::Topaz => "topaz", - LlmProvider::Sap => "sap", - LlmProvider::Assemblyai => "assemblyai", - LlmProvider::CharityEngine => "charity_engine", - LlmProvider::GithubCopilot => "github_copilot", - LlmProvider::Snowflake => "snowflake", - LlmProvider::GradientAi => "gradient_ai", - LlmProvider::MetaLlama => "meta_llama", - LlmProvider::Nscale => "nscale", - LlmProvider::PgVector => "pg_vector", - LlmProvider::S3Vectors => "s3_vectors", - LlmProvider::Helicone => "helicone", - LlmProvider::Hyperbolic => "hyperbolic", - LlmProvider::Recraft => "recraft", - LlmProvider::FalAi => "fal_ai", - LlmProvider::Stability => "stability", - LlmProvider::Heroku => "heroku", - LlmProvider::Aiml => "aiml", - LlmProvider::Cometapi => "cometapi", - LlmProvider::Oci => "oci", - LlmProvider::AutoRouter => "auto_router", - LlmProvider::VercelAiGateway => "vercel_ai_gateway", - LlmProvider::Dotprompt => "dotprompt", - LlmProvider::Manus => "manus", - LlmProvider::Wandb => "wandb", - LlmProvider::Ovhcloud => "ovhcloud", - LlmProvider::Scaleway => "scaleway", - LlmProvider::Lemonade => "lemonade", - LlmProvider::AmazonNova => "amazon_nova", - LlmProvider::A2aAgent => "a2a_agent", - LlmProvider::Langgraph => "langgraph", - LlmProvider::Langflow => "langflow", - LlmProvider::Minimax => "minimax", - LlmProvider::Synthetic => "synthetic", - LlmProvider::Apertis => "apertis", - LlmProvider::NanoGpt => "nano-gpt", - LlmProvider::Poe => "poe", - LlmProvider::Chutes => "chutes", - LlmProvider::Neosantara => "neosantara", - LlmProvider::Parasail => "parasail", - LlmProvider::XiaomiMimo => "xiaomi_mimo", - LlmProvider::Tensormesh => "tensormesh", - LlmProvider::Libertai => "libertai", - LlmProvider::Pinstripes => "pinstripes", - LlmProvider::Darkbloom => "darkbloom", - LlmProvider::LitellmAgent => "litellm_agent", - LlmProvider::Cursor => "cursor", - LlmProvider::BedrockMantle => "bedrock_mantle", - } - } - - #[allow(clippy::too_many_lines)] - pub const ALL: [LlmProvider; 146] = [ - LlmProvider::Openai, - LlmProvider::Chatgpt, - LlmProvider::OpenaiLike, - LlmProvider::JinaAi, - LlmProvider::Xai, - LlmProvider::Zai, - LlmProvider::CustomOpenai, - LlmProvider::TextCompletionOpenai, - LlmProvider::Cohere, - LlmProvider::CohereChat, - LlmProvider::Clarifai, - LlmProvider::Anthropic, - LlmProvider::AnthropicText, - LlmProvider::Bytez, - LlmProvider::Replicate, - LlmProvider::Reducto, - LlmProvider::Runwayml, - LlmProvider::AwsPolly, - LlmProvider::Huggingface, - LlmProvider::TogetherAi, - LlmProvider::Openrouter, - LlmProvider::Datarobot, - LlmProvider::VertexAi, - LlmProvider::VertexAiBeta, - LlmProvider::Gemini, - LlmProvider::Ai21, - LlmProvider::Baseten, - LlmProvider::BlackForestLabs, - LlmProvider::Azure, - LlmProvider::AzureText, - LlmProvider::AzureAi, - LlmProvider::Sagemaker, - LlmProvider::SagemakerChat, - LlmProvider::SagemakerNova, - LlmProvider::Bedrock, - LlmProvider::Vllm, - LlmProvider::NlpCloud, - LlmProvider::Petals, - LlmProvider::Oobabooga, - LlmProvider::Ollama, - LlmProvider::OllamaChat, - LlmProvider::Deepinfra, - LlmProvider::Perplexity, - LlmProvider::Mistral, - LlmProvider::Milvus, - LlmProvider::Groq, - LlmProvider::A2a, - LlmProvider::Gigachat, - LlmProvider::NvidiaNim, - LlmProvider::NvidiaRiva, - LlmProvider::Soniox, - LlmProvider::Cerebras, - LlmProvider::Ai21Chat, - LlmProvider::Volcengine, - LlmProvider::Codestral, - LlmProvider::TextCompletionCodestral, - LlmProvider::Dashscope, - LlmProvider::Modelscope, - LlmProvider::Moonshot, - LlmProvider::Publicai, - LlmProvider::V0, - LlmProvider::Morph, - LlmProvider::LambdaAi, - LlmProvider::Inception, - LlmProvider::TextCompletionInception, - LlmProvider::Deepseek, - LlmProvider::Sambanova, - LlmProvider::Maritalk, - LlmProvider::Voyage, - LlmProvider::Cloudflare, - LlmProvider::Xinference, - LlmProvider::FireworksAi, - LlmProvider::Friendliai, - LlmProvider::FeatherlessAi, - LlmProvider::Watsonx, - LlmProvider::WatsonxText, - LlmProvider::Triton, - LlmProvider::Predibase, - LlmProvider::Databricks, - LlmProvider::Empower, - LlmProvider::Github, - LlmProvider::Ragflow, - LlmProvider::Compactifai, - LlmProvider::DockerModelRunner, - LlmProvider::Custom, - LlmProvider::LitellmProxy, - LlmProvider::HostedVllm, - LlmProvider::Llamafile, - LlmProvider::LmStudio, - LlmProvider::Galadriel, - LlmProvider::Nebius, - LlmProvider::Infinity, - LlmProvider::Deepgram, - LlmProvider::Elevenlabs, - LlmProvider::Novita, - LlmProvider::AiohttpOpenai, - LlmProvider::Langfuse, - LlmProvider::Humanloop, - LlmProvider::Topaz, - LlmProvider::Sap, - LlmProvider::Assemblyai, - LlmProvider::CharityEngine, - LlmProvider::GithubCopilot, - LlmProvider::Snowflake, - LlmProvider::GradientAi, - LlmProvider::MetaLlama, - LlmProvider::Nscale, - LlmProvider::PgVector, - LlmProvider::S3Vectors, - LlmProvider::Helicone, - LlmProvider::Hyperbolic, - LlmProvider::Recraft, - LlmProvider::FalAi, - LlmProvider::Stability, - LlmProvider::Heroku, - LlmProvider::Aiml, - LlmProvider::Cometapi, - LlmProvider::Oci, - LlmProvider::AutoRouter, - LlmProvider::VercelAiGateway, - LlmProvider::Dotprompt, - LlmProvider::Manus, - LlmProvider::Wandb, - LlmProvider::Ovhcloud, - LlmProvider::Scaleway, - LlmProvider::Lemonade, - LlmProvider::AmazonNova, - LlmProvider::A2aAgent, - LlmProvider::Langgraph, - LlmProvider::Langflow, - LlmProvider::Minimax, - LlmProvider::Synthetic, - LlmProvider::Apertis, - LlmProvider::NanoGpt, - LlmProvider::Poe, - LlmProvider::Chutes, - LlmProvider::Neosantara, - LlmProvider::Parasail, - LlmProvider::XiaomiMimo, - LlmProvider::Tensormesh, - LlmProvider::Libertai, - LlmProvider::Pinstripes, - LlmProvider::Darkbloom, - LlmProvider::LitellmAgent, - LlmProvider::Cursor, - LlmProvider::BedrockMantle, - ]; -} - -impl fmt::Display for LlmProvider { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -impl FromStr for LlmProvider { - type Err = CoreError; - - fn from_str(value: &str) -> Result { - LlmProvider::ALL - .iter() - .copied() - .find(|provider| provider.as_str() == value) - .ok_or_else(|| CoreError::InvalidProvider(value.to_string())) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const PYTHON_LLM_PROVIDER_VALUES: &[&str] = &[ - "openai", - "chatgpt", - "openai_like", - "jina_ai", - "xai", - "zai", - "custom_openai", - "text-completion-openai", - "cohere", - "cohere_chat", - "clarifai", - "anthropic", - "anthropic_text", - "bytez", - "replicate", - "reducto", - "runwayml", - "aws_polly", - "huggingface", - "together_ai", - "openrouter", - "datarobot", - "vertex_ai", - "vertex_ai_beta", - "gemini", - "ai21", - "baseten", - "black_forest_labs", - "azure", - "azure_text", - "azure_ai", - "sagemaker", - "sagemaker_chat", - "sagemaker_nova", - "bedrock", - "vllm", - "nlp_cloud", - "petals", - "oobabooga", - "ollama", - "ollama_chat", - "deepinfra", - "perplexity", - "mistral", - "milvus", - "groq", - "a2a", - "gigachat", - "nvidia_nim", - "nvidia_riva", - "soniox", - "cerebras", - "ai21_chat", - "volcengine", - "codestral", - "text-completion-codestral", - "dashscope", - "modelscope", - "moonshot", - "publicai", - "v0", - "morph", - "lambda_ai", - "inception", - "text-completion-inception", - "deepseek", - "sambanova", - "maritalk", - "voyage", - "cloudflare", - "xinference", - "fireworks_ai", - "friendliai", - "featherless_ai", - "watsonx", - "watsonx_text", - "triton", - "predibase", - "databricks", - "empower", - "github", - "ragflow", - "compactifai", - "docker_model_runner", - "custom", - "litellm_proxy", - "hosted_vllm", - "llamafile", - "lm_studio", - "galadriel", - "nebius", - "infinity", - "deepgram", - "elevenlabs", - "novita", - "aiohttp_openai", - "langfuse", - "humanloop", - "topaz", - "sap", - "assemblyai", - "charity_engine", - "github_copilot", - "snowflake", - "gradient_ai", - "meta_llama", - "nscale", - "pg_vector", - "s3_vectors", - "helicone", - "hyperbolic", - "recraft", - "fal_ai", - "stability", - "heroku", - "aiml", - "cometapi", - "oci", - "auto_router", - "vercel_ai_gateway", - "dotprompt", - "manus", - "wandb", - "ovhcloud", - "scaleway", - "lemonade", - "amazon_nova", - "a2a_agent", - "langgraph", - "langflow", - "minimax", - "synthetic", - "apertis", - "nano-gpt", - "poe", - "chutes", - "neosantara", - "parasail", - "xiaomi_mimo", - "tensormesh", - "libertai", - "pinstripes", - "darkbloom", - "litellm_agent", - "cursor", - "bedrock_mantle", - ]; - - #[test] - fn provider_values_match_python_enum_order() { - assert_eq!(LlmProvider::ALL.len(), 146); - assert_eq!( - LlmProvider::ALL - .iter() - .map(|provider| provider.as_str()) - .collect::>(), - PYTHON_LLM_PROVIDER_VALUES - ); - } - - #[test] - fn from_str_round_trips_all_providers() { - for provider in LlmProvider::ALL { - assert_eq!(LlmProvider::from_str(provider.as_str()), Ok(provider)); - assert_eq!(provider.to_string(), provider.as_str()); - } - } - - #[test] - fn from_str_rejects_unknown_provider() { - assert_eq!( - LlmProvider::from_str("not-a-provider"), - Err(CoreError::InvalidProvider("not-a-provider".to_string())) - ); - } -} +include!(concat!(env!("OUT_DIR"), "/provider_generated.rs")); diff --git a/litellm-rust/providers/README.md b/litellm-rust/providers/README.md new file mode 100644 index 00000000000..0191656373e --- /dev/null +++ b/litellm-rust/providers/README.md @@ -0,0 +1,21 @@ +# Rust Provider Registry + +`providers.json` is the source of truth for provider identity in `litellm-rust`. +`crates/core/build.rs` reads it at compile time and generates the typed +`LlmProvider` enum plus static provider metadata. Runtime code does not parse +this JSON. + +To add a provider: + +- Add a `providers.json` entry with `routing_name` matching Python + `LlmProviders.value` in `litellm/types/utils.py`. +- Set `display_name` to the human-readable provider name for docs/errors. +- Set `default_api_base` to a stable provider-level default base URL, or `null` + when it is unknown, dynamic, or route-specific. +- Set `api_key_env_var` to the canonical LiteLLM env var, or `null` when there + is no single provider-level key. +- Put request/response logic under + `crates/providers/src///transformation.rs`; do not put + transforms, signing logic, or secrets in this registry. +- Run `cargo test -p litellm-core --locked`; it verifies the Rust registry stays + in parity with Python `LlmProviders`. diff --git a/litellm-rust/providers/providers.json b/litellm-rust/providers/providers.json new file mode 100644 index 00000000000..60e2c3e8bc4 --- /dev/null +++ b/litellm-rust/providers/providers.json @@ -0,0 +1,878 @@ +[ + { + "routing_name": "openai", + "display_name": "OpenAI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "chatgpt", + "display_name": "ChatGPT", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "openai_like", + "display_name": "OpenAI Like", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "jina_ai", + "display_name": "Jina AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "xai", + "display_name": "xAI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "zai", + "display_name": "Z.ai", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "custom_openai", + "display_name": "Custom OpenAI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "text-completion-openai", + "display_name": "Text Completion OpenAI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "cohere", + "display_name": "Cohere", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "cohere_chat", + "display_name": "Cohere Chat", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "clarifai", + "display_name": "Clarifai", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "anthropic", + "display_name": "Anthropic", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "anthropic_text", + "display_name": "Anthropic Text", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "bytez", + "display_name": "Bytez", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "replicate", + "display_name": "Replicate", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "reducto", + "display_name": "Reducto", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "runwayml", + "display_name": "Runwayml", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "aws_polly", + "display_name": "AWS Polly", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "huggingface", + "display_name": "Hugging Face", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "together_ai", + "display_name": "Together AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "openrouter", + "display_name": "Openrouter", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "datarobot", + "display_name": "Datarobot", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "vertex_ai", + "display_name": "Vertex AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "vertex_ai_beta", + "display_name": "Vertex AI Beta", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "gemini", + "display_name": "Gemini", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "ai21", + "display_name": "AI21", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "baseten", + "display_name": "Baseten", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "black_forest_labs", + "display_name": "Black Forest Labs", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "azure", + "display_name": "Azure", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "azure_text", + "display_name": "Azure Text", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "azure_ai", + "display_name": "Azure AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "sagemaker", + "display_name": "Sagemaker", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "sagemaker_chat", + "display_name": "Sagemaker Chat", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "sagemaker_nova", + "display_name": "Sagemaker Nova", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "bedrock", + "display_name": "Bedrock", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "vllm", + "display_name": "vLLM", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "nlp_cloud", + "display_name": "NLP Cloud", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "petals", + "display_name": "Petals", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "oobabooga", + "display_name": "Oobabooga", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "ollama", + "display_name": "Ollama", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "ollama_chat", + "display_name": "Ollama Chat", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "deepinfra", + "display_name": "Deepinfra", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "perplexity", + "display_name": "Perplexity", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "mistral", + "display_name": "Mistral", + "default_api_base": "https://api.mistral.ai/v1", + "api_key_env_var": "MISTRAL_API_KEY" + }, + { + "routing_name": "milvus", + "display_name": "Milvus", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "groq", + "display_name": "Groq", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "a2a", + "display_name": "A2A", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "gigachat", + "display_name": "Gigachat", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "nvidia_nim", + "display_name": "NVIDIA NIM", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "nvidia_riva", + "display_name": "NVIDIA Riva", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "soniox", + "display_name": "Soniox", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "cerebras", + "display_name": "Cerebras", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "ai21_chat", + "display_name": "AI21 Chat", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "volcengine", + "display_name": "Volcengine", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "codestral", + "display_name": "Codestral", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "text-completion-codestral", + "display_name": "Text Completion Codestral", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "dashscope", + "display_name": "Dashscope", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "modelscope", + "display_name": "Modelscope", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "moonshot", + "display_name": "Moonshot", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "publicai", + "display_name": "Publicai", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "v0", + "display_name": "v0", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "morph", + "display_name": "Morph", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "lambda_ai", + "display_name": "Lambda AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "inception", + "display_name": "Inception", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "text-completion-inception", + "display_name": "Text Completion Inception", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "deepseek", + "display_name": "Deepseek", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "sambanova", + "display_name": "Sambanova", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "maritalk", + "display_name": "Maritalk", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "voyage", + "display_name": "Voyage", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "cloudflare", + "display_name": "Cloudflare", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "xinference", + "display_name": "Xinference", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "fireworks_ai", + "display_name": "Fireworks AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "friendliai", + "display_name": "Friendliai", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "featherless_ai", + "display_name": "Featherless AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "watsonx", + "display_name": "Watsonx", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "watsonx_text", + "display_name": "Watsonx Text", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "triton", + "display_name": "Triton", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "predibase", + "display_name": "Predibase", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "databricks", + "display_name": "Databricks", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "empower", + "display_name": "Empower", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "github", + "display_name": "GitHub", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "ragflow", + "display_name": "Ragflow", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "compactifai", + "display_name": "Compactifai", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "docker_model_runner", + "display_name": "Docker Model Runner", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "custom", + "display_name": "Custom", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "litellm_proxy", + "display_name": "LiteLLM Proxy", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "hosted_vllm", + "display_name": "Hosted vLLM", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "llamafile", + "display_name": "Llamafile", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "lm_studio", + "display_name": "LM Studio", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "galadriel", + "display_name": "Galadriel", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "nebius", + "display_name": "Nebius", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "infinity", + "display_name": "Infinity", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "deepgram", + "display_name": "Deepgram", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "elevenlabs", + "display_name": "Elevenlabs", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "novita", + "display_name": "Novita", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "aiohttp_openai", + "display_name": "AIOHTTP OpenAI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "langfuse", + "display_name": "Langfuse", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "humanloop", + "display_name": "Humanloop", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "topaz", + "display_name": "Topaz", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "sap", + "display_name": "Sap Generative AI Hub", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "assemblyai", + "display_name": "Assemblyai", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "charity_engine", + "display_name": "Charity Engine", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "github_copilot", + "display_name": "GitHub Copilot", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "snowflake", + "display_name": "Snowflake", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "gradient_ai", + "display_name": "Gradient AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "meta_llama", + "display_name": "Llama", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "nscale", + "display_name": "Nscale", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "pg_vector", + "display_name": "Pg Vector", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "s3_vectors", + "display_name": "S3 Vectors", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "helicone", + "display_name": "Helicone", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "hyperbolic", + "display_name": "Hyperbolic", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "recraft", + "display_name": "Recraft", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "fal_ai", + "display_name": "Fal AI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "stability", + "display_name": "Stability", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "heroku", + "display_name": "Heroku", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "aiml", + "display_name": "Aiml", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "cometapi", + "display_name": "Cometapi", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "oci", + "display_name": "OCI", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "auto_router", + "display_name": "Auto Router", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "vercel_ai_gateway", + "display_name": "Vercel AI Gateway", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "dotprompt", + "display_name": "Dotprompt", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "manus", + "display_name": "Manus", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "wandb", + "display_name": "Wandb", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "ovhcloud", + "display_name": "OVHcloud", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "scaleway", + "display_name": "Scaleway", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "lemonade", + "display_name": "Lemonade", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "amazon_nova", + "display_name": "Amazon Nova", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "a2a_agent", + "display_name": "A2A Agent", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "langgraph", + "display_name": "Langgraph", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "langflow", + "display_name": "Langflow", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "minimax", + "display_name": "Minimax", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "synthetic", + "display_name": "Synthetic", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "apertis", + "display_name": "Apertis", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "nano-gpt", + "display_name": "NanoGPT", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "poe", + "display_name": "Poe", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "chutes", + "display_name": "Chutes", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "neosantara", + "display_name": "Neosantara", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "parasail", + "display_name": "Parasail", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "xiaomi_mimo", + "display_name": "Xiaomi MiMo", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "tensormesh", + "display_name": "Tensormesh", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "libertai", + "display_name": "Libertai", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "pinstripes", + "display_name": "Pinstripes", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "darkbloom", + "display_name": "Darkbloom", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "litellm_agent", + "display_name": "LiteLLM Agent", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "cursor", + "display_name": "Cursor", + "default_api_base": null, + "api_key_env_var": null + }, + { + "routing_name": "bedrock_mantle", + "display_name": "Bedrock Mantle", + "default_api_base": null, + "api_key_env_var": null + } +] From 3ecd1202f27d65bcc0366b345d77ac58a1ca935c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 16:34:48 -0700 Subject: [PATCH 21/29] chore: move rust provider registry into core --- litellm-rust/ADDING_A_PROVIDER.md | 2 +- litellm-rust/crates/core/build.rs | 8 ++++---- litellm-rust/{ => crates/core/src}/providers/README.md | 4 ++-- .../core/src/providers/provider_registry.json} | 0 4 files changed, 7 insertions(+), 7 deletions(-) rename litellm-rust/{ => crates/core/src}/providers/README.md (84%) rename litellm-rust/{providers/providers.json => crates/core/src/providers/provider_registry.json} (100%) diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index ac454727c8a..48f8f75dceb 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -1,7 +1,7 @@ # Adding a provider / route to litellm-rust - Keep the route contract pure in `crates/core/src//`: define the typed request/response structs and a provider config trait with no network, env, auth, or logging. -- Add provider identity to `providers/providers.json`: `routing_name` must match Python `LlmProviders.value`; `display_name` is human-facing; `default_api_base` and `api_key_env_var` are optional provider defaults. +- Add provider identity to `crates/core/src/providers/provider_registry.json`: `routing_name` must match Python `LlmProviders.value`; `display_name` is human-facing; `default_api_base` and `api_key_env_var` are optional provider defaults. - Put provider-specific transforms in `crates/providers/src///transformation.rs`, mirroring the Python provider tree and exposing a `const __CONFIG`. - The provider config owns three pure steps: map LiteLLM params, transform the LiteLLM request into the provider request, and transform the provider response back into the LiteLLM response. - If the provider has a reverse or normalization step, keep it pure and explicit next to the transforms; do not hide reverse mapping inside the HTTP transport. diff --git a/litellm-rust/crates/core/build.rs b/litellm-rust/crates/core/build.rs index e31152c2035..dfc415345a1 100644 --- a/litellm-rust/crates/core/build.rs +++ b/litellm-rust/crates/core/build.rs @@ -119,7 +119,7 @@ fn generate_provider_code(providers: &[ProviderMetadataInput]) -> String { .join("\n"); format!( - r#"// @generated by crates/core/build.rs from litellm-rust/providers/providers.json. + r#"// @generated by crates/core/build.rs from litellm-rust/crates/core/src/providers/provider_registry.json. // Do not edit this file by hand. use std::fmt; @@ -204,10 +204,10 @@ mod tests {{ fn registry_provider_values() -> Vec {{ let raw = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/../../providers/providers.json" + "/src/providers/provider_registry.json" )); serde_json::from_str::>(raw) - .expect("providers.json parses") + .expect("provider_registry.json parses") .into_iter() .map(|provider| provider.routing_name) .collect() @@ -309,7 +309,7 @@ mod tests {{ fn main() { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let providers_path = manifest_dir.join("../../providers/providers.json"); + let providers_path = manifest_dir.join("src/providers/provider_registry.json"); println!("cargo:rerun-if-changed={}", providers_path.display()); println!( "cargo:rerun-if-changed={}", diff --git a/litellm-rust/providers/README.md b/litellm-rust/crates/core/src/providers/README.md similarity index 84% rename from litellm-rust/providers/README.md rename to litellm-rust/crates/core/src/providers/README.md index 0191656373e..c8e2b8a2526 100644 --- a/litellm-rust/providers/README.md +++ b/litellm-rust/crates/core/src/providers/README.md @@ -1,13 +1,13 @@ # Rust Provider Registry -`providers.json` is the source of truth for provider identity in `litellm-rust`. +`provider_registry.json` is the source of truth for provider identity in `litellm-rust`. `crates/core/build.rs` reads it at compile time and generates the typed `LlmProvider` enum plus static provider metadata. Runtime code does not parse this JSON. To add a provider: -- Add a `providers.json` entry with `routing_name` matching Python +- Add a `provider_registry.json` entry with `routing_name` matching Python `LlmProviders.value` in `litellm/types/utils.py`. - Set `display_name` to the human-readable provider name for docs/errors. - Set `default_api_base` to a stable provider-level default base URL, or `null` diff --git a/litellm-rust/providers/providers.json b/litellm-rust/crates/core/src/providers/provider_registry.json similarity index 100% rename from litellm-rust/providers/providers.json rename to litellm-rust/crates/core/src/providers/provider_registry.json From 31ef925ffef789a1f938e460ff5720d61148236d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 16:41:53 -0700 Subject: [PATCH 22/29] chore: source rust providers from endpoint registry --- litellm-rust/ADDING_A_PROVIDER.md | 2 +- litellm-rust/crates/core/build.rs | 156 ++-- .../crates/core/src/providers/README.md | 25 +- .../core/src/providers/provider_registry.json | 878 ------------------ .../provider_endpoints_support_backup.json | 11 + provider_endpoints_support.json | 11 + 6 files changed, 123 insertions(+), 960 deletions(-) delete mode 100644 litellm-rust/crates/core/src/providers/provider_registry.json diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index 48f8f75dceb..9555a0c53a3 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -1,7 +1,7 @@ # Adding a provider / route to litellm-rust - Keep the route contract pure in `crates/core/src//`: define the typed request/response structs and a provider config trait with no network, env, auth, or logging. -- Add provider identity to `crates/core/src/providers/provider_registry.json`: `routing_name` must match Python `LlmProviders.value`; `display_name` is human-facing; `default_api_base` and `api_key_env_var` are optional provider defaults. +- Add provider identity to the repo-root `provider_endpoints_support.json`: use the LiteLLM provider slug, display name, docs URL, and endpoint support flags. Put optional stable base URL / API key env var defaults under the top-level `default_creds` map. - Put provider-specific transforms in `crates/providers/src///transformation.rs`, mirroring the Python provider tree and exposing a `const __CONFIG`. - The provider config owns three pure steps: map LiteLLM params, transform the LiteLLM request into the provider request, and transform the provider response back into the LiteLLM response. - If the provider has a reverse or normalization step, keep it pure and explicit next to the transforms; do not hide reverse mapping inside the HTTP transport. diff --git a/litellm-rust/crates/core/build.rs b/litellm-rust/crates/core/build.rs index dfc415345a1..07be1279a0a 100644 --- a/litellm-rust/crates/core/build.rs +++ b/litellm-rust/crates/core/build.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::env; use std::fs; use std::path::PathBuf; @@ -6,9 +6,29 @@ use std::path::PathBuf; use serde::Deserialize; #[derive(Debug, Deserialize)] +struct ProviderEndpointSupportInput { + providers: BTreeMap, + #[serde(default)] + default_creds: BTreeMap, +} + +#[derive(Debug, Deserialize)] +struct ProviderInput { + display_name: String, + url: String, +} + +#[derive(Debug, Deserialize)] +struct ProviderDefaultCredsInput { + default_api_base: Option, + api_key_env_var: Option, +} + +#[derive(Debug)] struct ProviderMetadataInput { routing_name: String, display_name: String, + docs_url: String, default_api_base: Option, api_key_env_var: Option, } @@ -25,7 +45,7 @@ fn rust_option(value: Option<&str>) -> String { fn variant_name(routing_name: &str) -> String { routing_name - .split(['_', '-', '.']) + .split(['_', '-', '.', '/']) .filter(|part| !part.is_empty()) .map(|part| { let mut chars = part.chars(); @@ -42,6 +62,29 @@ fn variant_name(routing_name: &str) -> String { .collect() } +fn provider_metadata_from_endpoint_support( + registry: ProviderEndpointSupportInput, +) -> Vec { + let ProviderEndpointSupportInput { + providers, + default_creds, + } = registry; + + providers + .into_iter() + .map(|(routing_name, provider)| { + let default_creds = default_creds.get(&routing_name); + ProviderMetadataInput { + routing_name, + display_name: provider.display_name, + docs_url: provider.url, + default_api_base: default_creds.and_then(|creds| creds.default_api_base.clone()), + api_key_env_var: default_creds.and_then(|creds| creds.api_key_env_var.clone()), + } + }) + .collect() +} + fn validate_providers(providers: &[ProviderMetadataInput]) { let mut routing_names = HashSet::new(); let mut variants = HashSet::new(); @@ -56,6 +99,9 @@ fn validate_providers(providers: &[ProviderMetadataInput]) { provider.routing_name ); } + if provider.docs_url.trim().is_empty() { + panic!("provider {} has an empty docs_url", provider.routing_name); + } if !routing_names.insert(provider.routing_name.as_str()) { panic!("duplicate provider routing_name: {}", provider.routing_name); } @@ -67,6 +113,12 @@ fn validate_providers(providers: &[ProviderMetadataInput]) { provider.routing_name ); } + if variant.chars().next().is_some_and(|ch| ch.is_ascii_digit()) { + panic!( + "provider {} generated Rust variant {variant} starting with a digit", + provider.routing_name + ); + } if !variants.insert(variant.clone()) { panic!( "provider {} generated duplicate Rust variant {variant}", @@ -99,9 +151,10 @@ fn generate_provider_code(providers: &[ProviderMetadataInput]) -> String { .zip(variants.iter()) .map(|(provider, variant)| { format!( - " ProviderMetadata {{ provider: LlmProvider::{variant}, routing_name: {}, display_name: {}, default_api_base: {}, api_key_env_var: {} }},", + " ProviderMetadata {{ provider: LlmProvider::{variant}, routing_name: {}, display_name: {}, docs_url: {}, default_api_base: {}, api_key_env_var: {} }},", rust_string(&provider.routing_name), rust_string(&provider.display_name), + rust_string(&provider.docs_url), rust_option(provider.default_api_base.as_deref()), rust_option(provider.api_key_env_var.as_deref()), ) @@ -119,7 +172,7 @@ fn generate_provider_code(providers: &[ProviderMetadataInput]) -> String { .join("\n"); format!( - r#"// @generated by crates/core/build.rs from litellm-rust/crates/core/src/providers/provider_registry.json. + r#"// @generated by crates/core/build.rs from provider_endpoints_support.json. // Do not edit this file by hand. use std::fmt; @@ -132,6 +185,7 @@ pub struct ProviderMetadata {{ pub provider: LlmProvider, pub routing_name: &'static str, pub display_name: &'static str, + pub docs_url: &'static str, pub default_api_base: Option<&'static str>, pub api_key_env_var: Option<&'static str>, }} @@ -165,6 +219,10 @@ impl LlmProvider {{ self.metadata().display_name }} + pub fn docs_url(self) -> &'static str {{ + self.metadata().docs_url + }} + pub fn default_api_base(self) -> Option<&'static str> {{ self.metadata().default_api_base }} @@ -196,67 +254,27 @@ impl FromStr for LlmProvider {{ mod tests {{ use super::*; - #[derive(serde::Deserialize)] - struct RegistryProvider {{ - routing_name: String, - }} - - fn registry_provider_values() -> Vec {{ + fn endpoint_support_provider_values() -> Vec {{ let raw = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/src/providers/provider_registry.json" + "/../../../provider_endpoints_support.json" )); - serde_json::from_str::>(raw) - .expect("provider_registry.json parses") - .into_iter() - .map(|provider| provider.routing_name) - .collect() - }} - - fn python_provider_values() -> Vec {{ - let source = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../litellm/types/utils.py" - )); - let mut values = Vec::new(); - let mut in_llm_providers = false; - - for line in source.lines() {{ - if line.starts_with("class LlmProviders(") {{ - in_llm_providers = true; - continue; - }} - if in_llm_providers && line.starts_with("class ") {{ - break; - }} - if !in_llm_providers {{ - continue; - }} - - let trimmed = line.trim(); - let Some((name, value_expr)) = trimmed.split_once('=') else {{ - continue; - }}; - if !name - .trim() - .chars() - .all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_') - {{ - continue; - }} - let Some(value) = value_expr.split('"').nth(1) else {{ - continue; - }}; - values.push(value.to_string()); - }} - - assert!(!values.is_empty(), "parsed Python LlmProviders values"); + let registry = serde_json::from_str::(raw) + .expect("provider_endpoints_support.json parses"); + let mut values = registry + .get("providers") + .and_then(|providers| providers.as_object()) + .expect("provider_endpoints_support.json has providers object") + .keys() + .cloned() + .collect::>(); + values.sort(); values }} #[test] - fn provider_values_match_registry_order() {{ - let registry_values = registry_provider_values(); + fn provider_values_match_endpoint_support_registry() {{ + let registry_values = endpoint_support_provider_values(); assert_eq!(LlmProvider::ALL.len(), registry_values.len()); assert_eq!( LlmProvider::ALL @@ -267,16 +285,12 @@ mod tests {{ ); }} - #[test] - fn provider_registry_matches_python_llm_providers() {{ - assert_eq!(registry_provider_values(), python_provider_values()); - }} - #[test] fn from_str_round_trips_all_providers() {{ for provider in LlmProvider::ALL {{ assert_eq!(LlmProvider::from_str(provider.as_str()), Ok(provider)); assert_eq!(provider.to_string(), provider.as_str()); + assert!(!provider.docs_url().is_empty()); }} }} @@ -290,7 +304,14 @@ mod tests {{ #[test] fn provider_metadata_exposes_optional_defaults() {{ - assert_eq!(LlmProvider::Mistral.display_name(), "Mistral"); + assert_eq!( + LlmProvider::Mistral.display_name(), + "Mistral AI API (`mistral`)" + ); + assert_eq!( + LlmProvider::Mistral.docs_url(), + "https://docs.litellm.ai/docs/providers/mistral" + ); assert_eq!( LlmProvider::Mistral.default_api_base(), Some("https://api.mistral.ai/v1") @@ -309,7 +330,7 @@ mod tests {{ fn main() { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let providers_path = manifest_dir.join("src/providers/provider_registry.json"); + let providers_path = manifest_dir.join("../../../provider_endpoints_support.json"); println!("cargo:rerun-if-changed={}", providers_path.display()); println!( "cargo:rerun-if-changed={}", @@ -318,12 +339,13 @@ fn main() { let raw = fs::read_to_string(&providers_path).unwrap_or_else(|err| { panic!( - "failed to read provider registry {}: {err}", + "failed to read provider endpoint support registry {}: {err}", providers_path.display() ) }); - let providers: Vec = serde_json::from_str(&raw) + let registry: ProviderEndpointSupportInput = serde_json::from_str(&raw) .unwrap_or_else(|err| panic!("failed to parse {}: {err}", providers_path.display())); + let providers = provider_metadata_from_endpoint_support(registry); validate_providers(&providers); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); diff --git a/litellm-rust/crates/core/src/providers/README.md b/litellm-rust/crates/core/src/providers/README.md index c8e2b8a2526..84ef3731f3f 100644 --- a/litellm-rust/crates/core/src/providers/README.md +++ b/litellm-rust/crates/core/src/providers/README.md @@ -1,21 +1,18 @@ -# Rust Provider Registry +# Rust Provider Metadata -`provider_registry.json` is the source of truth for provider identity in `litellm-rust`. -`crates/core/build.rs` reads it at compile time and generates the typed -`LlmProvider` enum plus static provider metadata. Runtime code does not parse -this JSON. +The repo-root `provider_endpoints_support.json` is the shared source of truth +for provider identity and docs metadata in `litellm-rust`. `crates/core/build.rs` +reads it at compile time and generates the typed `LlmProvider` enum plus static +provider metadata. Runtime code does not parse this JSON. To add a provider: -- Add a `provider_registry.json` entry with `routing_name` matching Python - `LlmProviders.value` in `litellm/types/utils.py`. -- Set `display_name` to the human-readable provider name for docs/errors. -- Set `default_api_base` to a stable provider-level default base URL, or `null` - when it is unknown, dynamic, or route-specific. -- Set `api_key_env_var` to the canonical LiteLLM env var, or `null` when there - is no single provider-level key. +- Add a `provider_endpoints_support.json` provider entry using the LiteLLM + provider slug, display name, docs URL, and endpoint support flags. +- Add optional defaults under the top-level `default_creds` map only when there + is a stable provider-level base URL or canonical API key environment variable. - Put request/response logic under `crates/providers/src///transformation.rs`; do not put - transforms, signing logic, or secrets in this registry. + transforms, signing logic, or secrets in provider metadata. - Run `cargo test -p litellm-core --locked`; it verifies the Rust registry stays - in parity with Python `LlmProviders`. + in parity with `provider_endpoints_support.json`. diff --git a/litellm-rust/crates/core/src/providers/provider_registry.json b/litellm-rust/crates/core/src/providers/provider_registry.json deleted file mode 100644 index 60e2c3e8bc4..00000000000 --- a/litellm-rust/crates/core/src/providers/provider_registry.json +++ /dev/null @@ -1,878 +0,0 @@ -[ - { - "routing_name": "openai", - "display_name": "OpenAI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "chatgpt", - "display_name": "ChatGPT", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "openai_like", - "display_name": "OpenAI Like", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "jina_ai", - "display_name": "Jina AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "xai", - "display_name": "xAI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "zai", - "display_name": "Z.ai", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "custom_openai", - "display_name": "Custom OpenAI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "text-completion-openai", - "display_name": "Text Completion OpenAI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "cohere", - "display_name": "Cohere", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "cohere_chat", - "display_name": "Cohere Chat", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "clarifai", - "display_name": "Clarifai", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "anthropic", - "display_name": "Anthropic", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "anthropic_text", - "display_name": "Anthropic Text", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "bytez", - "display_name": "Bytez", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "replicate", - "display_name": "Replicate", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "reducto", - "display_name": "Reducto", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "runwayml", - "display_name": "Runwayml", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "aws_polly", - "display_name": "AWS Polly", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "huggingface", - "display_name": "Hugging Face", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "together_ai", - "display_name": "Together AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "openrouter", - "display_name": "Openrouter", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "datarobot", - "display_name": "Datarobot", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "vertex_ai", - "display_name": "Vertex AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "vertex_ai_beta", - "display_name": "Vertex AI Beta", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "gemini", - "display_name": "Gemini", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "ai21", - "display_name": "AI21", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "baseten", - "display_name": "Baseten", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "black_forest_labs", - "display_name": "Black Forest Labs", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "azure", - "display_name": "Azure", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "azure_text", - "display_name": "Azure Text", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "azure_ai", - "display_name": "Azure AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "sagemaker", - "display_name": "Sagemaker", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "sagemaker_chat", - "display_name": "Sagemaker Chat", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "sagemaker_nova", - "display_name": "Sagemaker Nova", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "bedrock", - "display_name": "Bedrock", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "vllm", - "display_name": "vLLM", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "nlp_cloud", - "display_name": "NLP Cloud", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "petals", - "display_name": "Petals", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "oobabooga", - "display_name": "Oobabooga", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "ollama", - "display_name": "Ollama", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "ollama_chat", - "display_name": "Ollama Chat", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "deepinfra", - "display_name": "Deepinfra", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "perplexity", - "display_name": "Perplexity", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "mistral", - "display_name": "Mistral", - "default_api_base": "https://api.mistral.ai/v1", - "api_key_env_var": "MISTRAL_API_KEY" - }, - { - "routing_name": "milvus", - "display_name": "Milvus", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "groq", - "display_name": "Groq", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "a2a", - "display_name": "A2A", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "gigachat", - "display_name": "Gigachat", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "nvidia_nim", - "display_name": "NVIDIA NIM", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "nvidia_riva", - "display_name": "NVIDIA Riva", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "soniox", - "display_name": "Soniox", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "cerebras", - "display_name": "Cerebras", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "ai21_chat", - "display_name": "AI21 Chat", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "volcengine", - "display_name": "Volcengine", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "codestral", - "display_name": "Codestral", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "text-completion-codestral", - "display_name": "Text Completion Codestral", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "dashscope", - "display_name": "Dashscope", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "modelscope", - "display_name": "Modelscope", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "moonshot", - "display_name": "Moonshot", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "publicai", - "display_name": "Publicai", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "v0", - "display_name": "v0", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "morph", - "display_name": "Morph", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "lambda_ai", - "display_name": "Lambda AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "inception", - "display_name": "Inception", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "text-completion-inception", - "display_name": "Text Completion Inception", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "deepseek", - "display_name": "Deepseek", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "sambanova", - "display_name": "Sambanova", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "maritalk", - "display_name": "Maritalk", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "voyage", - "display_name": "Voyage", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "cloudflare", - "display_name": "Cloudflare", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "xinference", - "display_name": "Xinference", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "fireworks_ai", - "display_name": "Fireworks AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "friendliai", - "display_name": "Friendliai", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "featherless_ai", - "display_name": "Featherless AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "watsonx", - "display_name": "Watsonx", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "watsonx_text", - "display_name": "Watsonx Text", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "triton", - "display_name": "Triton", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "predibase", - "display_name": "Predibase", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "databricks", - "display_name": "Databricks", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "empower", - "display_name": "Empower", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "github", - "display_name": "GitHub", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "ragflow", - "display_name": "Ragflow", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "compactifai", - "display_name": "Compactifai", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "docker_model_runner", - "display_name": "Docker Model Runner", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "custom", - "display_name": "Custom", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "litellm_proxy", - "display_name": "LiteLLM Proxy", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "hosted_vllm", - "display_name": "Hosted vLLM", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "llamafile", - "display_name": "Llamafile", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "lm_studio", - "display_name": "LM Studio", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "galadriel", - "display_name": "Galadriel", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "nebius", - "display_name": "Nebius", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "infinity", - "display_name": "Infinity", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "deepgram", - "display_name": "Deepgram", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "elevenlabs", - "display_name": "Elevenlabs", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "novita", - "display_name": "Novita", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "aiohttp_openai", - "display_name": "AIOHTTP OpenAI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "langfuse", - "display_name": "Langfuse", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "humanloop", - "display_name": "Humanloop", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "topaz", - "display_name": "Topaz", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "sap", - "display_name": "Sap Generative AI Hub", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "assemblyai", - "display_name": "Assemblyai", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "charity_engine", - "display_name": "Charity Engine", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "github_copilot", - "display_name": "GitHub Copilot", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "snowflake", - "display_name": "Snowflake", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "gradient_ai", - "display_name": "Gradient AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "meta_llama", - "display_name": "Llama", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "nscale", - "display_name": "Nscale", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "pg_vector", - "display_name": "Pg Vector", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "s3_vectors", - "display_name": "S3 Vectors", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "helicone", - "display_name": "Helicone", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "hyperbolic", - "display_name": "Hyperbolic", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "recraft", - "display_name": "Recraft", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "fal_ai", - "display_name": "Fal AI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "stability", - "display_name": "Stability", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "heroku", - "display_name": "Heroku", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "aiml", - "display_name": "Aiml", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "cometapi", - "display_name": "Cometapi", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "oci", - "display_name": "OCI", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "auto_router", - "display_name": "Auto Router", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "vercel_ai_gateway", - "display_name": "Vercel AI Gateway", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "dotprompt", - "display_name": "Dotprompt", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "manus", - "display_name": "Manus", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "wandb", - "display_name": "Wandb", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "ovhcloud", - "display_name": "OVHcloud", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "scaleway", - "display_name": "Scaleway", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "lemonade", - "display_name": "Lemonade", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "amazon_nova", - "display_name": "Amazon Nova", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "a2a_agent", - "display_name": "A2A Agent", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "langgraph", - "display_name": "Langgraph", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "langflow", - "display_name": "Langflow", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "minimax", - "display_name": "Minimax", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "synthetic", - "display_name": "Synthetic", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "apertis", - "display_name": "Apertis", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "nano-gpt", - "display_name": "NanoGPT", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "poe", - "display_name": "Poe", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "chutes", - "display_name": "Chutes", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "neosantara", - "display_name": "Neosantara", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "parasail", - "display_name": "Parasail", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "xiaomi_mimo", - "display_name": "Xiaomi MiMo", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "tensormesh", - "display_name": "Tensormesh", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "libertai", - "display_name": "Libertai", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "pinstripes", - "display_name": "Pinstripes", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "darkbloom", - "display_name": "Darkbloom", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "litellm_agent", - "display_name": "LiteLLM Agent", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "cursor", - "display_name": "Cursor", - "default_api_base": null, - "api_key_env_var": null - }, - { - "routing_name": "bedrock_mantle", - "display_name": "Bedrock Mantle", - "default_api_base": null, - "api_key_env_var": null - } -] diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index dd7712aabca..bd539102000 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -31,6 +31,12 @@ } } }, + "default_creds": { + "mistral": { + "default_api_base": "https://api.mistral.ai/v1", + "api_key_env_var": "MISTRAL_API_KEY" + } + }, "providers": { "a2a": { "display_name": "A2A (Agent-to-Agent) (`a2a`)", @@ -179,6 +185,7 @@ }, "apertis": { "display_name": "Apertis (`apertis`)", + "url": "https://docs.litellm.ai/docs/providers/apertis", "endpoints": { "chat_completions": true, "messages": false, @@ -460,6 +467,7 @@ }, "chutes": { "display_name": "Chutes (`chutes`)", + "url": "https://docs.litellm.ai/docs/providers/chutes", "endpoints": { "chat_completions": true, "messages": false, @@ -1470,6 +1478,7 @@ }, "nanogpt": { "display_name": "NanoGPT (`nanogpt`)", + "url": "https://docs.litellm.ai/docs/providers/nanogpt", "endpoints": { "chat_completions": true, "messages": false, @@ -1803,6 +1812,7 @@ }, "poe": { "display_name": "Poe (`poe`)", + "url": "https://docs.litellm.ai/docs/providers/poe", "endpoints": { "chat_completions": true, "messages": false, @@ -2030,6 +2040,7 @@ }, "synthetic": { "display_name": "Synthetic (`synthetic`)", + "url": "https://docs.litellm.ai/docs/providers/synthetic", "endpoints": { "chat_completions": true, "messages": true, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b137ec59a1f..27124b5634d 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -31,6 +31,12 @@ } } }, + "default_creds": { + "mistral": { + "default_api_base": "https://api.mistral.ai/v1", + "api_key_env_var": "MISTRAL_API_KEY" + } + }, "providers": { "a2a": { "display_name": "A2A (Agent-to-Agent) (`a2a`)", @@ -179,6 +185,7 @@ }, "apertis": { "display_name": "Apertis (`apertis`)", + "url": "https://docs.litellm.ai/docs/providers/apertis", "endpoints": { "chat_completions": true, "messages": false, @@ -495,6 +502,7 @@ }, "chutes": { "display_name": "Chutes (`chutes`)", + "url": "https://docs.litellm.ai/docs/providers/chutes", "endpoints": { "chat_completions": true, "messages": false, @@ -1593,6 +1601,7 @@ }, "nanogpt": { "display_name": "NanoGPT (`nanogpt`)", + "url": "https://docs.litellm.ai/docs/providers/nanogpt", "endpoints": { "chat_completions": true, "messages": false, @@ -1993,6 +2002,7 @@ }, "poe": { "display_name": "Poe (`poe`)", + "url": "https://docs.litellm.ai/docs/providers/poe", "endpoints": { "chat_completions": true, "messages": false, @@ -2253,6 +2263,7 @@ }, "synthetic": { "display_name": "Synthetic (`synthetic`)", + "url": "https://docs.litellm.ai/docs/providers/synthetic", "endpoints": { "chat_completions": true, "messages": true, From 7180f79887e1f0639c14b9fcb8b63a6bd4758176 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 16:49:35 -0700 Subject: [PATCH 23/29] fix: satisfy OCR lint budget --- litellm/ocr/main.py | 149 ++++++--------------- tests/test_litellm/ocr/test_rust_bridge.py | 106 +++++++-------- 2 files changed, 94 insertions(+), 161 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index f79c17256ca..f1c8682af99 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -9,7 +9,7 @@ import os import re from dataclasses import dataclass from io import IOBase -from typing import Any, Callable, Coroutine, Dict, Optional, Union, cast +from typing import Any, Callable, Coroutine, Optional, Union, cast import httpx @@ -74,12 +74,12 @@ def _timeout_to_seconds( def _prepare_ocr_request( model: str, - document: Dict[str, Any], + document: dict[str, Any], api_key: Optional[str], api_base: Optional[str], timeout: Optional[Union[float, httpx.Timeout]], custom_llm_provider: Optional[str], - extra_headers: Optional[Dict[str, Any]], + extra_headers: Optional[dict[str, Any]], kwargs: dict[str, object], ) -> _PreparedOCRRequest: litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj")) @@ -174,42 +174,35 @@ def _prepare_ocr_request( def _prepare_rust_ocr_call( - logging_obj: LiteLLMLoggingObj, - provider_config: BaseOCRConfig, + prepared_request: _PreparedOCRRequest, resolve_api_key: Callable[[str], Optional[str]], - model: str, - document: dict[str, object], - api_key: Optional[str], - api_base: Optional[str], - extra_headers: Optional[dict[str, object]], - optional_params: dict[str, object], - litellm_params: dict[str, object], ) -> _PreparedRustOCRCall: + provider_config = prepared_request.provider_config api_key_env_var = provider_config.get_api_key_env_var() - resolved_api_key = api_key or ( + resolved_api_key = prepared_request.api_key or ( resolve_api_key(api_key_env_var) if api_key_env_var is not None else None ) resolved_headers = provider_config.validate_environment( - headers=extra_headers or {}, - model=model, + headers=prepared_request.extra_headers or {}, + model=prepared_request.model, api_key=resolved_api_key, - api_base=api_base, - litellm_params=litellm_params, + api_base=prepared_request.api_base, + litellm_params=prepared_request.litellm_params, ) resolved_complete_url = provider_config.get_complete_url( - api_base=api_base, - model=model, - optional_params=optional_params, - litellm_params=litellm_params, + api_base=prepared_request.api_base, + model=prepared_request.model, + optional_params=prepared_request.optional_params, + litellm_params=prepared_request.litellm_params, ) - logging_obj.pre_call( + prepared_request.litellm_logging_obj.pre_call( input="OCR document processing", api_key=resolved_api_key, additional_args={ "complete_input_dict": { - "model": model, - "document": document, - **optional_params, + "model": prepared_request.model, + "document": prepared_request.document, + **prepared_request.optional_params, }, "api_base": resolved_complete_url, "headers": resolved_headers, @@ -224,18 +217,8 @@ def _prepare_rust_ocr_call( def _run_rust_ocr( rust_ocr: RustOcr, - logging_obj: LiteLLMLoggingObj, - provider_config: BaseOCRConfig, + prepared_request: _PreparedOCRRequest, resolve_api_key: Callable[[str], Optional[str]], - model: str, - document: dict[str, object], - api_key: Optional[str], - api_base: Optional[str], - custom_llm_provider: str, - extra_headers: Optional[dict[str, object]], - optional_params: dict[str, object], - litellm_params: dict[str, object], - timeout_seconds: Optional[float], ) -> OCRResponse: """Run the Mistral OCR call through the Rust bridge and wrap the result. @@ -246,68 +229,42 @@ def _run_rust_ocr( injected so this stays unit-testable without patching module globals. """ prepared = _prepare_rust_ocr_call( - logging_obj=logging_obj, - provider_config=provider_config, + prepared_request=prepared_request, resolve_api_key=resolve_api_key, - model=model, - document=document, - api_key=api_key, - api_base=api_base, - extra_headers=extra_headers, - optional_params=optional_params, - litellm_params=litellm_params, ) return OCRResponse.model_validate( rust_ocr( - model=model, - document=document, + model=prepared_request.model, + document=prepared_request.document, api_key=prepared.api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - optional_params=optional_params, - timeout_seconds=timeout_seconds, + api_base=prepared_request.api_base, + custom_llm_provider=prepared_request.custom_llm_provider, + extra_headers=prepared_request.extra_headers, + optional_params=prepared_request.optional_params, + timeout_seconds=_timeout_to_seconds(prepared_request.effective_timeout), ) ) async def _run_rust_aocr( rust_aocr: RustAocr, - logging_obj: LiteLLMLoggingObj, - provider_config: BaseOCRConfig, + prepared_request: _PreparedOCRRequest, resolve_api_key: Callable[[str], Optional[str]], - model: str, - document: dict[str, object], - api_key: Optional[str], - api_base: Optional[str], - custom_llm_provider: str, - extra_headers: Optional[dict[str, object]], - optional_params: dict[str, object], - litellm_params: dict[str, object], - timeout_seconds: Optional[float], ) -> OCRResponse: prepared = _prepare_rust_ocr_call( - logging_obj=logging_obj, - provider_config=provider_config, + prepared_request=prepared_request, resolve_api_key=resolve_api_key, - model=model, - document=document, - api_key=api_key, - api_base=api_base, - extra_headers=extra_headers, - optional_params=optional_params, - litellm_params=litellm_params, ) return OCRResponse.model_validate( await rust_aocr( - model=model, - document=document, + model=prepared_request.model, + document=prepared_request.document, api_key=prepared.api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - optional_params=optional_params, - timeout_seconds=timeout_seconds, + api_base=prepared_request.api_base, + custom_llm_provider=prepared_request.custom_llm_provider, + extra_headers=prepared_request.extra_headers, + optional_params=prepared_request.optional_params, + timeout_seconds=_timeout_to_seconds(prepared_request.effective_timeout), ) ) @@ -315,12 +272,12 @@ async def _run_rust_aocr( @client async def aocr( model: str, - document: Dict[str, Any], + document: dict[str, Any], api_key: Optional[str] = None, api_base: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[dict[str, Any]] = None, **kwargs, ) -> OCRResponse: """ @@ -405,18 +362,8 @@ async def aocr( response = await _run_rust_aocr( rust_aocr=rust_aocr, - logging_obj=prepared.litellm_logging_obj, - provider_config=prepared.provider_config, + prepared_request=prepared, resolve_api_key=get_secret_str, - model=prepared.model, - document=prepared.document, - api_key=prepared.api_key, - api_base=prepared.api_base, - custom_llm_provider=prepared.custom_llm_provider, - extra_headers=prepared.extra_headers, - optional_params=prepared.optional_params, - litellm_params=prepared.litellm_params, - timeout_seconds=_timeout_to_seconds(prepared.effective_timeout), ) if response is None: raise ValueError( @@ -461,12 +408,12 @@ async def aocr( @client def ocr( model: str, - document: Dict[str, Any], + document: dict[str, Any], api_key: Optional[str] = None, api_base: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, custom_llm_provider: Optional[str] = None, - extra_headers: Optional[Dict[str, Any]] = None, + extra_headers: Optional[dict[str, Any]] = None, **kwargs, ) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: """ @@ -557,18 +504,8 @@ def ocr( return _run_rust_ocr( rust_ocr=rust_ocr, - logging_obj=prepared.litellm_logging_obj, - provider_config=prepared.provider_config, + prepared_request=prepared, resolve_api_key=get_secret_str, - model=prepared.model, - document=prepared.document, - api_key=prepared.api_key, - api_base=prepared.api_base, - custom_llm_provider=prepared.custom_llm_provider, - extra_headers=prepared.extra_headers, - optional_params=prepared.optional_params, - litellm_params=prepared.litellm_params, - timeout_seconds=_timeout_to_seconds(prepared.effective_timeout), ) response = base_llm_http_handler.ocr( @@ -630,7 +567,7 @@ def get_mime_type(file_path: str) -> str: return guessed or "application/octet-stream" -def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, str]: +def convert_file_document_to_url_document(document: dict[str, Any]) -> dict[str, str]: """ Convert a file-type document dict to a document_url-type document dict with an inline base64 data URI. diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index d51b56330d2..c9b116ece00 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -124,6 +124,35 @@ class FakeOCRConfig: return f"{api_base or 'https://api.mistral.ai/v1'}/ocr" +def build_prepared_request( + *, + logging_obj=None, + provider_config=None, + model="mistral-ocr-latest", + document=DOCUMENT, + api_key="sk-test", + api_base=None, + custom_llm_provider="mistral", + extra_headers=None, + optional_params=None, + litellm_params=None, + timeout=12.5, +): + return ocr_main._PreparedOCRRequest( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + provider_config=provider_config or FakeOCRConfig(), + optional_params=optional_params or {}, + litellm_params=litellm_params or {}, + effective_timeout=timeout, + litellm_logging_obj=logging_obj or RecordingLogging(), + ) + + @pytest.fixture(autouse=True) def _reset_rust_flag(): """Keep the global toggle isolated between tests.""" @@ -231,18 +260,14 @@ def test_run_rust_ocr_forwards_args_and_wraps_response(): response = ocr_main._run_rust_ocr( rust_ocr=bridge, - logging_obj=logging_obj, - provider_config=FakeOCRConfig(), + prepared_request=build_prepared_request( + logging_obj=logging_obj, + api_base="https://proxy.internal", + extra_headers={"x-trace-id": "trace-1"}, + optional_params={"include_image_base64": True}, + timeout=12.5, + ), resolve_api_key=lambda _name: None, - model="mistral-ocr-latest", - document=DOCUMENT, - api_key="sk-test", - api_base="https://proxy.internal", - custom_llm_provider="mistral", - extra_headers={"x-trace-id": "trace-1"}, - optional_params={"include_image_base64": True}, - litellm_params={}, - timeout_seconds=12.5, ) assert isinstance(response, OCRResponse) @@ -267,20 +292,10 @@ def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): ocr_main._run_rust_ocr( rust_ocr=bridge, - logging_obj=RecordingLogging(), - provider_config=FakeOCRConfig(), + prepared_request=build_prepared_request(api_key=None, timeout=None), resolve_api_key=lambda name: ( "sk-from-vault" if name == "MISTRAL_API_KEY" else None ), - model="mistral-ocr-latest", - document=DOCUMENT, - api_key=None, - api_base=None, - custom_llm_provider="mistral", - extra_headers=None, - optional_params={}, - litellm_params={}, - timeout_seconds=None, ) assert bridge.calls[0]["api_key"] == "sk-from-vault" @@ -296,18 +311,13 @@ def test_run_rust_ocr_uses_provider_api_key_env_var(): ocr_main._run_rust_ocr( rust_ocr=bridge, - logging_obj=RecordingLogging(), - provider_config=FakeOCRConfig(api_key_env_var="PROVIDER_OCR_API_KEY"), + prepared_request=build_prepared_request( + provider_config=FakeOCRConfig(api_key_env_var="PROVIDER_OCR_API_KEY"), + model="provider-ocr-model", + api_key=None, + timeout=None, + ), resolve_api_key=_resolver, - model="provider-ocr-model", - document=DOCUMENT, - api_key=None, - api_base=None, - custom_llm_provider="mistral", - extra_headers=None, - optional_params={}, - litellm_params={}, - timeout_seconds=None, ) assert resolver_calls == ["PROVIDER_OCR_API_KEY"] @@ -324,18 +334,8 @@ def test_run_rust_ocr_prefers_explicit_key_over_resolver(): ocr_main._run_rust_ocr( rust_ocr=bridge, - logging_obj=RecordingLogging(), - provider_config=FakeOCRConfig(), + prepared_request=build_prepared_request(api_key="sk-explicit", timeout=None), resolve_api_key=_resolver, - model="mistral-ocr-latest", - document=DOCUMENT, - api_key="sk-explicit", - api_base=None, - custom_llm_provider="mistral", - extra_headers=None, - optional_params={}, - litellm_params={}, - timeout_seconds=None, ) assert bridge.calls[0]["api_key"] == "sk-explicit" @@ -348,18 +348,14 @@ def test_run_rust_ocr_runs_pre_call_logging(): ocr_main._run_rust_ocr( rust_ocr=RecordingBridge(), - logging_obj=logging_obj, - provider_config=FakeOCRConfig(), + prepared_request=build_prepared_request( + logging_obj=logging_obj, + api_base="https://api.mistral.ai/v1", + extra_headers={"x-trace-id": "trace-1"}, + optional_params={"include_image_base64": True}, + timeout=None, + ), resolve_api_key=lambda _name: None, - model="mistral-ocr-latest", - document=DOCUMENT, - api_key="sk-test", - api_base="https://api.mistral.ai/v1", - custom_llm_provider="mistral", - extra_headers={"x-trace-id": "trace-1"}, - optional_params={"include_image_base64": True}, - litellm_params={}, - timeout_seconds=None, ) assert logging_obj.pre_call_kwargs is not None From 65ce6a1522f36bf358b751271d73b323fb147a64 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 16:59:30 -0700 Subject: [PATCH 24/29] fix: reduce OCR basedpyright argument errors --- litellm/ocr/main.py | 10 +- tests/test_litellm/ocr/test_rust_bridge.py | 111 +++++++++++++-------- 2 files changed, 72 insertions(+), 49 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index f1c8682af99..f7cf9c4d96f 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -37,7 +37,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() @dataclass class _PreparedOCRRequest: model: str - document: dict[str, object] + document: dict[str, Any] api_key: Optional[str] api_base: Optional[str] custom_llm_provider: str @@ -80,7 +80,7 @@ def _prepare_ocr_request( timeout: Optional[Union[float, httpx.Timeout]], custom_llm_provider: Optional[str], extra_headers: Optional[dict[str, Any]], - kwargs: dict[str, object], + kwargs: dict[str, Any], ) -> _PreparedOCRRequest: litellm_logging_obj = cast(LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj")) litellm_call_id = cast(Optional[str], kwargs.get("litellm_call_id", None)) @@ -160,7 +160,7 @@ def _prepare_ocr_request( return _PreparedOCRRequest( model=model, - document=cast(dict[str, object], document), + document=document, api_key=api_key, api_base=api_base, custom_llm_provider=custom_llm_provider, @@ -235,7 +235,7 @@ def _run_rust_ocr( return OCRResponse.model_validate( rust_ocr( model=prepared_request.model, - document=prepared_request.document, + document=cast(dict[str, object], prepared_request.document), api_key=prepared.api_key, api_base=prepared_request.api_base, custom_llm_provider=prepared_request.custom_llm_provider, @@ -258,7 +258,7 @@ async def _run_rust_aocr( return OCRResponse.model_validate( await rust_aocr( model=prepared_request.model, - document=prepared_request.document, + document=cast(dict[str, object], prepared_request.document), api_key=prepared.api_key, api_base=prepared_request.api_base, custom_llm_provider=prepared_request.custom_llm_provider, diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index c9b116ece00..31a346c09c3 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -3,6 +3,7 @@ import importlib import sys import types +from typing import Any import httpx import pytest @@ -17,9 +18,12 @@ ocr_main = importlib.import_module("litellm.ocr.main") rust_bridge = importlib.import_module("litellm.ocr.rust_bridge") MODEL = "mistral/mistral-ocr-latest" -DOCUMENT = {"type": "document_url", "document_url": "https://example.com/doc.pdf"} +DOCUMENT: dict[str, object] = { + "type": "document_url", + "document_url": "https://example.com/doc.pdf", +} -FAKE_OCR_RESPONSE = { +FAKE_OCR_RESPONSE: dict[str, object] = { "pages": [{"index": 0, "markdown": "hello world"}], "model": "mistral-ocr-2505-completion", "document_annotation": None, @@ -31,20 +35,20 @@ FAKE_OCR_RESPONSE = { class RecordingBridge: """A fake ``RustOcr`` callable that records the args it was handed.""" - def __init__(self): - self.calls = [] + def __init__(self) -> None: + self.calls: list[dict[str, object]] = [] def __call__( self, - model, - document, - api_key, - api_base, - custom_llm_provider, - extra_headers, - optional_params, - timeout_seconds, - ): + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: self.calls.append( { "model": model, @@ -63,20 +67,20 @@ class RecordingBridge: class RecordingAsyncBridge: """A fake async ``RustAocr`` callable that records the args it was handed.""" - def __init__(self): - self.calls = [] + def __init__(self) -> None: + self.calls: list[dict[str, object]] = [] async def __call__( self, - model, - document, - api_key, - api_base, - custom_llm_provider, - extra_headers, - optional_params, - timeout_seconds, - ): + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: self.calls.append( { "model": model, @@ -95,10 +99,16 @@ class RecordingAsyncBridge: class RecordingLogging: """A spy standing in for ``LiteLLMLoggingObj`` to capture ``pre_call``.""" - def __init__(self): - self.pre_call_kwargs = None + def __init__(self) -> None: + self.pre_call_kwargs: dict[str, object] | None = None - def pre_call(self, *, input, api_key, additional_args): + def pre_call( + self, + *, + input: str, + api_key: str | None, + additional_args: dict[str, object], + ) -> None: self.pre_call_kwargs = { "input": input, "api_key": api_key, @@ -109,35 +119,48 @@ class RecordingLogging: class FakeOCRConfig: """A stand-in ``BaseOCRConfig`` that echoes the request it would build.""" - def __init__(self, api_key_env_var="MISTRAL_API_KEY"): + def __init__(self, api_key_env_var: str = "MISTRAL_API_KEY") -> None: self.api_key_env_var = api_key_env_var - def get_api_key_env_var(self): + def get_api_key_env_var(self) -> str: return self.api_key_env_var def validate_environment( - self, *, headers, model, api_key, api_base, litellm_params - ): + self, + *, + headers: dict[str, object], + model: str, + api_key: str | None, + api_base: str | None, + litellm_params: dict[str, object], + ) -> dict[str, object]: return {"Authorization": f"Bearer {api_key}", **headers} - def get_complete_url(self, *, api_base, model, optional_params, litellm_params): + def get_complete_url( + self, + *, + api_base: str | None, + model: str, + optional_params: dict[str, object], + litellm_params: dict[str, object], + ) -> str: return f"{api_base or 'https://api.mistral.ai/v1'}/ocr" def build_prepared_request( *, - logging_obj=None, - provider_config=None, - model="mistral-ocr-latest", - document=DOCUMENT, - api_key="sk-test", - api_base=None, - custom_llm_provider="mistral", - extra_headers=None, - optional_params=None, - litellm_params=None, - timeout=12.5, -): + logging_obj: RecordingLogging | None = None, + provider_config: FakeOCRConfig | None = None, + model: str = "mistral-ocr-latest", + document: dict[str, object] = DOCUMENT, + api_key: str | None = "sk-test", + api_base: str | None = None, + custom_llm_provider: str = "mistral", + extra_headers: dict[str, object] | None = None, + optional_params: dict[str, object] | None = None, + litellm_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = 12.5, +) -> Any: return ocr_main._PreparedOCRRequest( model=model, document=document, From 6b1e1dcbaf0da8295b5ce3ea7df1b9af0cbfc765 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 17:05:05 -0700 Subject: [PATCH 25/29] fix: address OCR greptile feedback --- litellm-rust/ADDING_A_PROVIDER.md | 2 +- litellm-rust/crates/core/build.rs | 2 +- litellm-rust/crates/core/src/providers/README.md | 3 ++- litellm/ocr/main.py | 4 ---- litellm/provider_endpoints_support_backup.json | 3 +-- provider_endpoints_support.json | 3 +-- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index 9555a0c53a3..113d27e84e5 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -1,7 +1,7 @@ # Adding a provider / route to litellm-rust - Keep the route contract pure in `crates/core/src//`: define the typed request/response structs and a provider config trait with no network, env, auth, or logging. -- Add provider identity to the repo-root `provider_endpoints_support.json`: use the LiteLLM provider slug, display name, docs URL, and endpoint support flags. Put optional stable base URL / API key env var defaults under the top-level `default_creds` map. +- Add provider identity to the repo-root `provider_endpoints_support.json`: use the LiteLLM provider slug, display name, docs URL, and endpoint support flags. Put optional stable provider-level base URL defaults under the top-level `default_creds` map; keep route-specific API key env vars in provider config/transform code so key resolution has one owner. - Put provider-specific transforms in `crates/providers/src///transformation.rs`, mirroring the Python provider tree and exposing a `const __CONFIG`. - The provider config owns three pure steps: map LiteLLM params, transform the LiteLLM request into the provider request, and transform the provider response back into the LiteLLM response. - If the provider has a reverse or normalization step, keep it pure and explicit next to the transforms; do not hide reverse mapping inside the HTTP transport. diff --git a/litellm-rust/crates/core/build.rs b/litellm-rust/crates/core/build.rs index 07be1279a0a..efc1da37f97 100644 --- a/litellm-rust/crates/core/build.rs +++ b/litellm-rust/crates/core/build.rs @@ -316,7 +316,7 @@ mod tests {{ LlmProvider::Mistral.default_api_base(), Some("https://api.mistral.ai/v1") ); - assert_eq!(LlmProvider::Mistral.api_key_env_var(), Some("MISTRAL_API_KEY")); + assert_eq!(LlmProvider::Mistral.api_key_env_var(), None); }} }} "#, diff --git a/litellm-rust/crates/core/src/providers/README.md b/litellm-rust/crates/core/src/providers/README.md index 84ef3731f3f..5c00534fd6e 100644 --- a/litellm-rust/crates/core/src/providers/README.md +++ b/litellm-rust/crates/core/src/providers/README.md @@ -10,7 +10,8 @@ To add a provider: - Add a `provider_endpoints_support.json` provider entry using the LiteLLM provider slug, display name, docs URL, and endpoint support flags. - Add optional defaults under the top-level `default_creds` map only when there - is a stable provider-level base URL or canonical API key environment variable. + is a stable provider-level base URL. Keep route-specific key env var names in + the provider transform/config so auth resolution has one owner. - Put request/response logic under `crates/providers/src///transformation.rs`; do not put transforms, signing logic, or secrets in provider metadata. diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index f7cf9c4d96f..479f184dcca 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -365,10 +365,6 @@ async def aocr( prepared_request=prepared, resolve_api_key=get_secret_str, ) - if response is None: - raise ValueError( - f"Got an unexpected None response from the OCR API: {response}" - ) return response response = base_llm_http_handler.ocr( diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index bd539102000..bf4ca71043d 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -33,8 +33,7 @@ }, "default_creds": { "mistral": { - "default_api_base": "https://api.mistral.ai/v1", - "api_key_env_var": "MISTRAL_API_KEY" + "default_api_base": "https://api.mistral.ai/v1" } }, "providers": { diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 27124b5634d..eb34389bcf9 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -33,8 +33,7 @@ }, "default_creds": { "mistral": { - "default_api_base": "https://api.mistral.ai/v1", - "api_key_env_var": "MISTRAL_API_KEY" + "default_api_base": "https://api.mistral.ai/v1" } }, "providers": { From b4b032116fcff8252a87027e9ba18edd59ea5cca Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 17:10:17 -0700 Subject: [PATCH 26/29] fix: align rust OCR request preparation --- litellm/ocr/main.py | 12 ++- tests/test_litellm/ocr/test_rust_bridge.py | 88 +++++++++++++++++++++- 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 479f184dcca..8bd87ae21d3 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -53,7 +53,6 @@ class _PreparedOCRRequest: class _PreparedRustOCRCall: api_key: Optional[str] headers: dict[str, object] - complete_url: str def _timeout_to_seconds( @@ -211,7 +210,6 @@ def _prepare_rust_ocr_call( return _PreparedRustOCRCall( api_key=resolved_api_key, headers=cast(dict[str, object], resolved_headers), - complete_url=resolved_complete_url, ) @@ -239,7 +237,7 @@ def _run_rust_ocr( api_key=prepared.api_key, api_base=prepared_request.api_base, custom_llm_provider=prepared_request.custom_llm_provider, - extra_headers=prepared_request.extra_headers, + extra_headers=prepared.headers, optional_params=prepared_request.optional_params, timeout_seconds=_timeout_to_seconds(prepared_request.effective_timeout), ) @@ -262,7 +260,7 @@ async def _run_rust_aocr( api_key=prepared.api_key, api_base=prepared_request.api_base, custom_llm_provider=prepared_request.custom_llm_provider, - extra_headers=prepared_request.extra_headers, + extra_headers=prepared.headers, optional_params=prepared_request.optional_params, timeout_seconds=_timeout_to_seconds(prepared_request.effective_timeout), ) @@ -350,6 +348,9 @@ async def aocr( extra_headers=extra_headers, kwargs=kwargs, ) + model = prepared.model + custom_llm_provider = prepared.custom_llm_provider + local_vars.update({"model": model, "custom_llm_provider": custom_llm_provider}) if prepared.custom_llm_provider == "mistral" and rust_ocr_enabled(): rust_aocr = load_rust_aocr() @@ -487,6 +488,9 @@ def ocr( extra_headers=extra_headers, timeout=timeout, ) + model = prepared.model + custom_llm_provider = prepared.custom_llm_provider + local_vars.update({"model": model, "custom_llm_provider": custom_llm_provider}) # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. if prepared.custom_llm_provider == "mistral" and rust_ocr_enabled(): diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 31a346c09c3..aad5a4d6f42 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -32,6 +32,10 @@ FAKE_OCR_RESPONSE: dict[str, object] = { } +class CapturedException(Exception): + pass + + class RecordingBridge: """A fake ``RustOcr`` callable that records the args it was handed.""" @@ -96,6 +100,36 @@ class RecordingAsyncBridge: return dict(FAKE_OCR_RESPONSE) +class RaisingBridge: + def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: + raise RuntimeError("bridge failed") + + +class RaisingAsyncBridge: + async def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: + raise RuntimeError("bridge failed") + + class RecordingLogging: """A spy standing in for ``LiteLLMLoggingObj`` to capture ``pre_call``.""" @@ -302,7 +336,10 @@ def test_run_rust_ocr_forwards_args_and_wraps_response(): "api_key": "sk-test", "api_base": "https://proxy.internal", "custom_llm_provider": "mistral", - "extra_headers": {"x-trace-id": "trace-1"}, + "extra_headers": { + "Authorization": "Bearer sk-test", + "x-trace-id": "trace-1", + }, "optional_params": {"include_image_base64": True}, "timeout_seconds": 12.5, } @@ -413,11 +450,33 @@ def test_ocr_routes_to_rust_when_enabled(fake_bridge): assert call["document"] == DOCUMENT assert call["api_key"] == "sk-test" assert call["custom_llm_provider"] == "mistral" - assert call["extra_headers"] == {"x-trace-id": "trace-1"} + assert call["extra_headers"] == { + "Authorization": "Bearer sk-test", + "x-trace-id": "trace-1", + } # Raw OCR params ride along in optional_params; Rust filters to supported keys. assert call["optional_params"].get("include_image_base64") is True +def test_ocr_exception_type_uses_resolved_provider_context( + monkeypatch: pytest.MonkeyPatch, +): + captured: dict[str, object] = {} + + def fake_exception_type(**kwargs: object) -> CapturedException: + captured.update(kwargs) + return CapturedException("wrapped") + + monkeypatch.setattr(ocr_main.litellm, "exception_type", fake_exception_type) + litellm.use_litellm_rust(True, ocr=RaisingBridge()) + + with pytest.raises(CapturedException): + litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert captured["model"] == "mistral-ocr-latest" + assert captured["custom_llm_provider"] == "mistral" + + @pytest.mark.asyncio async def test_aocr_routes_to_async_rust_when_enabled(fake_async_bridge): response = await litellm.aocr( @@ -436,10 +495,33 @@ async def test_aocr_routes_to_async_rust_when_enabled(fake_async_bridge): assert call["document"] == DOCUMENT assert call["api_key"] == "sk-test" assert call["custom_llm_provider"] == "mistral" - assert call["extra_headers"] == {"x-trace-id": "trace-1"} + assert call["extra_headers"] == { + "Authorization": "Bearer sk-test", + "x-trace-id": "trace-1", + } assert call["optional_params"].get("include_image_base64") is True +@pytest.mark.asyncio +async def test_aocr_exception_type_uses_resolved_provider_context( + monkeypatch: pytest.MonkeyPatch, +): + captured: dict[str, object] = {} + + def fake_exception_type(**kwargs: object) -> CapturedException: + captured.update(kwargs) + return CapturedException("wrapped") + + monkeypatch.setattr(ocr_main.litellm, "exception_type", fake_exception_type) + litellm.use_litellm_rust(True, aocr=RaisingAsyncBridge()) + + with pytest.raises(CapturedException): + await litellm.aocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert captured["model"] == "mistral-ocr-latest" + assert captured["custom_llm_provider"] == "mistral" + + def test_ocr_forwards_timeout_to_rust(fake_bridge): """Caller-supplied timeout must flow into the Rust bridge so the fixed 600s client ceiling doesn't silently override shorter deadlines.""" From 2977e2bf427238fad8c097eba9185fbbba39354a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 17:20:39 -0700 Subject: [PATCH 27/29] fix: resolve OCR CodeQL alerts --- litellm/ocr/main.py | 303 ++++++++++++----------- litellm/proxy/ocr_endpoints/endpoints.py | 3 +- 2 files changed, 165 insertions(+), 141 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 8bd87ae21d3..80fde8ff9f3 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -336,7 +336,16 @@ async def aocr( ) ``` """ - local_vars = locals() + completion_kwargs: dict[str, object] = { + "model": model, + "document": document, + "api_key": api_key, + "api_base": api_base, + "timeout": timeout, + "custom_llm_provider": custom_llm_provider, + "extra_headers": extra_headers, + "kwargs": kwargs, + } try: prepared = _prepare_ocr_request( model=model, @@ -350,7 +359,9 @@ async def aocr( ) model = prepared.model custom_llm_provider = prepared.custom_llm_provider - local_vars.update({"model": model, "custom_llm_provider": custom_llm_provider}) + completion_kwargs.update( + {"model": model, "custom_llm_provider": custom_llm_provider} + ) if prepared.custom_llm_provider == "mistral" and rust_ocr_enabled(): rust_aocr = load_rust_aocr() @@ -397,139 +408,7 @@ async def aocr( model=model, custom_llm_provider=custom_llm_provider, original_exception=e, - completion_kwargs=local_vars, - extra_kwargs=kwargs, - ) - - -@client -def ocr( - model: str, - document: dict[str, Any], - api_key: Optional[str] = None, - api_base: Optional[str] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - custom_llm_provider: Optional[str] = None, - extra_headers: Optional[dict[str, Any]] = None, - **kwargs, -) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: - """ - Synchronous OCR function. - - Args: - model: Model name (e.g., "mistral/mistral-ocr-latest") - document: Document to process in Mistral format: - {"type": "document_url", "document_url": "https://..."} for PDFs/docs, - {"type": "image_url", "image_url": "https://..."} for images, or - {"type": "file", "file": } for local files - api_key: Optional API key - api_base: Optional API base URL - timeout: Optional timeout - custom_llm_provider: Optional custom LLM provider - extra_headers: Optional extra headers - **kwargs: Additional parameters (e.g., include_image_base64, pages, image_limit) - - Returns: - OCRResponse in Mistral OCR format with pages, model, usage_info, etc. - - Example: - ```python - import litellm - - # OCR with PDF - response = litellm.ocr( - model="mistral/mistral-ocr-latest", - document={ - "type": "document_url", - "document_url": "https://arxiv.org/pdf/2201.04234" - }, - include_image_base64=True - ) - - # OCR with image - response = litellm.ocr( - model="mistral/mistral-ocr-latest", - document={ - "type": "image_url", - "image_url": "https://example.com/image.png" - } - ) - - # OCR with base64 encoded PDF - response = litellm.ocr( - model="mistral/mistral-ocr-latest", - document={ - "type": "document_url", - "document_url": f"data:application/pdf;base64,{base64_pdf}" - } - ) - - # OCR with local file - response = litellm.ocr( - model="mistral/mistral-ocr-latest", - document={"type": "file", "file": "/path/to/document.pdf"} - ) - - # Access pages - for page in response.pages: - print(f"Page {page.index}: {page.markdown}") - ``` - """ - local_vars = locals() - try: - _is_async = kwargs.pop("aocr", False) is True - prepared = _prepare_ocr_request( - model=model, - document=document, - api_key=api_key, - api_base=api_base, - kwargs=kwargs, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - timeout=timeout, - ) - model = prepared.model - custom_llm_provider = prepared.custom_llm_provider - local_vars.update({"model": model, "custom_llm_provider": custom_llm_provider}) - - # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. - if prepared.custom_llm_provider == "mistral" and rust_ocr_enabled(): - rust_ocr = load_rust_ocr() - if rust_ocr is None: - verbose_logger.debug( - "Rust OCR bridge unavailable; falling back to Python path" - ) - else: - from litellm.secret_managers.main import get_secret_str - - return _run_rust_ocr( - rust_ocr=rust_ocr, - prepared_request=prepared, - resolve_api_key=get_secret_str, - ) - - response = base_llm_http_handler.ocr( - model=prepared.model, - document=prepared.document, - optional_params=prepared.optional_params, - timeout=prepared.effective_timeout, - logging_obj=prepared.litellm_logging_obj, - api_key=prepared.api_key, - api_base=prepared.api_base, - custom_llm_provider=prepared.custom_llm_provider, - aocr=_is_async, - headers=prepared.extra_headers, - provider_config=prepared.provider_config, - litellm_params=prepared.litellm_params, - ) - - return response - except Exception as e: - raise litellm.exception_type( - model=model, - custom_llm_provider=custom_llm_provider, - original_exception=e, - completion_kwargs=local_vars, + completion_kwargs=completion_kwargs, extra_kwargs=kwargs, ) @@ -651,9 +530,153 @@ def convert_file_document_to_url_document(document: dict[str, Any]) -> dict[str, f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" ) return {"type": "image_url", "image_url": data_uri} - else: - verbose_logger.debug( - f"OCR file input: Converted file to document_url data URI " - f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + + verbose_logger.debug( + f"OCR file input: Converted file to document_url data URI " + f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + ) + return {"type": "document_url", "document_url": data_uri} + + +@client +def ocr( + model: str, + document: dict[str, Any], + api_key: Optional[str] = None, + api_base: Optional[str] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + custom_llm_provider: Optional[str] = None, + extra_headers: Optional[dict[str, Any]] = None, + **kwargs, +) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: + """ + Synchronous OCR function. + + Args: + model: Model name (e.g., "mistral/mistral-ocr-latest") + document: Document to process in Mistral format: + {"type": "document_url", "document_url": "https://..."} for PDFs/docs, + {"type": "image_url", "image_url": "https://..."} for images, or + {"type": "file", "file": } for local files + api_key: Optional API key + api_base: Optional API base URL + timeout: Optional timeout + custom_llm_provider: Optional custom LLM provider + extra_headers: Optional extra headers + **kwargs: Additional parameters (e.g., include_image_base64, pages, image_limit) + + Returns: + OCRResponse in Mistral OCR format with pages, model, usage_info, etc. + + Example: + ```python + import litellm + + # OCR with PDF + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "document_url", + "document_url": "https://arxiv.org/pdf/2201.04234" + }, + include_image_base64=True + ) + + # OCR with image + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "image_url", + "image_url": "https://example.com/image.png" + } + ) + + # OCR with base64 encoded PDF + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "document_url", + "document_url": f"data:application/pdf;base64,{base64_pdf}" + } + ) + + # OCR with local file + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": "/path/to/document.pdf"} + ) + + # Access pages + for page in response.pages: + print(f"Page {page.index}: {page.markdown}") + ``` + """ + completion_kwargs: dict[str, object] = { + "model": model, + "document": document, + "api_key": api_key, + "api_base": api_base, + "timeout": timeout, + "custom_llm_provider": custom_llm_provider, + "extra_headers": extra_headers, + "kwargs": kwargs, + } + try: + _is_async = kwargs.pop("aocr", False) is True + completion_kwargs["aocr"] = _is_async + prepared = _prepare_ocr_request( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + kwargs=kwargs, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + timeout=timeout, + ) + model = prepared.model + custom_llm_provider = prepared.custom_llm_provider + completion_kwargs.update( + {"model": model, "custom_llm_provider": custom_llm_provider} + ) + + # Optional Rust path: hand the whole Mistral OCR call to the Rust bridge. + if prepared.custom_llm_provider == "mistral" and rust_ocr_enabled(): + rust_ocr = load_rust_ocr() + if rust_ocr is None: + verbose_logger.debug( + "Rust OCR bridge unavailable; falling back to Python path" + ) + else: + from litellm.secret_managers.main import get_secret_str + + return _run_rust_ocr( + rust_ocr=rust_ocr, + prepared_request=prepared, + resolve_api_key=get_secret_str, + ) + + response = base_llm_http_handler.ocr( + model=prepared.model, + document=prepared.document, + optional_params=prepared.optional_params, + timeout=prepared.effective_timeout, + logging_obj=prepared.litellm_logging_obj, + api_key=prepared.api_key, + api_base=prepared.api_base, + custom_llm_provider=prepared.custom_llm_provider, + aocr=_is_async, + headers=prepared.extra_headers, + provider_config=prepared.provider_config, + litellm_params=prepared.litellm_params, + ) + + return response + except Exception as e: + raise litellm.exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=completion_kwargs, + extra_kwargs=kwargs, ) - return {"type": "document_url", "document_url": data_uri} diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index e32fee6afc5..823b4a5e86c 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -8,7 +8,6 @@ from fastapi import APIRouter, Depends, Request, Response, UploadFile from fastapi.responses import ORJSONResponse from litellm._logging import verbose_proxy_logger -from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -27,6 +26,8 @@ def _build_document_from_upload( Delegates to convert_file_document_to_url_document after resolving MIME type from the upload's content_type header or filename. """ + from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type + mime_type = content_type.split(";")[0].strip() if content_type else None if not mime_type or mime_type == "application/octet-stream": if filename: From 7edddbbcebc7e29c850fbab4596ff99e5f9d9a71 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 17:56:53 -0700 Subject: [PATCH 28/29] fix: avoid duplicate Rust OCR authorization header --- litellm-rust/crates/providers/src/ocr.rs | 108 ++++++++++++++++++++++- 1 file changed, 106 insertions(+), 2 deletions(-) diff --git a/litellm-rust/crates/providers/src/ocr.rs b/litellm-rust/crates/providers/src/ocr.rs index 03d7ce508c8..4f45c9a6dd1 100644 --- a/litellm-rust/crates/providers/src/ocr.rs +++ b/litellm-rust/crates/providers/src/ocr.rs @@ -76,6 +76,12 @@ fn string_headers(extra_headers: Option>) -> CoreResult bool { + headers + .iter() + .any(|(key, _)| key.eq_ignore_ascii_case("authorization")) +} + pub struct OcrRequest<'a> { pub model: &'a str, pub document: Value, @@ -107,8 +113,12 @@ pub async fn ocr(request: OcrRequest<'_>) -> CoreResult { .transform_ocr_request(model, request.document, filtered_params)? .data; - let mut request_builder = http_client().post(&url).bearer_auth(&api_key).json(&body); - for (key, value) in string_headers(request.extra_headers)? { + let headers = string_headers(request.extra_headers)?; + let mut request_builder = http_client().post(&url).json(&body); + if !has_authorization_header(&headers) { + request_builder = request_builder.bearer_auth(&api_key); + } + for (key, value) in headers { request_builder = request_builder.header(&key, value); } if let Some(duration) = request.timeout { @@ -145,6 +155,8 @@ pub async fn ocr(request: OcrRequest<'_>) -> CoreResult { mod tests { use super::*; use serde_json::json; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; #[test] fn truncate_error_body_passes_short_strings_through() { @@ -194,6 +206,98 @@ mod tests { ); } + #[test] + fn has_authorization_header_is_case_insensitive() { + let headers = vec![ + ("x-trace-id".to_string(), "trace-1".to_string()), + ("authorization".to_string(), "Bearer sk-test".to_string()), + ]; + + assert!(has_authorization_header(&headers)); + + let headers = vec![("Authorization".to_string(), "Bearer sk-test".to_string())]; + assert!(has_authorization_header(&headers)); + + let headers = vec![("x-trace-id".to_string(), "trace-1".to_string())]; + assert!(!has_authorization_header(&headers)); + } + + #[tokio::test] + async fn ocr_does_not_duplicate_authorization_header_when_header_is_supplied() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("test listener binds"); + let addr = listener.local_addr().expect("listener has local addr"); + + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts one request"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let n = socket.read(&mut buffer).await.expect("reads request"); + if n == 0 { + break; + } + request.extend_from_slice(&buffer[..n]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + + let response_body = r#"{"pages":[{"index":0,"markdown":"ok"}],"model":"mistral-ocr-latest","usage_info":{"pages_processed":1}}"#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + response_body.len(), + response_body + ); + socket + .write_all(response.as_bytes()) + .await + .expect("writes response"); + String::from_utf8(request).expect("request is utf8") + }); + + let mut headers = Map::new(); + headers.insert( + "Authorization".to_string(), + Value::String("Bearer sk-from-python".to_string()), + ); + headers.insert( + "x-trace-id".to_string(), + Value::String("trace-1".to_string()), + ); + + let response = ocr(OcrRequest { + model: "mistral-ocr-latest", + document: json!({ + "type": "document_url", + "document_url": "https://example.com/doc.pdf" + }), + api_key: Some("sk-for-rust-fallback"), + api_base: Some(&format!("http://{addr}")), + custom_llm_provider: "mistral", + extra_headers: Some(headers), + optional_params: Map::new(), + timeout: Some(Duration::from_secs(5)), + }) + .await + .expect("ocr request succeeds"); + + assert_eq!(response["pages"][0]["markdown"], "ok"); + + let request = server.await.expect("server task completes"); + let authorization_count = request + .lines() + .filter(|line| line.to_ascii_lowercase().starts_with("authorization:")) + .count(); + assert_eq!(authorization_count, 1, "{request}"); + assert!( + request.contains("authorization: Bearer sk-from-python") + || request.contains("Authorization: Bearer sk-from-python"), + "{request}" + ); + } + #[test] fn string_headers_rejects_non_string_values() { let headers = json!({ From 554cf886842964031ca78b901b84b8283c2156e9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jun 2026 18:25:57 -0700 Subject: [PATCH 29/29] ci: rerun CircleCI