diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index 1b52c0d58..ed31b4377 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -90,6 +90,30 @@ output_cost_per_mtok = 16.00 cache_input_cost_per_mtok = 0.60 ``` +For [LiteLLM](/integrations/litellm), Fabro ships a disabled provider entry. Enable it in settings and declare the models your proxy exposes: + +```toml title="settings.toml" +[llm.providers.litellm] +enabled = true +base_url = "http://localhost:4000/v1" + +[llm.models."litellm-gpt-5"] +provider = "litellm" +api_id = "gpt-5" +display_name = "LiteLLM GPT-5" +family = "litellm" +default = true + +[llm.models."litellm-gpt-5".limits] +context_window = 128000 +max_output = 8192 + +[llm.models."litellm-gpt-5".features] +tools = true +vision = false +reasoning = false +``` + `api_id` is the model name sent to the provider API. Omit it when the Fabro model ID and provider model ID are the same. Header values and credentials are typed references, not raw secrets. Use `env:` or `credential:` for provider credentials, and `{ env = "NAME" }`, `{ credential = "id" }`, or `{ literal = "value" }` for extra headers. diff --git a/docs/public/docs.json b/docs/public/docs.json index fc3d932dc..977006732 100644 --- a/docs/public/docs.json +++ b/docs/public/docs.json @@ -93,6 +93,7 @@ "pages": [ "integrations/github", "integrations/daytona", + "integrations/litellm", "integrations/slack", "integrations/brave-search" ] diff --git a/docs/public/integrations/litellm.mdx b/docs/public/integrations/litellm.mdx new file mode 100644 index 000000000..f39712079 --- /dev/null +++ b/docs/public/integrations/litellm.mdx @@ -0,0 +1,133 @@ +--- +title: "LiteLLM" +description: "Route Fabro models through a LiteLLM proxy" +--- + +[LiteLLM](https://docs.litellm.ai/) can run as an OpenAI-compatible proxy in front of many model providers. Fabro includes a disabled `litellm` provider entry so you can opt in from `settings.toml` without changing Fabro code. + +## Prerequisites + +- A running LiteLLM proxy reachable from the Fabro process +- At least one LiteLLM model name you want Fabro to route to +- A LiteLLM key or placeholder key available to Fabro + +Fabro's built-in LiteLLM provider points at `http://localhost:4000/v1`. Change `base_url` if your proxy is hosted elsewhere. + +## Enable the provider + +Add the provider override and one or more model entries to `~/.fabro/settings.toml`: + +```toml title="settings.toml" +_version = 1 + +[llm.providers.litellm] +enabled = true +base_url = "http://localhost:4000/v1" + +[llm.models."litellm-gpt-5"] +provider = "litellm" +api_id = "gpt-5" +display_name = "LiteLLM GPT-5" +family = "litellm" +default = true + +[llm.models."litellm-gpt-5".limits] +context_window = 128000 +max_output = 8192 + +[llm.models."litellm-gpt-5".features] +tools = true +vision = false +reasoning = false +``` + +`api_id` is the model name Fabro sends to LiteLLM. It should match a model name configured in your LiteLLM proxy. + +## Configure credentials + +The LiteLLM provider checks `credential:litellm` first, then `LITELLM_API_KEY` from the Fabro process environment. + +For a server-owned secret: + +```bash +fabro secret set litellm sk-proxy-key +``` + +For a process environment variable: + +```bash +export LITELLM_API_KEY=sk-proxy-key +``` + +If your local LiteLLM proxy does not enforce authentication, use a placeholder value such as `anything`; the OpenAI-compatible client still needs a credential value. + +## Use LiteLLM models + +Once the provider is enabled and at least one model is declared, use the Fabro model ID like any other catalog model: + +```bash +fabro model list --provider litellm +fabro model test --model litellm-gpt-5 +fabro run workflow.fabro --model litellm-gpt-5 +``` + +In workflow stylesheets: + +```dot title="workflow.fabro" +digraph Example { + graph [ + model_stylesheet=" + * { model: litellm-gpt-5; } + " + ] + + start [shape=Mdiamond, label="Start"] + work [label="Work", prompt="Use the configured LiteLLM model."] + exit [shape=Msquare, label="Exit"] + + start -> work -> exit +} +``` + +## Declaring more models + +Declare each LiteLLM-routed model explicitly so Fabro knows its provider, context window, tool support, and routing defaults: + +```toml title="settings.toml" +[llm.models."litellm-fast"] +provider = "litellm" +api_id = "fast-model" +display_name = "LiteLLM Fast" +family = "litellm" +aliases = ["fast"] + +[llm.models."litellm-fast".limits] +context_window = 64000 +max_output = 4096 + +[llm.models."litellm-fast".features] +tools = true +vision = false +reasoning = false +``` + +Only one model for a provider should set `default = true`. + +## Troubleshooting + +**"No API key configured"** — Set `credential:litellm` with `fabro secret set litellm ...` or export `LITELLM_API_KEY` in the Fabro process environment. + +**Connection refused** — Confirm the LiteLLM proxy is running and that `base_url` is reachable from the Fabro process. For Docker deployments, `localhost` means the Fabro container unless you point it at a host or service name. + +**Unknown model from LiteLLM** — Check that the model's `api_id` matches the model name configured in LiteLLM, then run `fabro model test --model `. + +## Further reading + + + + How Fabro routes model IDs, providers, and fallbacks. + + + Full reference for `[llm.providers.]` and `[llm.models.]`. + + diff --git a/lib/crates/fabro-model/src/catalog/providers/litellm.toml b/lib/crates/fabro-model/src/catalog/providers/litellm.toml new file mode 100644 index 000000000..3acb6e852 --- /dev/null +++ b/lib/crates/fabro-model/src/catalog/providers/litellm.toml @@ -0,0 +1,29 @@ +[providers.litellm] +display_name = "LiteLLM" +adapter = "openai_compatible" +base_url = "http://localhost:4000/v1" +credentials = ["credential:litellm", "env:LITELLM_API_KEY"] +priority = 50 +enabled = false + +# To enable LiteLLM, add entries like these to settings.toml: +# +# [llm.providers.litellm] +# enabled = true +# base_url = "http://localhost:4000/v1" +# +# [llm.models."litellm-gpt-5"] +# provider = "litellm" +# api_id = "gpt-5" +# display_name = "LiteLLM GPT-5" +# family = "litellm" +# default = true +# +# [llm.models."litellm-gpt-5".limits] +# context_window = 128000 +# max_output = 8192 +# +# [llm.models."litellm-gpt-5".features] +# tools = true +# vision = false +# reasoning = false