diff --git a/docs/my-website/docs/getting_started.md b/docs/my-website/docs/getting_started.md index e9b2a0db616..eed9d338479 100644 --- a/docs/my-website/docs/getting_started.md +++ b/docs/my-website/docs/getting_started.md @@ -86,7 +86,7 @@ LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone from litellm import completion ## set env variables for logging tools -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" +os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings os.environ["HELICONE_API_KEY"] = "your-helicone-key" os.environ["LANGFUSE_PUBLIC_KEY"] = "" os.environ["LANGFUSE_SECRET_KEY"] = "" diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index e5c3fdaa3be..c65f0f95e6a 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -399,10 +399,10 @@ LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone from litellm import completion ## set env variables for logging tools +os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings os.environ["HELICONE_API_KEY"] = "your-helicone-key" os.environ["LANGFUSE_PUBLIC_KEY"] = "" os.environ["LANGFUSE_SECRET_KEY"] = "" -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" os.environ["OPENAI_API_KEY"] diff --git a/docs/my-website/docs/langchain/langchain.md b/docs/my-website/docs/langchain/langchain.md index efa6b29250c..960ac2fd48e 100644 --- a/docs/my-website/docs/langchain/langchain.md +++ b/docs/my-website/docs/langchain/langchain.md @@ -111,5 +111,30 @@ chat.invoke(messages) +## Use Langchain ChatLiteLLM with Lunary +```python +import os +from langchain.chat_models import ChatLiteLLM +from langchain.schema import HumanMessage +import litellm + +os.environ["LUNARY_PUBLIC_KEY"] = "" # from https://app.lunary.ai/settings +os.environ['OPENAI_API_KEY']="sk-..." + +litellm.success_callback = ["lunary"] +litellm.failure_callback = ["lunary"] + +chat = ChatLiteLLM( + model="gpt-4o" + messages = [ + HumanMessage( + content="what model are you" + ) +] +chat(messages) +``` + +Get more details [here](../observability/lunary_integration.md) + ## Use LangChain ChatLiteLLM + Langfuse Checkout this section [here](../observability/langfuse_integration#use-langchain-chatlitellm--langfuse) for more details on how to integrate Langfuse with ChatLiteLLM. diff --git a/docs/my-website/docs/observability/callbacks.md b/docs/my-website/docs/observability/callbacks.md index b959e8aae7d..69cb0d053ee 100644 --- a/docs/my-website/docs/observability/callbacks.md +++ b/docs/my-website/docs/observability/callbacks.md @@ -7,11 +7,11 @@ liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, liteLLM supports: - [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback) +- [Lunary](https://lunary.ai/docs) - [Langfuse](https://langfuse.com/docs) - [LangSmith](https://www.langchain.com/langsmith) - [Helicone](https://docs.helicone.ai/introduction) - [Traceloop](https://traceloop.com/docs) -- [Lunary](https://lunary.ai/docs) - [Athina](https://docs.athina.ai/) - [Sentry](https://docs.sentry.io/platforms/python/) - [PostHog](https://posthog.com/docs/libraries/python) @@ -30,6 +30,7 @@ litellm.success_callback=["posthog", "helicone", "langfuse", "lunary", "athina"] litellm.failure_callback=["sentry", "lunary", "langfuse"] ## set env variables +os.environ['LUNARY_PUBLIC_KEY'] = "" os.environ['SENTRY_DSN'], os.environ['SENTRY_API_TRACE_RATE']= "" os.environ['POSTHOG_API_KEY'], os.environ['POSTHOG_API_URL'] = "api-key", "api-url" os.environ["HELICONE_API_KEY"] = "" diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index 5be4ad64134..8f55c854db8 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -59,7 +59,7 @@ os.environ["LANGSMITH_API_KEY"] = "" # LLM API Keys os.environ['OPENAI_API_KEY']="" -# set langfuse as a callback, litellm will send the data to langfuse +# set langsmith as a callback, litellm will send the data to langsmith litellm.success_callback = ["langsmith"] response = litellm.completion( diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md index 56e74132f78..8d28321c807 100644 --- a/docs/my-website/docs/observability/lunary_integration.md +++ b/docs/my-website/docs/observability/lunary_integration.md @@ -1,72 +1,78 @@ -# Lunary - Logging and tracing LLM input/output +import Image from '@theme/IdealImage'; -:::tip +# 🌙 Lunary - GenAI Observability -This is community maintained, Please make an issue if you run into a bug -https://github.com/BerriAI/litellm +[Lunary](https://lunary.ai/) is an open-source platform providing [observability](https://lunary.ai/docs/features/observe), [prompt management](https://lunary.ai/docs/features/prompts), and [analytics](https://lunary.ai/docs/features/observe#analytics) to help team manage and improve LLM chatbots. -::: - - -[Lunary](https://lunary.ai/) is an open-source AI developer platform providing observability, prompt management, and evaluation tools for AI developers. +You can reach out to us anytime by [email](mailto:hello@lunary.ai) or directly [schedule a Demo](https://lunary.ai/schedule). -## Use Lunary to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM) -liteLLM provides `callbacks`, making it easy for you to log data depending on the status of your responses. +## Usage with LiteLLM Python SDK +### Pre-Requisites -:::info -We want to learn how we can make the callbacks better! Meet the [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or -join our [discord](https://discord.gg/wuPM9dRgDw) -::: +```shell +pip install litellm lunary +``` -### Using Callbacks +### Quick Start -First, sign up to get a public key on the [Lunary dashboard](https://lunary.ai). +First, get your Lunary public key on the [Lunary dashboard](https://app.lunary.ai/). -Use just 2 lines of code, to instantly log your responses **across all providers** with lunary: +Use just 2 lines of code, to instantly log your responses **across all providers** with Lunary: ```python litellm.success_callback = ["lunary"] litellm.failure_callback = ["lunary"] ``` -Complete code - +Complete code: ```python from litellm import completion -## set env variables -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" - +os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # from https://app.lunary.ai/) os.environ["OPENAI_API_KEY"] = "" -# set callbacks litellm.success_callback = ["lunary"] litellm.failure_callback = ["lunary"] -#openai call response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}], + model="gpt-4o", + messages=[{"role": "user", "content": "Hi there 👋"}], user="ishaan_litellm" ) ``` -## Templates +### Usage with LangChain ChatLiteLLM +```python +import os +from langchain.chat_models import ChatLiteLLM +from langchain.schema import HumanMessage +import litellm -You can use Lunary to manage prompt templates and use them across all your LLM providers. +os.environ["LUNARY_PUBLIC_KEY"] = "" # from https://app.lunary.ai/settings +os.environ['OPENAI_API_KEY']="sk-..." -Make sure to have `lunary` installed: +litellm.success_callback = ["lunary"] +litellm.failure_callback = ["lunary"] -```bash -pip install lunary +chat = ChatLiteLLM( + model="gpt-4o" + messages = [ + HumanMessage( + content="what model are you" + ) +] +chat(messages) ``` -Then, use the following code to pull templates into Lunary: + +### Usage with Prompt Templates + +You can use Lunary to manage [prompt templates](https://lunary.ai/docs/features/prompts) and use them across all your LLM providers with LiteLLM. ```python from litellm import completion @@ -81,9 +87,93 @@ litellm.success_callback = ["lunary"] result = completion(**template) ``` +### Usage with custom chains +You can wrap your LLM calls inside custom chains, so that you can visualize them as traces. + +```python +import litellm +from litellm import completion +import lunary + +litellm.success_callback = ["lunary"] +litellm.failure_callback = ["lunary"] + +@lunary.chain("My custom chain name") +def my_chain(chain_input): + chain_run_id = lunary.run_manager.current_run_id + response = completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Say 1"}], + metadata={"parent_run_id": chain_run_id}, + ) + + response = completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Say 2"}], + metadata={"parent_run_id": chain_run_id}, + ) + chain_output = response.choices[0].message + return chain_output + +my_chain("Chain input") +``` + + + +## Usage with LiteLLM Proxy Server +### Step1: Install dependencies and set your environment variables +Install the dependencies +```shell +pip install litellm lunary +``` + +Get you Lunary public key from from https://app.lunary.ai/settings +```shell +export LUNARY_PUBLIC_KEY="" +``` + +### Step 2: Create a `config.yaml` and set `lunary` callbacks + +```yaml +model_list: + - model_name: "*" + litellm_params: + model: "*" +litellm_settings: + success_callback: ["lunary"] + failure_callback: ["lunary"] +``` + +### Step 3: Start the LiteLLM proxy +```shell +litellm --config config.yaml +``` + +### Step 4: Make a request + +```shell +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-d '{ + "model": "gpt-4o", + "messages": [ + { + "role": "system", + "content": "You are a helpful math tutor. Guide the user through the solution step by step." + }, + { + "role": "user", + "content": "how can I solve 8x + 7 = -23" + } + ] +}' +``` + +You can find more details about the different ways of making requests to the LiteLLM proxy on [this page](https://docs.litellm.ai/docs/proxy/user_keys) + + ## Support & Talk to Founders -- Meet the Lunary team via [email](mailto:hello@lunary.ai). - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) - Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ diff --git a/docs/my-website/docs/proxy/architecture.md b/docs/my-website/docs/proxy/architecture.md index 4cd23adb5e4..f8804e2d238 100644 --- a/docs/my-website/docs/proxy/architecture.md +++ b/docs/my-website/docs/proxy/architecture.md @@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem'; 6. [**litellm.completion() / litellm.embedding()**:](../index#litellm-python-sdk) The litellm Python SDK is used to call the LLM in the OpenAI API format (Translation and parameter mapping) 7. **Post-Request Processing**: After the response is sent back to the client, the following **asynchronous** tasks are performed: - - [Logging to LangFuse (logging destination is configurable)](./logging) + - [Logging to Lunary, LangFuse or other logging destinations](./logging) - The [MaxParallelRequestsHandler](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py) updates the rpm/tpm usage for the - Global Server Rate Limit - Virtual Key Rate Limit diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 3b1588da8da..ab6dcd338ac 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -2,6 +2,7 @@ Log Proxy input, output, and exceptions using: +- Lunary - Langfuse - OpenTelemetry - GCS, s3, Azure (Blob) Buckets @@ -182,6 +183,55 @@ Found under `kwargs["standard_logging_object"]`. This is a standard payload, log [👉 **Standard Logging Payload Specification**](./logging_spec) +## Lunary +### Step1: Install dependencies and set your environment variables +Install the dependencies +```shell +pip install litellm lunary +``` + +Get you Lunary public key from from https://app.lunary.ai/settings +```shell +export LUNARY_PUBLIC_KEY="" +``` + +### Step 2: Create a `config.yaml` and set `lunary` callbacks + +```yaml +model_list: + - model_name: "*" + litellm_params: + model: "*" +litellm_settings: + success_callback: ["lunary"] + failure_callback: ["lunary"] +``` + +### Step 3: Start the LiteLLM proxy +```shell +litellm --config config.yaml +``` + +### Step 4: Make a request + +```shell +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-d '{ + "model": "gpt-4o", + "messages": [ + { + "role": "system", + "content": "You are a helpful math tutor. Guide the user through the solution step by step." + }, + { + "role": "user", + "content": "how can I solve 8x + 7 = -23" + } + ] +}' +``` + ## Langfuse We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment diff --git a/docs/my-website/img/lunary-trace.png b/docs/my-website/img/lunary-trace.png new file mode 100644 index 00000000000..509e63ad543 Binary files /dev/null and b/docs/my-website/img/lunary-trace.png differ diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 93400a81c9c..b11ba575f09 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -379,6 +379,7 @@ const sidebars = { type: "category", label: "Logging & Observability", items: [ + "observability/lunary_integration", "observability/mlflow", "observability/langfuse_integration", "observability/gcs_bucket_integration", @@ -402,7 +403,6 @@ const sidebars = { "observability/wandb_integration", "observability/slack_integration", "observability/athina_integration", - "observability/lunary_integration", "observability/greenscale_integration", "observability/supabase_integration", `observability/telemetry`, diff --git a/docs/my-website/src/pages/index.md b/docs/my-website/src/pages/index.md index cea3dc52b56..a13e556eab3 100644 --- a/docs/my-website/src/pages/index.md +++ b/docs/my-website/src/pages/index.md @@ -337,10 +337,10 @@ LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone from litellm import completion ## set env variables for logging tools +os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your key at https://app.lunary.ai/settings os.environ["HELICONE_API_KEY"] = "your-helicone-key" os.environ["LANGFUSE_PUBLIC_KEY"] = "" os.environ["LANGFUSE_SECRET_KEY"] = "" -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" os.environ["OPENAI_API_KEY"]