From f51036e092683a7d83a86d08b7d293203f7c7267 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 10 Sep 2025 00:32:26 -0400 Subject: [PATCH 1/2] added tags to langchain --- docs/my-website/docs/langchain/langchain.md | 318 ++++++++++++++++++++ docs/my-website/docs/proxy/user_keys.md | 100 ++++++ 2 files changed, 418 insertions(+) diff --git a/docs/my-website/docs/langchain/langchain.md b/docs/my-website/docs/langchain/langchain.md index 78425a73b99..c67375ce1be 100644 --- a/docs/my-website/docs/langchain/langchain.md +++ b/docs/my-website/docs/langchain/langchain.md @@ -162,3 +162,321 @@ Get more details [here](../observability/lunary_integration.md) ## Use LangChain ChatLiteLLM + Langfuse Checkout this section [here](../observability/langfuse_integration#use-langchain-chatlitellm--langfuse) for more details on how to integrate Langfuse with ChatLiteLLM. + +## Using Tags with LangChain and LiteLLM + +Tags are a powerful feature in LiteLLM that allow you to categorize, filter, and track your LLM requests. When using LangChain with LiteLLM, you can pass tags through the `extra_body` parameter in the metadata. + +### Basic Tag Usage + + + + +```python +import os +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage, SystemMessage + +os.environ['OPENAI_API_KEY'] = "sk-your-key-here" + +chat = ChatOpenAI( + model="gpt-4o", + temperature=0.7, + extra_body={ + "metadata": { + "tags": ["production", "customer-support", "high-priority"] + } + } +) + +messages = [ + SystemMessage(content="You are a helpful customer support assistant."), + HumanMessage(content="How do I reset my password?") +] + +response = chat.invoke(messages) +print(response) +``` + + + + + +```python +import os +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage, SystemMessage + +os.environ['ANTHROPIC_API_KEY'] = "sk-ant-your-key-here" + +chat = ChatOpenAI( + model="claude-3-sonnet-20240229", + temperature=0.7, + extra_body={ + "metadata": { + "tags": ["research", "analysis", "claude-model"] + } + } +) + +messages = [ + SystemMessage(content="You are a research analyst."), + HumanMessage(content="Analyze this market trend...") +] + +response = chat.invoke(messages) +print(response) +``` + + + + + +```python +import os +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage, SystemMessage + +# No API key needed when using proxy +chat = ChatOpenAI( + openai_api_base="http://localhost:4000", # Your proxy URL + model="gpt-4o", + temperature=0.7, + extra_body={ + "metadata": { + "tags": ["proxy", "team-alpha", "feature-flagged"], + "generation_name": "customer-onboarding", + "trace_user_id": "user-12345" + } + } +) + +messages = [ + SystemMessage(content="You are an onboarding assistant."), + HumanMessage(content="Welcome our new customer!") +] + +response = chat.invoke(messages) +print(response) +``` + + + + +### Advanced Tag Patterns + +#### Dynamic Tags Based on Context + +```python +import os +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage, SystemMessage + +def create_chat_with_tags(user_type: str, feature: str): + """Create a chat instance with dynamic tags based on context""" + + # Build tags dynamically + tags = ["langchain-integration"] + + if user_type == "premium": + tags.extend(["premium-user", "high-priority"]) + elif user_type == "enterprise": + tags.extend(["enterprise", "custom-sla"]) + else: + tags.append("standard-user") + + # Add feature-specific tags + if feature == "code-review": + tags.extend(["development", "code-analysis"]) + elif feature == "content-gen": + tags.extend(["marketing", "content-creation"]) + + return ChatOpenAI( + openai_api_base="http://localhost:4000", + model="gpt-4o", + temperature=0.7, + extra_body={ + "metadata": { + "tags": tags, + "user_type": user_type, + "feature": feature, + "trace_user_id": f"user-{user_type}-{feature}" + } + } + ) + +# Usage examples +premium_chat = create_chat_with_tags("premium", "code-review") +enterprise_chat = create_chat_with_tags("enterprise", "content-gen") + +messages = [HumanMessage(content="Help me with this task")] +response = premium_chat.invoke(messages) +``` + +#### Tags for Cost Tracking and Analytics + +```python +import os +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage, SystemMessage + +# Tags for cost tracking +cost_tracking_chat = ChatOpenAI( + openai_api_base="http://localhost:4000", + model="gpt-4o", + temperature=0.7, + extra_body={ + "metadata": { + "tags": [ + "cost-center-marketing", + "budget-q4-2024", + "project-launch-campaign", + "high-cost-model" # Flag for expensive models + ], + "department": "marketing", + "project_id": "campaign-2024-q4", + "cost_threshold": "high" + } + } +) + +messages = [ + SystemMessage(content="You are a marketing copywriter."), + HumanMessage(content="Create compelling ad copy for our new product launch.") +] + +response = cost_tracking_chat.invoke(messages) +``` + +#### Tags for A/B Testing + +```python +import os +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage, SystemMessage +import random + +def create_ab_test_chat(test_variant: str = None): + """Create chat instance for A/B testing with appropriate tags""" + + if test_variant is None: + test_variant = random.choice(["variant-a", "variant-b"]) + + return ChatOpenAI( + openai_api_base="http://localhost:4000", + model="gpt-4o", + temperature=0.7 if test_variant == "variant-a" else 0.9, # Different temp for variants + extra_body={ + "metadata": { + "tags": [ + "ab-test-experiment-1", + f"variant-{test_variant}", + "temperature-test", + "user-experience" + ], + "experiment_id": "ab-test-001", + "variant": test_variant, + "test_group": "temperature-optimization" + } + } + ) + +# Run A/B test +variant_a_chat = create_ab_test_chat("variant-a") +variant_b_chat = create_ab_test_chat("variant-b") + +test_message = [HumanMessage(content="Explain quantum computing in simple terms")] + +response_a = variant_a_chat.invoke(test_message) +response_b = variant_b_chat.invoke(test_message) +``` + +### Tag Best Practices + +#### 1. **Consistent Naming Convention** +```python +# ✅ Good: Consistent, descriptive tags +tags = ["production", "api-v2", "customer-support", "urgent"] + +# ❌ Avoid: Inconsistent or unclear tags +tags = ["prod", "v2", "support", "urgent123"] +``` + +#### 2. **Hierarchical Tags** +```python +# ✅ Good: Hierarchical structure +tags = ["env:production", "team:backend", "service:api", "priority:high"] + +# This allows for easy filtering and grouping +``` + +#### 3. **Include Context Information** +```python +extra_body={ + "metadata": { + "tags": ["production", "user-onboarding"], + "user_id": "user-12345", + "session_id": "session-abc123", + "feature_flag": "new-onboarding-flow", + "environment": "production" + } +} +``` + +#### 4. **Tag Categories** +Consider organizing tags into categories: +- **Environment**: `production`, `staging`, `development` +- **Team/Service**: `backend`, `frontend`, `api`, `worker` +- **Feature**: `authentication`, `payment`, `notification` +- **Priority**: `critical`, `high`, `medium`, `low` +- **User Type**: `premium`, `enterprise`, `free` + +### Using Tags with LiteLLM Proxy + +When using tags with LiteLLM Proxy, you can: + +1. **Filter requests** based on tags +2. **Track costs** by tags in spend reports +3. **Apply routing rules** based on tags +4. **Monitor usage** with tag-based analytics + +#### Example Proxy Configuration with Tags + +```yaml +# config.yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: gpt-4o + api_key: your-key + +# Tag-based routing rules +tag_routing: + - tags: ["premium", "high-priority"] + models: ["gpt-4o", "claude-3-opus"] + - tags: ["standard"] + models: ["gpt-3.5-turbo", "claude-3-haiku"] +``` + +### Monitoring and Analytics + +Tags enable powerful analytics capabilities: + +```python +# Example: Get spend reports by tags +import requests + +response = requests.get( + "http://localhost:4000/global/spend/report", + headers={"Authorization": "Bearer sk-your-key"}, + params={ + "start_date": "2024-01-01", + "end_date": "2024-12-31", + "group_by": "tags" + } +) + +spend_by_tags = response.json() +``` + +This documentation covers the essential patterns for using tags effectively with LangChain and LiteLLM, enabling better organization, tracking, and analytics of your LLM requests. diff --git a/docs/my-website/docs/proxy/user_keys.md b/docs/my-website/docs/proxy/user_keys.md index ecf6f2d0532..21e1d3dbf40 100644 --- a/docs/my-website/docs/proxy/user_keys.md +++ b/docs/my-website/docs/proxy/user_keys.md @@ -357,6 +357,106 @@ assert user.age == 25 +## Using Tags for Categorization and Tracking + +Tags allow you to categorize, filter, and track your LLM requests. Add tags to your metadata for better organization and analytics. + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello!"}], + extra_body={ + "metadata": { + "tags": ["production", "customer-support", "urgent"], + "generation_name": "support-bot", + "trace_user_id": "user-123" + } + } +) +``` + + + + + +```python +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model="gpt-4o", + extra_body={ + "metadata": { + "tags": ["langchain-integration", "content-gen"], + "trace_user_id": "user-456" + } + } +) + +response = chat.invoke([HumanMessage(content="Generate a blog post")]) +``` + + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello!"}], + "metadata": { + "tags": ["api-test", "development"], + "trace_user_id": "test-user" + } +}' +``` + + + + + +```js +const { OpenAI } = require('openai'); + +const openai = new OpenAI({ + apiKey: "sk-1234", + baseURL: "http://0.0.0.0:4000" +}); + +async function main() { + const response = await openai.chat.completions.create({ + messages: [{ role: 'user', content: 'Hello!' }], + model: 'gpt-3.5-turbo', + metadata: { + tags: ["javascript-client", "api-test"], + trace_user_id: "js-user-789" + } + }); +} +``` + + + + +### Tag Benefits + +- **Cost Tracking**: Monitor spending by project/team/feature +- **Analytics**: Filter requests by tags in logs and dashboards +- **Routing**: Use tags for conditional model routing +- **Debugging**: Easier troubleshooting with categorized requests + ### Response Format ```json From c20a5b0bbd764d381ad3729a519f28b844878c49 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 10 Sep 2025 00:51:37 -0400 Subject: [PATCH 2/2] more info for turn_off_logging --- docs/my-website/docs/proxy/config_settings.md | 4 ++-- docs/my-website/docs/proxy/logging.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 7eb355d39f2..e33301bcd2e 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -21,7 +21,7 @@ litellm_settings: failure_callback: ["sentry"] # list of failure callbacks callbacks: ["otel"] # list of callbacks - runs on success and failure service_callbacks: ["datadog", "prometheus"] # logs redis, postgres failures on datadog, prometheus - turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged. + turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data. redact_user_api_key_info: boolean # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging. langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging @@ -131,7 +131,7 @@ general_settings: | failure_callback | array of strings | List of failure callbacks [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) | | callbacks | array of strings | List of callbacks - runs on success and failure [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) | | service_callbacks | array of strings | System health monitoring - Logs redis, postgres failures on specified services (e.g. datadog, prometheus) [Doc Metrics](prometheus) | -| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged [Proxy Logging](logging) | +| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data [Proxy Logging](logging) | | modify_params | boolean | If true, allows modifying the parameters of the request before it is sent to the LLM provider | | enable_preview_features | boolean | If true, enables preview features - e.g. Azure O1 Models with streaming support.| | redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) | diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 5d3f8417222..ff2591daad2 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -60,7 +60,7 @@ components in your system, including in logging tools. ### Redact Messages, Response Content -Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata - e.g. spend, will still be tracked. +Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata - e.g. spend, will still be tracked. Useful for privacy/compliance when handling sensitive data.