mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
chore: remove accidentally committed demo files
Some checks failed
Unit Tests: Proxy DB Operations / proxy-db (auth-checks, tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py, 20, 8) (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-db (key-generation, tests/proxy_unit_tests/test_key_generate_prisma.py, 30, 0) (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-db (remaining, tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py, 30, 8) (push) Has been cancelled
Unit Tests: Security / security (push) Has been cancelled
Some checks failed
Unit Tests: Proxy DB Operations / proxy-db (auth-checks, tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py, 20, 8) (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-db (key-generation, tests/proxy_unit_tests/test_key_generate_prisma.py, 30, 0) (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-db (remaining, tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py, 30, 8) (push) Has been cancelled
Unit Tests: Security / security (push) Has been cancelled
This commit is contained in:
parent
af24a3f720
commit
3dcb4b3046
2 changed files with 0 additions and 87 deletions
|
|
@ -1,12 +0,0 @@
|
|||
model_list:
|
||||
- model_name: bedrock-claude-haiku
|
||||
litellm_params:
|
||||
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
|
||||
aws_region_name: us-east-1
|
||||
|
||||
litellm_settings:
|
||||
success_callback: []
|
||||
failure_callback: []
|
||||
|
||||
general_settings:
|
||||
store_model_in_db: false
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
"""
|
||||
Demo script: back-to-back Bedrock streaming requests with prompt caching.
|
||||
Request 1: populates the cache (cache_creation_input_tokens)
|
||||
Request 2: reads from cache (cache_read_input_tokens)
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
PROXY_URL = "http://localhost:4001"
|
||||
API_KEY = "sk-1234"
|
||||
|
||||
# ~5000 token system prompt (above claude-haiku-4-5's 2048-token min for caching on Bedrock)
|
||||
LARGE_SYSTEM_PROMPT = (
|
||||
"AWS Bedrock provides managed ML infrastructure for enterprise workloads. "
|
||||
"Anthropic Claude models support prompt caching for cost optimization. "
|
||||
) * 200
|
||||
|
||||
|
||||
def make_streaming_request(req_num: int, label: str) -> None:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Request {req_num}: {label}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
payload = {
|
||||
"model": "bedrock-claude-haiku",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": LARGE_SYSTEM_PROMPT,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": f"Say only: 'Request {req_num} done'"},
|
||||
],
|
||||
"stream": True,
|
||||
"max_tokens": 20,
|
||||
}
|
||||
|
||||
full_response = ""
|
||||
with httpx.Client(timeout=60) as client:
|
||||
with client.stream(
|
||||
"POST",
|
||||
f"{PROXY_URL}/v1/chat/completions",
|
||||
json=payload,
|
||||
headers={"Authorization": f"Bearer {API_KEY}"},
|
||||
) as r:
|
||||
r.raise_for_status()
|
||||
for line in r.iter_lines():
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
chunk = json.loads(line[6:])
|
||||
delta = chunk.get("choices", [{}])[0].get("delta", {})
|
||||
if content := delta.get("content"):
|
||||
full_response += content
|
||||
|
||||
print(f"Response: {full_response!r}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Sending request 1 (cache write)...")
|
||||
make_streaming_request(1, "cache WRITE (populates cache)")
|
||||
|
||||
print("\nWaiting 2s between requests...")
|
||||
time.sleep(2)
|
||||
|
||||
print("Sending request 2 (cache read)...")
|
||||
make_streaming_request(2, "cache READ (hits cache)")
|
||||
|
||||
print("\n\nDone. Check SpendLogs in the DB or the UI at http://localhost:4001")
|
||||
Loading…
Add table
Reference in a new issue