From 51b9178630ecb89864af2bddfb1b7cdddc4feabe Mon Sep 17 00:00:00 2001 From: Giri Tatavarty Date: Wed, 29 May 2024 15:08:56 -0700 Subject: [PATCH 01/99] #Fixed mypy errors. The requests package and stubs need to be imported - waiting to hear from Ishaan/Krrish before changing requirements.txt --- litellm/llms/triton.py | 181 ++++++++++++++--------------------------- litellm/main.py | 2 +- 2 files changed, 60 insertions(+), 123 deletions(-) diff --git a/litellm/llms/triton.py b/litellm/llms/triton.py index 43220eec111..626c7c5b415 100644 --- a/litellm/llms/triton.py +++ b/litellm/llms/triton.py @@ -1,19 +1,20 @@ -import os, types +import os import json from enum import Enum -import requests, copy # type: ignore +import requests import time -from typing import Callable, Optional, List -from litellm.utils import ModelResponse, Choices,Usage, map_finish_reason, CustomStreamWrapper, Message +from typing import Callable, Optional, List, Sequence, Any, Union, Dict +from litellm.utils import ModelResponse, Choices, Usage, map_finish_reason, CustomStreamWrapper, Message, EmbeddingResponse import litellm from .prompt_templates.factory import prompt_factory, custom_prompt from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from .base import BaseLLM -import httpx # type: ignore -import requests +import httpx +from typing import Union,Collection + class TritonError(Exception): - def __init__(self, status_code, message): + def __init__(self, status_code: int, message: str) -> None: self.status_code = status_code self.message = message self.request = httpx.Request( @@ -25,54 +26,10 @@ class TritonError(Exception): self.message ) # Call the base class constructor with the parameters it needs - class TritonChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() - async def acompletion( - self, - data: dict, - model_response: ModelResponse, - api_base: str, - logging_obj=None, - api_key: Optional[str] = None, - ): - - async_handler = httpx.AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=600.0, connect=5.0) - ) - - if api_base.endswith("generate") : ### This is a trtllm model - - async with httpx.AsyncClient() as client: - response = await client.post(url=api_base, json=data) - - - - if response.status_code != 200: - raise TritonError(status_code=response.status_code, message=response.text) - - _text_response = response.text - - - if logging_obj: - logging_obj.post_call(original_response=_text_response) - - _json_response = response.json() - - _output_text = _json_response["outputs"][0]["data"][0] - # decode the byte string - _output_text = _output_text.encode("latin-1").decode("unicode_escape").encode( - "latin-1" - ).decode("utf-8") - - model_response.model = _json_response.get("model_name", "None") - model_response.choices[0].message.content = _output_text - - return model_response - - async def aembedding( self, data: dict, @@ -80,8 +37,7 @@ class TritonChatCompletion(BaseLLM): api_base: str, logging_obj=None, api_key: Optional[str] = None, - ): - + ) -> EmbeddingResponse: async_handler = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0) ) @@ -98,7 +54,7 @@ class TritonChatCompletion(BaseLLM): _json_response = response.json() _outputs = _json_response["outputs"] - _output_data = [ output["data"] for output in _outputs ] + _output_data = [output["data"] for output in _outputs] _embedding_output = { "object": "embedding", "index": 0, @@ -110,10 +66,10 @@ class TritonChatCompletion(BaseLLM): return model_response - def embedding( + async def embedding( self, model: str, - input: list, + input: List[str], timeout: float, api_base: str, model_response: litellm.utils.EmbeddingResponse, @@ -121,21 +77,19 @@ class TritonChatCompletion(BaseLLM): logging_obj=None, optional_params=None, client=None, - aembedding=None, - ): + aembedding: bool = False, + ) -> EmbeddingResponse: data_for_triton = { "inputs": [ { "name": "input_text", - "shape": [len(input)], #size of the input data + "shape": [len(input)], # size of the input data "datatype": "BYTES", "data": input, } ] } - ## LOGGING - curl_string = f"curl {api_base} -X POST -H 'Content-Type: application/json' -d '{data_for_triton}'" logging_obj.pre_call( @@ -147,8 +101,8 @@ class TritonChatCompletion(BaseLLM): }, ) - if aembedding == True: - response = self.aembedding( + if aembedding: + response = await self.aembedding( data=data_for_triton, model_response=model_response, logging_obj=logging_obj, @@ -160,11 +114,11 @@ class TritonChatCompletion(BaseLLM): raise Exception( "Only async embedding supported for triton, please use litellm.aembedding() for now" ) - ## Using Sync completion for now - Async completion not supported yet. + def completion( self, model: str, - messages: list, + messages: List[dict], timeout: float, api_base: str, model_response: ModelResponse, @@ -172,48 +126,44 @@ class TritonChatCompletion(BaseLLM): logging_obj=None, optional_params=None, client=None, - stream=False, - ): - # check if model is llama - data_for_triton = {} - type_of_model = "" "" - if api_base.endswith("generate") : ### This is a trtllm model - # this is a llama model - text_input = messages[0]["content"] - data_for_triton = { - "text_input":f"{text_input}", - "parameters": { - "max_tokens": optional_params.get("max_tokens", 20), - "bad_words":[""], - "stop_words":[""] - }} - for k,v in optional_params.items(): - data_for_triton["parameters"][k] = v + stream: bool = False, + ) -> ModelResponse: + + type_of_model = "" + if api_base.endswith("generate"): ### This is a trtllm model + text_input = messages[0]["content"] + data_for_triton: Dict[str, Any] = { + "text_input": str(text_input), + "parameters": { + "max_tokens": int(optional_params.get("max_tokens", 20)), + "bad_words": [""], + "stop_words": [""] + } + } + data_for_triton["parameters"].update( optional_params) type_of_model = "trtllm" - elif api_base.endswith("infer"): ### This is a infer model with a custom model on triton - # this is a custom model - text_input = messages[0]["content"] - data_for_triton = { - "inputs": [{"name": "text_input","shape": [1],"datatype": "BYTES","data": [text_input] }] - } - - for k,v in optional_params.items(): - if not (k=="stream" or k=="max_retries"): ## skip these as they are added by litellm - datatype = "INT32" if type(v) == int else "BYTES" - datatype = "FP32" if type(v) == float else datatype - data_for_triton['inputs'].append({"name": k,"shape": [1],"datatype": datatype,"data": [v]}) - - # check for max_tokens which is required - if "max_tokens" not in optional_params: - data_for_triton['inputs'].append({"name": "max_tokens","shape": [1],"datatype": "INT32","data": [20]}) - - type_of_model = "infer" - else: ## Unknown model type passthrough + elif api_base.endswith("infer"): ### This is an infer model with a custom model on triton + text_input = messages[0]["content"] data_for_triton = { - messages[0]["content"] + "inputs": [{"name": "text_input", "shape": [1], "datatype": "BYTES", "data": [text_input]}] } + for k, v in optional_params.items(): + if not (k == "stream" or k == "max_retries"): + datatype = "INT32" if isinstance(v, int) else "BYTES" + datatype = "FP32" if isinstance(v, float) else datatype + data_for_triton['inputs'].append({"name": k, "shape": [1], "datatype": datatype, "data": [v]}) + + if "max_tokens" not in optional_params: + data_for_triton['inputs'].append({"name": "max_tokens", "shape": [1], "datatype": "INT32", "data": [20]}) + + type_of_model = "infer" + else: ## Unknown model type passthrough + data_for_triton = { + "inputs": [{"name": "text_input", "shape": [1], "datatype": "BYTES", "data": [messages[0]["content"]]}] + } + if logging_obj: logging_obj.pre_call( input=messages, @@ -226,35 +176,22 @@ class TritonChatCompletion(BaseLLM): ) handler = requests.Session() handler.timeout = (600.0, 5.0) - + response = handler.post(url=api_base, json=data_for_triton) - if logging_obj: logging_obj.post_call(original_response=response) if response.status_code != 200: raise TritonError(status_code=response.status_code, message=response.text) - _json_response=response.json() + _json_response = response.json() model_response.model = _json_response.get("model_name", "None") if type_of_model == "trtllm": - # The actual response is part of the text_output key in the response - model_response['choices'] = [ Choices(index=0, message= Message(content=_json_response['text_output']))] + model_response.choices = [Choices(index=0, message=Message(content=_json_response['text_output']))] elif type_of_model == "infer": - # The actual response is part of the outputs key in the response - model_response['choices'] = [ Choices(index=0, message= Message(content=_json_response['outputs'][0]['data']))] + model_response.choices = [Choices(index=0, message=Message(content=_json_response['outputs'][0]['data']))] else: - ## just passthrough the response - model_response['choices'] = [ Choices(index=0, message= Message(content=_json_response['outputs']))] - - """ - response = self.acompletion( - data=data_for_triton, - model_response=model_response, - logging_obj=logging_obj, - api_base=api_base, - api_key=api_key, - ) - """ - return model_response \ No newline at end of file + model_response.choices = [Choices(index=0, message=Message(content=_json_response['outputs']))] + + return model_response diff --git a/litellm/main.py b/litellm/main.py index 51da76028e9..d30f2e95d5d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2261,7 +2261,7 @@ def completion( ) model_response = triton_chat_completions.completion( api_base=api_base, - timeout=timeout, + timeout=timeout, # type: ignore model=model, messages=messages, model_response=model_response, From 1b3050477af4d8b90f90c42154bcee6c7447e056 Mon Sep 17 00:00:00 2001 From: Giri Tatavarty Date: Wed, 29 May 2024 15:47:23 -0700 Subject: [PATCH 02/99] #added type ignore for httpx and requests --- litellm/llms/triton.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/llms/triton.py b/litellm/llms/triton.py index 626c7c5b415..c681fd07278 100644 --- a/litellm/llms/triton.py +++ b/litellm/llms/triton.py @@ -1,7 +1,7 @@ import os import json from enum import Enum -import requests +import requests # type: ignore import time from typing import Callable, Optional, List, Sequence, Any, Union, Dict from litellm.utils import ModelResponse, Choices, Usage, map_finish_reason, CustomStreamWrapper, Message, EmbeddingResponse @@ -9,8 +9,8 @@ import litellm from .prompt_templates.factory import prompt_factory, custom_prompt from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from .base import BaseLLM -import httpx -from typing import Union,Collection +import httpx # type: ignore + class TritonError(Exception): @@ -163,7 +163,7 @@ class TritonChatCompletion(BaseLLM): data_for_triton = { "inputs": [{"name": "text_input", "shape": [1], "datatype": "BYTES", "data": [messages[0]["content"]]}] } - + if logging_obj: logging_obj.pre_call( input=messages, From d5c65c6be22e1a6e3d2ebbae4f028e57aef58320 Mon Sep 17 00:00:00 2001 From: Sophia Loris Date: Fri, 19 Jul 2024 09:35:27 -0500 Subject: [PATCH 03/99] Add support for Triton streaming & triton async completions --- litellm/llms/triton.py | 186 +++++++++++++++++++++++++++++++++-------- litellm/main.py | 3 + litellm/utils.py | 43 ++++++++++ 3 files changed, 199 insertions(+), 33 deletions(-) diff --git a/litellm/llms/triton.py b/litellm/llms/triton.py index c681fd07278..95cf38f1fc9 100644 --- a/litellm/llms/triton.py +++ b/litellm/llms/triton.py @@ -1,16 +1,24 @@ import os import json from enum import Enum -import requests # type: ignore +import requests # type: ignore import time from typing import Callable, Optional, List, Sequence, Any, Union, Dict -from litellm.utils import ModelResponse, Choices, Usage, map_finish_reason, CustomStreamWrapper, Message, EmbeddingResponse +from litellm.utils import ( + ModelResponse, + Choices, + Delta, + Usage, + map_finish_reason, + CustomStreamWrapper, + Message, + EmbeddingResponse, +) import litellm from .prompt_templates.factory import prompt_factory, custom_prompt -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from .base import BaseLLM -import httpx # type: ignore - +import httpx # type: ignore class TritonError(Exception): @@ -26,6 +34,7 @@ class TritonError(Exception): self.message ) # Call the base class constructor with the parameters it needs + class TritonChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() @@ -127,71 +136,182 @@ class TritonChatCompletion(BaseLLM): optional_params=None, client=None, stream: bool = False, + acompletion: bool = False, ) -> ModelResponse: - type_of_model = "" + optional_params.pop("stream", False) if api_base.endswith("generate"): ### This is a trtllm model text_input = messages[0]["content"] - data_for_triton: Dict[str, Any] = { - "text_input": str(text_input), + data_for_triton: Dict[str, Any] = { + "text_input": prompt_factory(model=model, messages=messages), "parameters": { - "max_tokens": int(optional_params.get("max_tokens", 20)), + "max_tokens": int(optional_params.get("max_tokens", 2000)), "bad_words": [""], - "stop_words": [""] - } + "stop_words": [""], + }, + "stream": bool(stream), } - data_for_triton["parameters"].update( optional_params) + data_for_triton["parameters"].update(optional_params) type_of_model = "trtllm" - elif api_base.endswith("infer"): ### This is an infer model with a custom model on triton + elif api_base.endswith( + "infer" + ): ### This is an infer model with a custom model on triton text_input = messages[0]["content"] data_for_triton = { - "inputs": [{"name": "text_input", "shape": [1], "datatype": "BYTES", "data": [text_input]}] + "inputs": [ + { + "name": "text_input", + "shape": [1], + "datatype": "BYTES", + "data": [text_input], + } + ] } for k, v in optional_params.items(): if not (k == "stream" or k == "max_retries"): datatype = "INT32" if isinstance(v, int) else "BYTES" datatype = "FP32" if isinstance(v, float) else datatype - data_for_triton['inputs'].append({"name": k, "shape": [1], "datatype": datatype, "data": [v]}) + data_for_triton["inputs"].append( + {"name": k, "shape": [1], "datatype": datatype, "data": [v]} + ) if "max_tokens" not in optional_params: - data_for_triton['inputs'].append({"name": "max_tokens", "shape": [1], "datatype": "INT32", "data": [20]}) + data_for_triton["inputs"].append( + { + "name": "max_tokens", + "shape": [1], + "datatype": "INT32", + "data": [20], + } + ) type_of_model = "infer" else: ## Unknown model type passthrough data_for_triton = { - "inputs": [{"name": "text_input", "shape": [1], "datatype": "BYTES", "data": [messages[0]["content"]]}] + "inputs": [ + { + "name": "text_input", + "shape": [1], + "datatype": "BYTES", + "data": [messages[0]["content"]], + } + ] } if logging_obj: logging_obj.pre_call( - input=messages, - api_key=api_key, - additional_args={ - "complete_input_dict": optional_params, - "api_base": api_base, - "http_client": client, - }, - ) - handler = requests.Session() - handler.timeout = (600.0, 5.0) + input=messages, + api_key=api_key, + additional_args={ + "complete_input_dict": optional_params, + "api_base": api_base, + "http_client": client, + }, + ) - response = handler.post(url=api_base, json=data_for_triton) + headers = {"Content-Type": "application/json"} + data_for_triton = json.dumps(data_for_triton) + if acompletion: + return self.acompletion( + model, + data_for_triton, + headers=headers, + logging_obj=logging_obj, + api_base=api_base, + stream=stream, + model_response=model_response, + type_of_model=type_of_model, + ) + else: + handler = HTTPHandler() + if stream: + return self._handle_stream( + handler, api_base, data_for_triton, model, logging_obj + ) + else: + response = handler.post(url=api_base, data=data_for_triton, headers=headers) + return self._handle_response( + response, model_response, logging_obj, type_of_model=type_of_model + ) + + async def acompletion( + self, + model: str, + data_for_triton, + api_base, + stream, + logging_obj, + headers, + model_response, + type_of_model, + ) -> ModelResponse: + handler = AsyncHTTPHandler() + if stream: + return self._ahandle_stream( + handler, api_base, data_for_triton, model, logging_obj + ) + else: + response = await handler.post( + url=api_base, data=data_for_triton, headers=headers + ) + + return self._handle_response( + response, model_response, logging_obj, type_of_model=type_of_model + ) + + def _handle_stream(self, handler, api_base, data_for_triton, model, logging_obj): + response = handler.post( + url=api_base + "_stream", data=data_for_triton, stream=True + ) + streamwrapper = litellm.CustomStreamWrapper( + response.iter_lines(), + model=model, + custom_llm_provider="triton", + logging_obj=logging_obj, + ) + for chunk in streamwrapper: + yield (chunk) + + async def _ahandle_stream( + self, handler, api_base, data_for_triton, model, logging_obj + ): + response = await handler.post( + url=api_base + "_stream", data=data_for_triton, stream=True + ) + streamwrapper = litellm.CustomStreamWrapper( + response.aiter_lines(), + model=model, + custom_llm_provider="triton", + logging_obj=logging_obj, + ) + async for chunk in streamwrapper: + yield (chunk) + + def _handle_response(self, response, model_response, logging_obj, type_of_model): if logging_obj: logging_obj.post_call(original_response=response) if response.status_code != 200: raise TritonError(status_code=response.status_code, message=response.text) - _json_response = response.json() + _json_response = response.json() model_response.model = _json_response.get("model_name", "None") if type_of_model == "trtllm": - model_response.choices = [Choices(index=0, message=Message(content=_json_response['text_output']))] + model_response.choices = [ + Choices(index=0, message=Message(content=_json_response["text_output"])) + ] elif type_of_model == "infer": - model_response.choices = [Choices(index=0, message=Message(content=_json_response['outputs'][0]['data']))] + model_response.choices = [ + Choices( + index=0, + message=Message(content=_json_response["outputs"][0]["data"]), + ) + ] else: - model_response.choices = [Choices(index=0, message=Message(content=_json_response['outputs']))] - + model_response.choices = [ + Choices(index=0, message=Message(content=_json_response["outputs"])) + ] return model_response diff --git a/litellm/main.py b/litellm/main.py index d30f2e95d5d..06d1abf829a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -333,6 +333,7 @@ async def acompletion( or custom_llm_provider == "predibase" or custom_llm_provider == "bedrock" or custom_llm_provider == "databricks" + or custom_llm_provider == "triton" or custom_llm_provider in litellm.openai_compatible_providers ): # currently implemented aiohttp calls for just azure, openai, hf, ollama, vertex ai soon all. init_response = await loop.run_in_executor(None, func_with_context) @@ -2267,6 +2268,8 @@ def completion( model_response=model_response, optional_params=optional_params, logging_obj=logging, + stream=stream, + acompletion=acompletion ) ## RESPONSE OBJECT diff --git a/litellm/utils.py b/litellm/utils.py index ea0f46c144b..64964364c9a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -11013,6 +11013,42 @@ class CustomStreamWrapper: except Exception as e: raise e + def handle_triton_stream(self, chunk): + try: + if isinstance(chunk, dict): + parsed_response = chunk + elif isinstance(chunk, (str, bytes)): + if isinstance(chunk, bytes): + chunk = chunk.decode("utf-8") + if "text_output" in chunk: + response = chunk.replace("data: ", "").strip() + parsed_response = json.loads(response) + else: + return { + "text": "", + "is_finished": False, + "prompt_tokens": 0, + "completion_tokens": 0, + } + else: + print_verbose(f"chunk: {chunk} (Type: {type(chunk)})") + raise ValueError( + f"Unable to parse response. Original response: {chunk}" + ) + text = parsed_response.get("text_output", "") + finish_reason = parsed_response.get("stop_reason") + is_finished = parsed_response.get("is_finished", False) + return { + "text": text, + "is_finished": is_finished, + "finish_reason": finish_reason, + "prompt_tokens": parsed_response.get("input_token_count", 0), + "completion_tokens": parsed_response.get("generated_token_count", 0), + } + return {"text": "", "is_finished": False} + except Exception as e: + raise e + def handle_clarifai_completion_chunk(self, chunk): try: if isinstance(chunk, dict): @@ -11337,6 +11373,12 @@ class CustomStreamWrapper: completion_obj["content"] = response_obj["text"] if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] + elif self.custom_llm_provider == "triton": + response_obj = self.handle_triton_stream(chunk) + completion_obj["content"] = response_obj["text"] + print_verbose(f"completion obj content: {completion_obj['content']}") + if response_obj["is_finished"]: + self.received_finish_reason = response_obj["finish_reason"] elif self.custom_llm_provider == "text-completion-openai": response_obj = self.handle_openai_text_completion_chunk(chunk) completion_obj["content"] = response_obj["text"] @@ -11773,6 +11815,7 @@ class CustomStreamWrapper: or self.custom_llm_provider == "predibase" or self.custom_llm_provider == "databricks" or self.custom_llm_provider == "bedrock" + or self.custom_llm_provider == "triton" or self.custom_llm_provider in litellm.openai_compatible_endpoints ): async for chunk in self.completion_stream: From 8b3c8102a719108eb8881563f5306bbe8ff6064e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 20 Jul 2024 18:39:05 -0700 Subject: [PATCH 04/99] feat(auth_checks.py): Allow admin to disable team from turning on/off guardrails. --- litellm/proxy/_types.py | 1 + litellm/proxy/auth/auth_checks.py | 17 ++++++ litellm/proxy/auth/user_api_key_auth.py | 1 + litellm/proxy/guardrails/guardrail_helpers.py | 19 ++++++- .../management_endpoints/team_endpoints.py | 1 + litellm/proxy/utils.py | 1 + litellm/tests/test_proxy_server.py | 57 +++++++++++++++++++ 7 files changed, 96 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e9371c1d8d9..feaa54cd4cd 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1232,6 +1232,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): soft_budget: Optional[float] = None team_model_aliases: Optional[Dict] = None team_member_spend: Optional[float] = None + team_metadata: Optional[Dict] = None # End User Params end_user_id: Optional[str] = None diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 96171f2efb7..f44485b6be8 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -57,6 +57,7 @@ def common_checks( 4. If end_user (either via JWT or 'user' passed to /chat/completions, /embeddings endpoint) is in budget 5. [OPTIONAL] If 'enforce_end_user' enabled - did developer pass in 'user' param for openai endpoints 6. [OPTIONAL] If 'litellm.max_budget' is set (>0), is proxy under budget + 7. [OPTIONAL] If guardrails modified - is request allowed to change this """ _model = request_body.get("model", None) if team_object is not None and team_object.blocked is True: @@ -158,6 +159,22 @@ def common_checks( raise litellm.BudgetExceededError( current_cost=global_proxy_spend, max_budget=litellm.max_budget ) + + _request_metadata: dict = request_body.get("metadata", {}) or {} + if _request_metadata.get("guardrails"): + # check if team allowed to modify guardrails + from litellm.proxy.guardrails.guardrail_helpers import can_modify_guardrails + + can_modify: bool = can_modify_guardrails(team_object) + if can_modify is False: + from fastapi import HTTPException + + raise HTTPException( + status_code=403, + detail={ + "error": "Your team does not have permission to modify guardrails." + }, + ) return True diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index c5549ffcb66..82425e418e2 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -924,6 +924,7 @@ async def user_api_key_auth( rpm_limit=valid_token.team_rpm_limit, blocked=valid_token.team_blocked, models=valid_token.team_models, + metadata=valid_token.team_metadata, ) user_api_key_cache.set_cache( diff --git a/litellm/proxy/guardrails/guardrail_helpers.py b/litellm/proxy/guardrails/guardrail_helpers.py index d6a081b4d54..e0a5f1eb3d9 100644 --- a/litellm/proxy/guardrails/guardrail_helpers.py +++ b/litellm/proxy/guardrails/guardrail_helpers.py @@ -1,9 +1,26 @@ +from typing import Dict + import litellm from litellm._logging import verbose_proxy_logger -from litellm.proxy.proxy_server import UserAPIKeyAuth +from litellm.proxy.proxy_server import LiteLLM_TeamTable, UserAPIKeyAuth from litellm.types.guardrails import * +def can_modify_guardrails(team_obj: Optional[LiteLLM_TeamTable]) -> bool: + if team_obj is None: + return True + + team_metadata = team_obj.metadata or {} + + if team_metadata.get("guardrails", None) is not None and isinstance( + team_metadata.get("guardrails"), Dict + ): + if team_metadata.get("guardrails", {}).get("modify_guardrails", None) is False: + return False + + return True + + async def should_proceed_based_on_metadata(data: dict, guardrail_name: str) -> bool: """ checks if this guardrail should be applied to this call diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index bb98a02ec3e..9ba76a20328 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -363,6 +363,7 @@ async def update_team( # set the budget_reset_at in DB updated_kv["budget_reset_at"] = reset_at + updated_kv = prisma_client.jsonify_object(data=updated_kv) team_row: Optional[ LiteLLM_TeamTable ] = await prisma_client.db.litellm_teamtable.update( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0f87e962abc..9cdfebf81f4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1315,6 +1315,7 @@ class PrismaClient: t.models AS team_models, t.blocked AS team_blocked, t.team_alias AS team_alias, + t.metadata AS team_metadata, tm.spend AS team_member_spend, m.aliases as team_model_aliases FROM "LiteLLM_VerificationToken" AS v diff --git a/litellm/tests/test_proxy_server.py b/litellm/tests/test_proxy_server.py index ed7451c27e6..f3cb69a082f 100644 --- a/litellm/tests/test_proxy_server.py +++ b/litellm/tests/test_proxy_server.py @@ -173,6 +173,63 @@ def test_chat_completion(mock_acompletion, client_no_auth): pytest.fail(f"LiteLLM Proxy test failed. Exception - {str(e)}") +@mock_patch_acompletion() +@pytest.mark.asyncio +async def test_team_disable_guardrails(mock_acompletion, client_no_auth): + """ + If team not allowed to turn on/off guardrails + + Raise 403 forbidden error, if request is made by team on `/key/generate` or `/chat/completions`. + """ + import asyncio + import json + import time + + from fastapi import HTTPException, Request + from starlette.datastructures import URL + + from litellm.proxy._types import LiteLLM_TeamTable, ProxyException, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import hash_token, user_api_key_cache + + _team_id = "1234" + user_key = "sk-12345678" + + valid_token = UserAPIKeyAuth( + team_id=_team_id, + team_blocked=True, + token=hash_token(user_key), + last_refreshed_at=time.time(), + ) + await asyncio.sleep(1) + team_obj = LiteLLM_TeamTable( + team_id=_team_id, + blocked=False, + last_refreshed_at=time.time(), + metadata={"guardrails": {"modify_guardrails": False}}, + ) + user_api_key_cache.set_cache(key=hash_token(user_key), value=valid_token) + user_api_key_cache.set_cache(key="team_id:{}".format(_team_id), value=team_obj) + + setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache) + setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") + setattr(litellm.proxy.proxy_server, "prisma_client", "hello-world") + + request = Request(scope={"type": "http"}) + request._url = URL(url="/chat/completions") + + body = {"metadata": {"guardrails": {"hide_secrets": False}}} + json_bytes = json.dumps(body).encode("utf-8") + + request._body = json_bytes + + try: + await user_api_key_auth(request=request, api_key="Bearer " + user_key) + pytest.fail("Expected to raise 403 forbidden error.") + except ProxyException as e: + assert e.code == 403 + + from litellm.tests.test_custom_callback_input import CompletionCustomHandler From d7556020b3b430ef60dffa7a5c3599b86f672d4d Mon Sep 17 00:00:00 2001 From: Wanis Elabbar <70503629+elabbarw@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:45:59 +0100 Subject: [PATCH 05/99] Fix errors with docker-compose file The Docker Compose file is causing an error during the healthcheck, stating "cannot find role 'account used to run compose'". I've modified the file to set a database, username, and password, and ensured the database and username are configured correctly in the healthcheck. --- docker-compose.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ca98ec784d2..be84462ef00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "4000:4000" # Map the container port to the host, change the host port if necessary environment: - DATABASE_URL: "postgresql://postgres:example@db:5432/postgres" + DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI env_file: - .env # Load local .env file @@ -25,11 +25,13 @@ services: image: postgres restart: always environment: - POSTGRES_PASSWORD: example + POSTGRES_DB: litellm + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD: dbpassword9090 healthcheck: - test: ["CMD-SHELL", "pg_isready"] + test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"] interval: 1s timeout: 5s retries: 10 -# ...rest of your docker-compose config if any \ No newline at end of file +# ...rest of your docker-compose config if any From c7f72cbbdedd0c58b4df4c49826d665813ea5816 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 10:58:20 -0700 Subject: [PATCH 06/99] feat - add support to init arize ai --- litellm/litellm_core_utils/litellm_logging.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 32633960f01..e78eb579327 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1954,6 +1954,43 @@ def _init_custom_logger_compatible_class( _langsmith_logger = LangsmithLogger() _in_memory_loggers.append(_langsmith_logger) return _langsmith_logger # type: ignore + elif logging_integration == "arize": + if "ARIZE_SPACE_KEY" not in os.environ: + raise ValueError("ARIZE_SPACE_KEY not found in environment variables") + if "ARIZE_API_KEY" not in os.environ: + raise ValueError("ARIZE_API_KEY not found in environment variables") + from litellm.integrations.opentelemetry import ( + OpenTelemetry, + OpenTelemetryConfig, + ) + + otel_config = OpenTelemetryConfig( + exporter="otlp_grpc", + endpoint="https://otlp.arize.com/v1", + ) + os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = ( + f"space_key={os.getenv('ARIZE_SPACE_KEY')},api_key={os.getenv('ARIZE_API_KEY')}" + ) + for callback in _in_memory_loggers: + if ( + isinstance(callback, OpenTelemetry) + and callback.callback_name == "arize" + ): + return callback # type: ignore + _otel_logger = OpenTelemetry(config=otel_config, callback_name="arize") + _in_memory_loggers.append(_otel_logger) + return _otel_logger # type: ignore + + elif logging_integration == "otel": + from litellm.integrations.opentelemetry import OpenTelemetry + + for callback in _in_memory_loggers: + if isinstance(callback, OpenTelemetry): + return callback # type: ignore + + otel_logger = OpenTelemetry() + _in_memory_loggers.append(otel_logger) + return otel_logger # type: ignore elif logging_integration == "galileo": for callback in _in_memory_loggers: @@ -2027,6 +2064,25 @@ def get_custom_logger_compatible_class( for callback in _in_memory_loggers: if isinstance(callback, LangsmithLogger): return callback + elif logging_integration == "otel": + from litellm.integrations.opentelemetry import OpenTelemetry + + for callback in _in_memory_loggers: + if isinstance(callback, OpenTelemetry): + return callback + elif logging_integration == "arize": + from litellm.integrations.opentelemetry import OpenTelemetry + + if "ARIZE_SPACE_KEY" not in os.environ: + raise ValueError("ARIZE_SPACE_KEY not found in environment variables") + if "ARIZE_API_KEY" not in os.environ: + raise ValueError("ARIZE_API_KEY not found in environment variables") + for callback in _in_memory_loggers: + if ( + isinstance(callback, OpenTelemetry) + and callback.callback_name == "arize" + ): + return callback elif logging_integration == "logfire": if "LOGFIRE_TOKEN" not in os.environ: raise ValueError("LOGFIRE_TOKEN not found in environment variables") From 68f8fe87e4a33a15c5ebd625013b24a9ce03e433 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 11:07:48 -0700 Subject: [PATCH 07/99] feat - arize ai open inference types --- litellm/integrations/_types/open_inference.py | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 litellm/integrations/_types/open_inference.py diff --git a/litellm/integrations/_types/open_inference.py b/litellm/integrations/_types/open_inference.py new file mode 100644 index 00000000000..bcfabe9b7b1 --- /dev/null +++ b/litellm/integrations/_types/open_inference.py @@ -0,0 +1,286 @@ +from enum import Enum + + +class SpanAttributes: + OUTPUT_VALUE = "output.value" + OUTPUT_MIME_TYPE = "output.mime_type" + """ + The type of output.value. If unspecified, the type is plain text by default. + If type is JSON, the value is a string representing a JSON object. + """ + INPUT_VALUE = "input.value" + INPUT_MIME_TYPE = "input.mime_type" + """ + The type of input.value. If unspecified, the type is plain text by default. + If type is JSON, the value is a string representing a JSON object. + """ + + EMBEDDING_EMBEDDINGS = "embedding.embeddings" + """ + A list of objects containing embedding data, including the vector and represented piece of text. + """ + EMBEDDING_MODEL_NAME = "embedding.model_name" + """ + The name of the embedding model. + """ + + LLM_FUNCTION_CALL = "llm.function_call" + """ + For models and APIs that support function calling. Records attributes such as the function + name and arguments to the called function. + """ + LLM_INVOCATION_PARAMETERS = "llm.invocation_parameters" + """ + Invocation parameters passed to the LLM or API, such as the model name, temperature, etc. + """ + LLM_INPUT_MESSAGES = "llm.input_messages" + """ + Messages provided to a chat API. + """ + LLM_OUTPUT_MESSAGES = "llm.output_messages" + """ + Messages received from a chat API. + """ + LLM_MODEL_NAME = "llm.model_name" + """ + The name of the model being used. + """ + LLM_PROMPTS = "llm.prompts" + """ + Prompts provided to a completions API. + """ + LLM_PROMPT_TEMPLATE = "llm.prompt_template.template" + """ + The prompt template as a Python f-string. + """ + LLM_PROMPT_TEMPLATE_VARIABLES = "llm.prompt_template.variables" + """ + A list of input variables to the prompt template. + """ + LLM_PROMPT_TEMPLATE_VERSION = "llm.prompt_template.version" + """ + The version of the prompt template being used. + """ + LLM_TOKEN_COUNT_PROMPT = "llm.token_count.prompt" + """ + Number of tokens in the prompt. + """ + LLM_TOKEN_COUNT_COMPLETION = "llm.token_count.completion" + """ + Number of tokens in the completion. + """ + LLM_TOKEN_COUNT_TOTAL = "llm.token_count.total" + """ + Total number of tokens, including both prompt and completion. + """ + + TOOL_NAME = "tool.name" + """ + Name of the tool being used. + """ + TOOL_DESCRIPTION = "tool.description" + """ + Description of the tool's purpose, typically used to select the tool. + """ + TOOL_PARAMETERS = "tool.parameters" + """ + Parameters of the tool represented a dictionary JSON string, e.g. + see https://platform.openai.com/docs/guides/gpt/function-calling + """ + + RETRIEVAL_DOCUMENTS = "retrieval.documents" + + METADATA = "metadata" + """ + Metadata attributes are used to store user-defined key-value pairs. + For example, LangChain uses metadata to store user-defined attributes for a chain. + """ + + TAG_TAGS = "tag.tags" + """ + Custom categorical tags for the span. + """ + + OPENINFERENCE_SPAN_KIND = "openinference.span.kind" + + SESSION_ID = "session.id" + """ + The id of the session + """ + USER_ID = "user.id" + """ + The id of the user + """ + + +class MessageAttributes: + """ + Attributes for a message sent to or from an LLM + """ + + MESSAGE_ROLE = "message.role" + """ + The role of the message, such as "user", "agent", "function". + """ + MESSAGE_CONTENT = "message.content" + """ + The content of the message to or from the llm, must be a string. + """ + MESSAGE_CONTENTS = "message.contents" + """ + The message contents to the llm, it is an array of + `message_content` prefixed attributes. + """ + MESSAGE_NAME = "message.name" + """ + The name of the message, often used to identify the function + that was used to generate the message. + """ + MESSAGE_TOOL_CALLS = "message.tool_calls" + """ + The tool calls generated by the model, such as function calls. + """ + MESSAGE_FUNCTION_CALL_NAME = "message.function_call_name" + """ + The function name that is a part of the message list. + This is populated for role 'function' or 'agent' as a mechanism to identify + the function that was called during the execution of a tool. + """ + MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON = "message.function_call_arguments_json" + """ + The JSON string representing the arguments passed to the function + during a function call. + """ + + +class MessageContentAttributes: + """ + Attributes for the contents of user messages sent to an LLM. + """ + + MESSAGE_CONTENT_TYPE = "message_content.type" + """ + The type of the content, such as "text" or "image". + """ + MESSAGE_CONTENT_TEXT = "message_content.text" + """ + The text content of the message, if the type is "text". + """ + MESSAGE_CONTENT_IMAGE = "message_content.image" + """ + The image content of the message, if the type is "image". + An image can be made available to the model by passing a link to + the image or by passing the base64 encoded image directly in the + request. + """ + + +class ImageAttributes: + """ + Attributes for images + """ + + IMAGE_URL = "image.url" + """ + An http or base64 image url + """ + + +class DocumentAttributes: + """ + Attributes for a document. + """ + + DOCUMENT_ID = "document.id" + """ + The id of the document. + """ + DOCUMENT_SCORE = "document.score" + """ + The score of the document + """ + DOCUMENT_CONTENT = "document.content" + """ + The content of the document. + """ + DOCUMENT_METADATA = "document.metadata" + """ + The metadata of the document represented as a dictionary + JSON string, e.g. `"{ 'title': 'foo' }"` + """ + + +class RerankerAttributes: + """ + Attributes for a reranker + """ + + RERANKER_INPUT_DOCUMENTS = "reranker.input_documents" + """ + List of documents as input to the reranker + """ + RERANKER_OUTPUT_DOCUMENTS = "reranker.output_documents" + """ + List of documents as output from the reranker + """ + RERANKER_QUERY = "reranker.query" + """ + Query string for the reranker + """ + RERANKER_MODEL_NAME = "reranker.model_name" + """ + Model name of the reranker + """ + RERANKER_TOP_K = "reranker.top_k" + """ + Top K parameter of the reranker + """ + + +class EmbeddingAttributes: + """ + Attributes for an embedding + """ + + EMBEDDING_TEXT = "embedding.text" + """ + The text represented by the embedding. + """ + EMBEDDING_VECTOR = "embedding.vector" + """ + The embedding vector. + """ + + +class ToolCallAttributes: + """ + Attributes for a tool call + """ + + TOOL_CALL_FUNCTION_NAME = "tool_call.function.name" + """ + The name of function that is being called during a tool call. + """ + TOOL_CALL_FUNCTION_ARGUMENTS_JSON = "tool_call.function.arguments" + """ + The JSON string representing the arguments passed to the function + during a tool call. + """ + + +class OpenInferenceSpanKindValues(Enum): + TOOL = "TOOL" + CHAIN = "CHAIN" + LLM = "LLM" + RETRIEVER = "RETRIEVER" + EMBEDDING = "EMBEDDING" + AGENT = "AGENT" + RERANKER = "RERANKER" + UNKNOWN = "UNKNOWN" + GUARDRAIL = "GUARDRAIL" + EVALUATOR = "EVALUATOR" + + +class OpenInferenceMimeTypeValues(Enum): + TEXT = "text/plain" + JSON = "application/json" From 1a5aaee3862c4d0b34d9351ea30e8687efc786cf Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 11:27:51 -0700 Subject: [PATCH 08/99] add arize.py --- litellm/integrations/arize_ai.py | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 litellm/integrations/arize_ai.py diff --git a/litellm/integrations/arize_ai.py b/litellm/integrations/arize_ai.py new file mode 100644 index 00000000000..d542e50c9a5 --- /dev/null +++ b/litellm/integrations/arize_ai.py @@ -0,0 +1,79 @@ +""" +arize AI is OTEL compatible + +this file has Arize ai specific helper functions +""" + +from typing import TYPE_CHECKING, Any, Optional, Union + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span = _Span +else: + Span = Any + + +def set_arize_ai_attributes(span: Span, kwargs, response_obj): + from litellm.integrations._types.open_inference import ( + MessageAttributes, + MessageContentAttributes, + SpanAttributes, + ) + + optional_params = kwargs.get("optional_params", {}) + litellm_params = kwargs.get("litellm_params", {}) or {} + + ############################################# + ############ LLM CALL METADATA ############## + ############################################# + metadata = litellm_params.get("metadata", {}) or {} + span.set_attribute(SpanAttributes.METADATA, str(metadata)) + + ############################################# + ########## LLM Request Attributes ########### + ############################################# + + # The name of the LLM a request is being made to + if kwargs.get("model"): + span.set_attribute(SpanAttributes.LLM_MODEL_NAME, kwargs.get("model")) + + span.set_attribute( + SpanAttributes.OPENINFERENCE_SPAN_KIND, + f"litellm-{str(kwargs.get('call_type', None))}", + ) + span.set_attribute(SpanAttributes.LLM_INPUT_MESSAGES, str(kwargs.get("messages"))) + + # The Generative AI Provider: Azure, OpenAI, etc. + span.set_attribute(SpanAttributes.LLM_INVOCATION_PARAMETERS, str(optional_params)) + + if optional_params.get("user"): + span.set_attribute(SpanAttributes.USER_ID, optional_params.get("user")) + + ############################################# + ########## LLM Response Attributes ########## + ############################################# + llm_output_messages = [] + for choice in response_obj.get("choices"): + llm_output_messages.append(choice.get("message")) + + span.set_attribute(SpanAttributes.LLM_OUTPUT_MESSAGES, str(llm_output_messages)) + usage = response_obj.get("usage") + if usage: + span.set_attribute( + SpanAttributes.LLM_TOKEN_COUNT_TOTAL, + usage.get("total_tokens"), + ) + + # The number of tokens used in the LLM response (completion). + span.set_attribute( + SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, + usage.get("completion_tokens"), + ) + + # The number of tokens used in the LLM prompt. + span.set_attribute( + SpanAttributes.LLM_TOKEN_COUNT_PROMPT, + usage.get("prompt_tokens"), + ) + pass From cdc8b4f0374429013493036c1104313715ef7167 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 11:30:24 -0700 Subject: [PATCH 09/99] test - arize ai basic logging --- litellm/tests/test_arize_ai.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 litellm/tests/test_arize_ai.py diff --git a/litellm/tests/test_arize_ai.py b/litellm/tests/test_arize_ai.py new file mode 100644 index 00000000000..7c38db4c602 --- /dev/null +++ b/litellm/tests/test_arize_ai.py @@ -0,0 +1,29 @@ +import asyncio +import logging +import os +import time + +import pytest +from dotenv import load_dotenv +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +import litellm +from litellm._logging import verbose_logger +from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig + +load_dotenv() +import logging + + +@pytest.mark.asyncio() +async def test_async_otel_callback(): + litellm.set_verbose = True + litellm.callbacks = ["arize"] + + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi test from local arize"}], + mock_response="hello", + temperature=0.1, + user="OTEL_USER", + ) From 0436eba2fa32a919fc17cebf5138f6de3ed6d48b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 13:40:42 -0700 Subject: [PATCH 10/99] otel - log to arize ai --- litellm/__init__.py | 8 ++++++- litellm/integrations/opentelemetry.py | 31 +++++++++++++++++---------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 7dcc934a683..4283f4586bc 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -38,7 +38,13 @@ success_callback: List[Union[str, Callable]] = [] failure_callback: List[Union[str, Callable]] = [] service_callback: List[Union[str, Callable]] = [] _custom_logger_compatible_callbacks_literal = Literal[ - "lago", "openmeter", "logfire", "dynamic_rate_limiter", "langsmith", "galileo" + "lago", + "openmeter", + "logfire", + "dynamic_rate_limiter", + "langsmith", + "galileo", + "arize", ] callbacks: List[Union[Callable, _custom_logger_compatible_callbacks_literal]] = [] _langfuse_default_tags: Optional[ diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 215a4f09f83..bc58efad318 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2,7 +2,7 @@ import os from dataclasses import dataclass from datetime import datetime from functools import wraps -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union import litellm from litellm._logging import verbose_logger @@ -27,9 +27,10 @@ else: LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm") -LITELLM_RESOURCE = { +LITELLM_RESOURCE: Dict[Any, Any] = { "service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"), "deployment.environment": os.getenv("OTEL_ENVIRONMENT_NAME", "production"), + "model_id": os.getenv("OTEL_SERVICE_NAME", "litellm"), } RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" LITELLM_REQUEST_SPAN_NAME = "litellm_request" @@ -68,7 +69,9 @@ class OpenTelemetryConfig: class OpenTelemetry(CustomLogger): - def __init__(self, config=OpenTelemetryConfig.from_env()): + def __init__( + self, config=OpenTelemetryConfig.from_env(), callback_name: Optional[str] = None + ): from opentelemetry import trace from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider @@ -79,6 +82,7 @@ class OpenTelemetry(CustomLogger): self.OTEL_HEADERS = self.config.headers provider = TracerProvider(resource=Resource(attributes=LITELLM_RESOURCE)) provider.add_span_processor(self._get_span_processor()) + self.callback_name = callback_name trace.set_tracer_provider(provider) self.tracer = trace.get_tracer(LITELLM_TRACER_NAME) @@ -120,8 +124,8 @@ class OpenTelemetry(CustomLogger): from opentelemetry import trace from opentelemetry.trace import Status, StatusCode - _start_time_ns = start_time - _end_time_ns = end_time + _start_time_ns = 0 + _end_time_ns = 0 if isinstance(start_time, float): _start_time_ns = int(int(start_time) * 1e9) @@ -159,8 +163,8 @@ class OpenTelemetry(CustomLogger): from opentelemetry import trace from opentelemetry.trace import Status, StatusCode - _start_time_ns = start_time - _end_time_ns = end_time + _start_time_ns = 0 + _end_time_ns = 0 if isinstance(start_time, float): _start_time_ns = int(int(start_time) * 1e9) @@ -294,6 +298,11 @@ class OpenTelemetry(CustomLogger): return isinstance(value, (str, bool, int, float)) def set_attributes(self, span: Span, kwargs, response_obj): + if self.callback_name == "arize": + from litellm.integrations.arize_ai import set_arize_ai_attributes + + set_arize_ai_attributes(span, kwargs, response_obj) + return from litellm.proxy._types import SpanAttributes optional_params = kwargs.get("optional_params", {}) @@ -612,8 +621,8 @@ class OpenTelemetry(CustomLogger): from opentelemetry import trace from opentelemetry.trace import Status, StatusCode - _start_time_ns = logging_payload.start_time - _end_time_ns = logging_payload.end_time + _start_time_ns = 0 + _end_time_ns = 0 start_time = logging_payload.start_time end_time = logging_payload.end_time @@ -658,8 +667,8 @@ class OpenTelemetry(CustomLogger): from opentelemetry import trace from opentelemetry.trace import Status, StatusCode - _start_time_ns = logging_payload.start_time - _end_time_ns = logging_payload.end_time + _start_time_ns = 0 + _end_time_ns = 0 start_time = logging_payload.start_time end_time = logging_payload.end_time From 41e6c4a573d05bf60c0864bb288deb3da5f4bc9b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 13:47:58 -0700 Subject: [PATCH 11/99] feat - arize ai log llm i/o --- litellm/integrations/arize_ai.py | 51 +++++++++++++++++++++++++++----- litellm/proxy/proxy_config.yaml | 15 ++++++---- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/litellm/integrations/arize_ai.py b/litellm/integrations/arize_ai.py index d542e50c9a5..45c6c160430 100644 --- a/litellm/integrations/arize_ai.py +++ b/litellm/integrations/arize_ai.py @@ -18,6 +18,7 @@ def set_arize_ai_attributes(span: Span, kwargs, response_obj): from litellm.integrations._types.open_inference import ( MessageAttributes, MessageContentAttributes, + OpenInferenceSpanKindValues, SpanAttributes, ) @@ -27,8 +28,9 @@ def set_arize_ai_attributes(span: Span, kwargs, response_obj): ############################################# ############ LLM CALL METADATA ############## ############################################# - metadata = litellm_params.get("metadata", {}) or {} - span.set_attribute(SpanAttributes.METADATA, str(metadata)) + # commented out for now - looks like Arize AI could not log this + # metadata = litellm_params.get("metadata", {}) or {} + # span.set_attribute(SpanAttributes.METADATA, str(metadata)) ############################################# ########## LLM Request Attributes ########### @@ -39,10 +41,30 @@ def set_arize_ai_attributes(span: Span, kwargs, response_obj): span.set_attribute(SpanAttributes.LLM_MODEL_NAME, kwargs.get("model")) span.set_attribute( - SpanAttributes.OPENINFERENCE_SPAN_KIND, - f"litellm-{str(kwargs.get('call_type', None))}", + SpanAttributes.OPENINFERENCE_SPAN_KIND, OpenInferenceSpanKindValues.LLM.value ) - span.set_attribute(SpanAttributes.LLM_INPUT_MESSAGES, str(kwargs.get("messages"))) + messages = kwargs.get("messages") + + # for /chat/completions + # https://docs.arize.com/arize/large-language-models/tracing/semantic-conventions + if messages: + span.set_attribute( + SpanAttributes.INPUT_VALUE, + messages[-1].get("content", ""), # get the last message for input + ) + + # LLM_INPUT_MESSAGES shows up under `input_messages` tab on the span page + for idx, msg in enumerate(messages): + # Set the role per message + span.set_attribute( + f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}.{MessageAttributes.MESSAGE_ROLE}", + msg["role"], + ) + # Set the content per message + span.set_attribute( + f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}.{MessageAttributes.MESSAGE_CONTENT}", + msg.get("content", ""), + ) # The Generative AI Provider: Azure, OpenAI, etc. span.set_attribute(SpanAttributes.LLM_INVOCATION_PARAMETERS, str(optional_params)) @@ -52,12 +74,25 @@ def set_arize_ai_attributes(span: Span, kwargs, response_obj): ############################################# ########## LLM Response Attributes ########## + # https://docs.arize.com/arize/large-language-models/tracing/semantic-conventions ############################################# - llm_output_messages = [] for choice in response_obj.get("choices"): - llm_output_messages.append(choice.get("message")) + response_message = choice.get("message", {}) + span.set_attribute( + SpanAttributes.OUTPUT_VALUE, response_message.get("content", "") + ) + + # This shows up under `output_messages` tab on the span page + # This code assumes a single response + span.set_attribute( + f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_ROLE}", + response_message["role"], + ) + span.set_attribute( + f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}", + response_message.get("content", ""), + ) - span.set_attribute(SpanAttributes.LLM_OUTPUT_MESSAGES, str(llm_output_messages)) usage = response_obj.get("usage") if usage: span.set_attribute( diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index c114db25f5a..2508a48a1df 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,10 +1,15 @@ model_list: + - model_name: gpt-4 + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ - model_name: fireworks-llama-v3-70b-instruct litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct - api_key: "os.environ/FIREWORKS_AI_API_KEY" - -router_settings: - enable_tag_filtering: True # ๐Ÿ‘ˆ Key Change + api_key: "os.environ/FIREWORKS" general_settings: - master_key: sk-1234 \ No newline at end of file + master_key: sk-1234 + +litellm_settings: + callbacks: ["arize"] \ No newline at end of file From 2f65c950fea232bac4093ffe9e6c0b368ab15724 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 14:00:33 -0700 Subject: [PATCH 12/99] fix(vertex_httpx.py): Change non-blocking vertex error to warning Fixes https://github.com/BerriAI/litellm/issues/4825 --- litellm/llms/vertex_httpx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index 03b0a56ccd7..a8de79affc8 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -1033,7 +1033,7 @@ class VertexLLM(BaseLLM): model=model, custom_llm_provider=_custom_llm_provider ) except Exception as e: - verbose_logger.error( + verbose_logger.warning( "Unable to identify if system message supported. Defaulting to 'False'. Received error message - {}\nAdd it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json".format( str(e) ) From 488aca98a13e36db2c2db38f1a9a69c138b7ba6d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 14:27:45 -0700 Subject: [PATCH 13/99] doc arize ai --- .../docs/observability/arize_integration.md | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/my-website/docs/observability/arize_integration.md diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md new file mode 100644 index 00000000000..d2592da6abd --- /dev/null +++ b/docs/my-website/docs/observability/arize_integration.md @@ -0,0 +1,72 @@ +import Image from '@theme/IdealImage'; + +# ๐Ÿ”ฅ Arize AI - Logging LLM Input/Output + +AI Observability and Evaluation Platform + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + + +## Pre-Requisites +Make an account on [Arize AI](https://app.arize.com/auth/login) + +## Quick Start +Use just 2 lines of code, to instantly log your responses **across all providers** with arize + + +```python +litellm.callbacks = ["arize"] +``` +```python +import litellm +import os + +os.environ["ARIZE_SPACE_KEY"] = "" +os.environ["ARIZE_API_KEY"] = "" # defaults to litellm-completion + +# LLM API Keys +os.environ['OPENAI_API_KEY']="" + +# set arize as a callback, litellm will send the data to arize +litellm.callbacks = ["arize"] + +# openai call +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Hi ๐Ÿ‘‹ - i'm openai"} + ] +) +``` + +### Using with LiteLLM Proxy + + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +litellm_settings: + callbacks: ["arize"] + +environment_variables: + ARIZE_SPACE_KEY: "d0*****" + ARIZE_API_KEY: "141a****" +``` + +## Support & Talk to Founders + +- [Schedule Demo ๐Ÿ‘‹](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) +- [Community Discord ๐Ÿ’ญ](https://discord.gg/wuPM9dRgDw) +- Our numbers ๐Ÿ“ž +1 (770) 8783-106 / โ€ญ+1 (412) 618-6238โ€ฌ +- Our emails โœ‰๏ธ ishaan@berri.ai / krrish@berri.ai From b54b1d958b03f5928175671873dc885c6a758ef6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 14:41:12 -0700 Subject: [PATCH 14/99] track anthropic_routes --- litellm/proxy/_types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e9371c1d8d9..0724867aa92 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -228,6 +228,10 @@ class LiteLLMRoutes(enum.Enum): "/utils/token_counter", ] + anthropic_routes: List = [ + "/v1/messages", + ] + info_routes: List = [ "/key/info", "/team/info", From b64755d2a1187bb33162bc2966a05ee3d66acc20 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 14:43:30 -0700 Subject: [PATCH 15/99] check is_llm_api_route --- litellm/proxy/auth/auth_checks.py | 8 ++++---- litellm/proxy/auth/auth_utils.py | 5 ++++- litellm/proxy/auth/user_api_key_auth.py | 8 ++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 96171f2efb7..1650eb8aacd 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -24,7 +24,7 @@ from litellm.proxy._types import ( LitellmUserRoles, UserAPIKeyAuth, ) -from litellm.proxy.auth.auth_utils import is_openai_route +from litellm.proxy.auth.auth_utils import is_llm_api_route from litellm.proxy.utils import PrismaClient, ProxyLogging, log_to_opentelemetry from litellm.types.services import ServiceLoggerPayload, ServiceTypes @@ -106,7 +106,7 @@ def common_checks( general_settings.get("enforce_user_param", None) is not None and general_settings["enforce_user_param"] == True ): - if is_openai_route(route=route) and "user" not in request_body: + if is_llm_api_route(route=route) and "user" not in request_body: raise Exception( f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}" ) @@ -122,7 +122,7 @@ def common_checks( + CommonProxyErrors.not_premium_user.value ) - if is_openai_route(route=route): + if is_llm_api_route(route=route): # loop through each enforced param # example enforced_params ['user', 'metadata', 'metadata.generation_name'] for enforced_param in general_settings["enforced_params"]: @@ -150,7 +150,7 @@ def common_checks( and global_proxy_spend is not None # only run global budget checks for OpenAI routes # Reason - the Admin UI should continue working if the proxy crosses it's global budget - and is_openai_route(route=route) + and is_llm_api_route(route=route) and route != "/v1/models" and route != "/models" ): diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index d3e03076252..bd1e50ed0be 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -46,7 +46,7 @@ def route_in_additonal_public_routes(current_route: str): return False -def is_openai_route(route: str) -> bool: +def is_llm_api_route(route: str) -> bool: """ Helper to checks if provided route is an OpenAI route @@ -59,6 +59,9 @@ def is_openai_route(route: str) -> bool: if route in LiteLLMRoutes.openai_routes.value: return True + if route in LiteLLMRoutes.anthropic_routes.value: + return True + # fuzzy match routes like "/v1/threads/thread_49EIN5QF32s4mH20M7GFKdlZ" # Check for routes with placeholders for openai_route in LiteLLMRoutes.openai_routes.value: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index c5549ffcb66..b4c88148e1d 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -57,7 +57,7 @@ from litellm.proxy.auth.auth_checks import ( log_to_opentelemetry, ) from litellm.proxy.auth.auth_utils import ( - is_openai_route, + is_llm_api_route, route_in_additonal_public_routes, ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body @@ -994,9 +994,9 @@ async def user_api_key_auth( _user_role = _get_user_role(user_id_information=user_id_information) if not _is_user_proxy_admin(user_id_information): # if non-admin - if is_openai_route(route=route): + if is_llm_api_route(route=route): pass - elif is_openai_route(route=request["route"].name): + elif is_llm_api_route(route=request["route"].name): pass elif ( route in LiteLLMRoutes.info_routes.value @@ -1049,7 +1049,7 @@ async def user_api_key_auth( pass elif _user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value: - if is_openai_route(route=route): + if is_llm_api_route(route=route): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"user not allowed to access this OpenAI routes, role= {_user_role}", From 673105c88ffdfce1f63e2e1806463eda831a2440 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 14:44:47 -0700 Subject: [PATCH 16/99] update tests --- litellm/proxy/tests/test_anthropic_sdk.py | 22 ++++++++++++++++++++++ litellm/tests/test_proxy_routes.py | 10 +++++----- 2 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 litellm/proxy/tests/test_anthropic_sdk.py diff --git a/litellm/proxy/tests/test_anthropic_sdk.py b/litellm/proxy/tests/test_anthropic_sdk.py new file mode 100644 index 00000000000..073fafb079b --- /dev/null +++ b/litellm/proxy/tests/test_anthropic_sdk.py @@ -0,0 +1,22 @@ +import os + +from anthropic import Anthropic + +client = Anthropic( + # This is the default and can be omitted + base_url="http://localhost:4000", + # this is a litellm proxy key :) - not a real anthropic key + api_key="sk-s4xN1IiLTCytwtZFJaYQrA", +) + +message = client.messages.create( + max_tokens=1024, + messages=[ + { + "role": "user", + "content": "Hello, Claude", + } + ], + model="claude-3-opus-20240229", +) +print(message.content) diff --git a/litellm/tests/test_proxy_routes.py b/litellm/tests/test_proxy_routes.py index 776ad1e788f..0e3f6339c7e 100644 --- a/litellm/tests/test_proxy_routes.py +++ b/litellm/tests/test_proxy_routes.py @@ -19,7 +19,7 @@ import pytest import litellm from litellm.proxy._types import LiteLLMRoutes -from litellm.proxy.auth.auth_utils import is_openai_route +from litellm.proxy.auth.auth_utils import is_llm_api_route from litellm.proxy.proxy_server import app # Configure logging @@ -77,8 +77,8 @@ def test_routes_on_litellm_proxy(): ("/v1/non_existent_endpoint", False), ], ) -def test_is_openai_route(route: str, expected: bool): - assert is_openai_route(route) == expected +def test_is_llm_api_route(route: str, expected: bool): + assert is_llm_api_route(route) == expected # Test case for routes that are similar but should return False @@ -91,5 +91,5 @@ def test_is_openai_route(route: str, expected: bool): "/engines/model/invalid/completions", ], ) -def test_is_openai_route_similar_but_false(route: str): - assert is_openai_route(route) == False +def test_is_llm_api_route_similar_but_false(route: str): + assert is_llm_api_route(route) == False From 9345c5de2a1dcc7c010730a107bf059433faa4b7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 14:48:22 -0700 Subject: [PATCH 17/99] add test for anthropic routes --- litellm/tests/test_proxy_routes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/litellm/tests/test_proxy_routes.py b/litellm/tests/test_proxy_routes.py index 0e3f6339c7e..6f5774d3e73 100644 --- a/litellm/tests/test_proxy_routes.py +++ b/litellm/tests/test_proxy_routes.py @@ -93,3 +93,8 @@ def test_is_llm_api_route(route: str, expected: bool): ) def test_is_llm_api_route_similar_but_false(route: str): assert is_llm_api_route(route) == False + + +def test_anthropic_api_routes(): + # allow non proxy admins to call anthropic api routes + assert is_llm_api_route(route="/v1/messages") is True From 1ce029c80e4ef741564c7d1bba540362dbf044fa Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 15:06:15 -0700 Subject: [PATCH 18/99] feat(braintrust.py): initial commit for braintrust integration --- litellm/integrations/braintrust.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 litellm/integrations/braintrust.py diff --git a/litellm/integrations/braintrust.py b/litellm/integrations/braintrust.py new file mode 100644 index 00000000000..e69de29bb2d From d3bced56bb3aea762264c5b4fdd8b5ecc00c2485 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 15:13:11 -0700 Subject: [PATCH 19/99] docs - langsmith --- .../observability/langsmith_integration.md | 2 +- docs/my-website/docs/proxy/logging.md | 46 +++++++++++++++++++ docs/my-website/sidebars.js | 3 +- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index 79d047e33a6..d57a64f0952 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Langsmith - Logging LLM Input/Output +# ๐Ÿฆœ Langsmith - Logging LLM Input/Output :::tip diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 0d501664543..680264ec759 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -1106,6 +1106,52 @@ environment_variables: ``` +2. Start Proxy + +``` +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "fake-openai-endpoint", + "messages": [ + { + "role": "user", + "content": "Hello, Claude gm!" + } + ], + } +' +``` +Expect to see your log on Langfuse + + + +## Logging LLM IO to Arize AI + +1. Set `success_callback: ["arize"]` on litellm config.yaml + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +litellm_settings: + callbacks: ["arize"] + +environment_variables: + ARIZE_SPACE_KEY: "d0*****" + ARIZE_API_KEY: "141a****" +``` + 2. Start Proxy ``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 204c273944d..cbe4c204646 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -192,6 +192,8 @@ const sidebars = { items: [ "observability/langfuse_integration", "observability/logfire_integration", + "observability/langsmith_integration", + "observability/arize_integration", "debugging/local_debugging", "observability/raw_request_response", "observability/custom_callback", @@ -202,7 +204,6 @@ const sidebars = { "observability/openmeter", "observability/promptlayer_integration", "observability/wandb_integration", - "observability/langsmith_integration", "observability/slack_integration", "observability/traceloop_integration", "observability/athina_integration", From a0600a30d83ae48c7391be95ac432abc0957d4a9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 15:32:49 -0700 Subject: [PATCH 20/99] fix using arize as success callback --- litellm/tests/test_arize_ai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_arize_ai.py b/litellm/tests/test_arize_ai.py index 7c38db4c602..dfc00446ef3 100644 --- a/litellm/tests/test_arize_ai.py +++ b/litellm/tests/test_arize_ai.py @@ -18,7 +18,7 @@ import logging @pytest.mark.asyncio() async def test_async_otel_callback(): litellm.set_verbose = True - litellm.callbacks = ["arize"] + litellm.success_callback = ["arize"] await litellm.acompletion( model="gpt-3.5-turbo", From d8d08a1ba380dd82b4df9e1d2aaf470fd2240138 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 15:38:46 -0700 Subject: [PATCH 21/99] set _known_custom_logger_compatible_callbacks in _init --- litellm/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 4283f4586bc..78043b90604 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -4,7 +4,7 @@ import warnings warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*") ### INIT VARIABLES ### import threading, requests, os -from typing import Callable, List, Optional, Dict, Union, Any, Literal +from typing import Callable, List, Optional, Dict, Union, Any, Literal, get_args from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.caching import Cache from litellm._logging import ( @@ -46,6 +46,9 @@ _custom_logger_compatible_callbacks_literal = Literal[ "galileo", "arize", ] +_known_custom_logger_compatible_callbacks: List = list( + get_args(_custom_logger_compatible_callbacks_literal) +) callbacks: List[Union[Callable, _custom_logger_compatible_callbacks_literal]] = [] _langfuse_default_tags: Optional[ List[ From 15c109f023f5f9ddc15e7919042153970c204f9a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 15:43:43 -0700 Subject: [PATCH 22/99] fix checking if _known_custom_logger_compatible_callbacks --- litellm/proxy/common_utils/init_callbacks.py | 8 ++++---- litellm/utils.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/common_utils/init_callbacks.py b/litellm/proxy/common_utils/init_callbacks.py index 489f9b3a6a6..eaa926fed5d 100644 --- a/litellm/proxy/common_utils/init_callbacks.py +++ b/litellm/proxy/common_utils/init_callbacks.py @@ -23,11 +23,11 @@ def initialize_callbacks_on_proxy( ) if isinstance(value, list): imported_list: List[Any] = [] - known_compatible_callbacks = list( - get_args(litellm._custom_logger_compatible_callbacks_literal) - ) for callback in value: # ["presidio", ] - if isinstance(callback, str) and callback in known_compatible_callbacks: + if ( + isinstance(callback, str) + and callback in litellm._known_custom_logger_compatible_callbacks + ): imported_list.append(callback) elif isinstance(callback, str) and callback == "otel": from litellm.integrations.opentelemetry import OpenTelemetry diff --git a/litellm/utils.py b/litellm/utils.py index 5ec7b52f5dc..42422fc445a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -158,6 +158,7 @@ from typing import ( Tuple, Union, cast, + get_args, ) from .caching import Cache @@ -405,7 +406,6 @@ def function_setup( # Pop the async items from input_callback in reverse order to avoid index issues for index in reversed(removed_async_items): litellm.input_callback.pop(index) - if len(litellm.success_callback) > 0: removed_async_items = [] for index, callback in enumerate(litellm.success_callback): # type: ignore @@ -417,9 +417,9 @@ def function_setup( # we only support async dynamo db logging for acompletion/aembedding since that's used on proxy litellm._async_success_callback.append(callback) removed_async_items.append(index) - elif callback == "langsmith": + elif callback in litellm._known_custom_logger_compatible_callbacks: callback_class = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class( # type: ignore - callback, internal_usage_cache=None, llm_router=None + callback, internal_usage_cache=None, llm_router=None # type: ignore ) # don't double add a callback From 8f9638f2c1fc7a9ec063d9e875c5e3e27b36dc8e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 16:03:15 -0700 Subject: [PATCH 23/99] fix raise correct provider on content policy violation --- litellm/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 42422fc445a..9d798f1196a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8808,11 +8808,14 @@ class CustomStreamWrapper: str_line.choices[0].content_filter_result ) else: - error_message = "Azure Response={}".format( - str(dict(str_line)) + error_message = "{} Response={}".format( + self.custom_llm_provider, str(dict(str_line)) ) - raise litellm.AzureOpenAIError( - status_code=400, message=error_message + + raise litellm.ContentPolicyViolationError( + message=error_message, + llm_provider=self.custom_llm_provider, + model=self.model, ) # checking for logprobs From 69e52e0a4775e2c8cbb087fd97c8a0ca6231999a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 16:24:03 -0700 Subject: [PATCH 24/99] test - openai content policy errors --- litellm/tests/test_exceptions.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/litellm/tests/test_exceptions.py b/litellm/tests/test_exceptions.py index 94ece7305d8..66c8594bba8 100644 --- a/litellm/tests/test_exceptions.py +++ b/litellm/tests/test_exceptions.py @@ -64,6 +64,30 @@ async def test_content_policy_exception_azure(): pytest.fail(f"An exception occurred - {str(e)}") +@pytest.mark.asyncio +async def test_content_policy_exception_openai(): + try: + # this is ony a test - we needed some way to invoke the exception :( + litellm.set_verbose = True + response = await litellm.acompletion( + model="gpt-3.5-turbo-0613", + stream=True, + messages=[ + {"role": "user", "content": "Gimme the lyrics to Don't Stop Me Now"} + ], + ) + async for chunk in response: + print(chunk) + except litellm.ContentPolicyViolationError as e: + print("caught a content policy violation error! Passed") + print("exception", e) + assert e.llm_provider == "openai" + pass + except Exception as e: + print() + pytest.fail(f"An exception occurred - {str(e)}") + + # Test 1: Context Window Errors @pytest.mark.skip(reason="AWS Suspended Account") @pytest.mark.parametrize("model", exception_models) From e4ab50e1a1e94e9b311d2a8b8eb15b304ecc2e36 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 17:04:55 -0700 Subject: [PATCH 25/99] feat(braintrust_logging.py): working braintrust logging for successful calls --- litellm/__init__.py | 8 +- litellm/integrations/braintrust.py | 0 litellm/integrations/braintrust_logging.py | 245 ++++++++++++++++++ litellm/litellm_core_utils/litellm_logging.py | 12 + litellm/proxy/_new_secret_config.yaml | 4 + litellm/proxy/common_utils/init_callbacks.py | 1 + 6 files changed, 269 insertions(+), 1 deletion(-) delete mode 100644 litellm/integrations/braintrust.py create mode 100644 litellm/integrations/braintrust_logging.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 7dcc934a683..bf3f77385ae 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -38,7 +38,13 @@ success_callback: List[Union[str, Callable]] = [] failure_callback: List[Union[str, Callable]] = [] service_callback: List[Union[str, Callable]] = [] _custom_logger_compatible_callbacks_literal = Literal[ - "lago", "openmeter", "logfire", "dynamic_rate_limiter", "langsmith", "galileo" + "lago", + "openmeter", + "logfire", + "dynamic_rate_limiter", + "langsmith", + "galileo", + "braintrust", ] callbacks: List[Union[Callable, _custom_logger_compatible_callbacks_literal]] = [] _langfuse_default_tags: Optional[ diff --git a/litellm/integrations/braintrust.py b/litellm/integrations/braintrust.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py new file mode 100644 index 00000000000..8bd813b69f9 --- /dev/null +++ b/litellm/integrations/braintrust_logging.py @@ -0,0 +1,245 @@ +# What is this? +## Log success + failure events to Braintrust + +import copy +import json +import os +import threading +import traceback +import uuid +from typing import Literal, Optional + +import dotenv +import httpx +from braintrust import Span, SpanTypeAttribute, init, start_span + +import litellm +from litellm import verbose_logger +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.utils import get_formatted_prompt + +global_braintrust_http_handler = AsyncHTTPHandler() +API_BASE = "https://api.braintrustdata.com/v1" + + +def get_utc_datetime(): + import datetime as dt + from datetime import datetime + + if hasattr(dt, "UTC"): + return datetime.now(dt.UTC) # type: ignore + else: + return datetime.utcnow() # type: ignore + + +class BraintrustLogger(CustomLogger): + def __init__( + self, api_key: Optional[str] = None, api_base: Optional[str] = None + ) -> None: + super().__init__() + self.validate_environment(api_key=api_key) + self.api_base = api_base or API_BASE + self.default_project_id = None + self.api_key: str = api_key or os.getenv("BRAINTRUST_API_KEY") # type: ignore + self.headers = { + "Authorization": "Bearer " + self.api_key, + "Content-Type": "application/json", + } + + def validate_environment(self, api_key: Optional[str]): + """ + Expects + BRAINTRUST_API_KEY + + in the environment + """ + missing_keys = [] + if api_key is None and os.getenv("BRAINTRUST_API_KEY", None) is None: + missing_keys.append("BRAINTRUST_API_KEY") + + if len(missing_keys) > 0: + raise Exception("Missing keys={} in environment.".format(missing_keys)) + + @staticmethod + def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: + """ + Adds metadata from proxy request headers to Langfuse logging if keys start with "langfuse_" + and overwrites litellm_params.metadata if already included. + + For example if you want to append your trace to an existing `trace_id` via header, send + `headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request. + """ + if litellm_params is None: + return metadata + + if litellm_params.get("proxy_server_request") is None: + return metadata + + if metadata is None: + metadata = {} + + proxy_headers = ( + litellm_params.get("proxy_server_request", {}).get("headers", {}) or {} + ) + + for metadata_param_key in proxy_headers: + if metadata_param_key.startswith("braintrust"): + trace_param_key = metadata_param_key.replace("braintrust", "", 1) + if trace_param_key in metadata: + verbose_logger.warning( + f"Overwriting Braintrust `{trace_param_key}` from request header" + ) + else: + verbose_logger.debug( + f"Found Braintrust `{trace_param_key}` in request header" + ) + metadata[trace_param_key] = proxy_headers.get(metadata_param_key) + + return metadata + + async def create_default_project_and_experiment(self): + project = await global_braintrust_http_handler.post( + f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} + ) + + project_dict = project.json() + + self.default_project_id = project_dict["id"] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + verbose_logger.debug("REACHES BRAINTRUST SUCCESS") + try: + litellm_call_id = kwargs.get("litellm_call_id") + trace_id = kwargs.get("trace_id", litellm_call_id) + project_id = kwargs.get("project_id", None) + if project_id is None: + if self.default_project_id is None: + await self.create_default_project_and_experiment() + project_id = self.default_project_id + + prompt = {"messages": kwargs.get("messages")} + + if response_obj is not None and ( + kwargs.get("call_type", None) == "embedding" + or isinstance(response_obj, litellm.EmbeddingResponse) + ): + input = prompt + output = None + elif response_obj is not None and isinstance( + response_obj, litellm.ModelResponse + ): + input = prompt + output = response_obj["choices"][0]["message"].json() + elif response_obj is not None and isinstance( + response_obj, litellm.TextCompletionResponse + ): + input = prompt + output = response_obj.choices[0].text + elif response_obj is not None and isinstance( + response_obj, litellm.ImageResponse + ): + input = prompt + output = response_obj["data"] + + litellm_params = kwargs.get("litellm_params", {}) + metadata = ( + litellm_params.get("metadata", {}) or {} + ) # if litellm_params['metadata'] == None + metadata = self.add_metadata_from_header(litellm_params, metadata) + clean_metadata = {} + try: + metadata = copy.deepcopy( + metadata + ) # Avoid modifying the original metadata + except: + new_metadata = {} + for key, value in metadata.items(): + if ( + isinstance(value, list) + or isinstance(value, dict) + or isinstance(value, str) + or isinstance(value, int) + or isinstance(value, float) + ): + new_metadata[key] = copy.deepcopy(value) + metadata = new_metadata + + tags = [] + if isinstance(metadata, dict): + for key, value in metadata.items(): + + # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy + if ( + litellm._langfuse_default_tags is not None + and isinstance(litellm._langfuse_default_tags, list) + and key in litellm._langfuse_default_tags + ): + tags.append(f"{key}:{value}") + + # clean litellm metadata before logging + if key in [ + "headers", + "endpoint", + "caching_groups", + "previous_models", + ]: + continue + else: + clean_metadata[key] = value + + session_id = clean_metadata.pop("session_id", None) + trace_name = clean_metadata.pop("trace_name", None) + trace_id = clean_metadata.pop("trace_id", litellm_call_id) + existing_trace_id = clean_metadata.pop("existing_trace_id", None) + update_trace_keys = clean_metadata.pop("update_trace_keys", []) + debug = clean_metadata.pop("debug_langfuse", None) + mask_input = clean_metadata.pop("mask_input", False) + mask_output = clean_metadata.pop("mask_output", False) + cost = kwargs.get("response_cost", None) + if cost is not None: + clean_metadata["litellm_response_cost"] = cost + + metrics: Optional[dict] = None + if ( + response_obj is not None + and hasattr(response_obj, "usage") + and isinstance(response_obj.usage, litellm.Usage) + ): + generation_id = litellm.utils.get_logging_id(start_time, response_obj) + metrics = { + "prompt_tokens": response_obj.usage.prompt_tokens, + "completion_tokens": response_obj.usage.completion_tokens, + "total_tokens": response_obj.usage.total_tokens, + "total_cost": cost, + } + + request_data = { + "id": litellm_call_id, + "input": prompt, + "output": output, + "metadata": clean_metadata, + "tags": tags, + } + + if metrics is not None: + request_data["metrics"] = metrics + + try: + await global_braintrust_http_handler.post( + url=f"{self.api_base}/project_logs/{project_id}/insert", + json={"events": [request_data]}, + headers=self.headers, + ) + except httpx.HTTPStatusError as e: + raise Exception(e.response.text) + except Exception as e: + verbose_logger.error( + "Error logging to braintrust - Exception received - {}\n{}".format( + str(e), traceback.format_exc() + ) + ) + raise e + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + return super().log_failure_event(kwargs, response_obj, start_time, end_time) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 32633960f01..17837c41e6f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -53,6 +53,7 @@ from litellm.utils import ( from ..integrations.aispend import AISpendLogger from ..integrations.athina import AthinaLogger from ..integrations.berrispend import BerriSpendLogger +from ..integrations.braintrust_logging import BraintrustLogger from ..integrations.clickhouse import ClickhouseLogger from ..integrations.custom_logger import CustomLogger from ..integrations.datadog import DataDogLogger @@ -1945,7 +1946,14 @@ def _init_custom_logger_compatible_class( _openmeter_logger = OpenMeterLogger() _in_memory_loggers.append(_openmeter_logger) return _openmeter_logger # type: ignore + elif logging_integration == "braintrust": + for callback in _in_memory_loggers: + if isinstance(callback, BraintrustLogger): + return callback # type: ignore + braintrust_logger = BraintrustLogger() + _in_memory_loggers.append(braintrust_logger) + return braintrust_logger # type: ignore elif logging_integration == "langsmith": for callback in _in_memory_loggers: if isinstance(callback, LangsmithLogger): @@ -2019,6 +2027,10 @@ def get_custom_logger_compatible_class( for callback in _in_memory_loggers: if isinstance(callback, OpenMeterLogger): return callback + elif logging_integration == "braintrust": + for callback in _in_memory_loggers: + if isinstance(callback, BraintrustLogger): + return callback elif logging_integration == "galileo": for callback in _in_memory_loggers: if isinstance(callback, GalileoObserve): diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 81244f0fa09..7a35650e58a 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -3,3 +3,7 @@ model_list: litellm_params: model: groq/llama3-groq-70b-8192-tool-use-preview api_key: os.environ/GROQ_API_KEY + + +litellm_settings: + callbacks: ["braintrust"] diff --git a/litellm/proxy/common_utils/init_callbacks.py b/litellm/proxy/common_utils/init_callbacks.py index 489f9b3a6a6..2fcceaa2984 100644 --- a/litellm/proxy/common_utils/init_callbacks.py +++ b/litellm/proxy/common_utils/init_callbacks.py @@ -27,6 +27,7 @@ def initialize_callbacks_on_proxy( get_args(litellm._custom_logger_compatible_callbacks_literal) ) for callback in value: # ["presidio", ] + if isinstance(callback, str) and callback in known_compatible_callbacks: imported_list.append(callback) elif isinstance(callback, str) and callback == "otel": From dd6d58d29bd97f7b5d59fa87275dd92911fab72f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 17:47:08 -0700 Subject: [PATCH 26/99] docs(braintrust.md): add braintrust.md to docs --- .../docs/observability/braintrust.md | 147 ++++++++++++++++++ .../observability/helicone_integration.md | 2 +- docs/my-website/sidebars.js | 3 +- 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 docs/my-website/docs/observability/braintrust.md diff --git a/docs/my-website/docs/observability/braintrust.md b/docs/my-website/docs/observability/braintrust.md new file mode 100644 index 00000000000..573223f6e94 --- /dev/null +++ b/docs/my-website/docs/observability/braintrust.md @@ -0,0 +1,147 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# โšก๏ธ Braintrust - Evals + Logging + +[Braintrust](https://www.braintrust.dev/) manages evaluations, logging, prompt playground, to data management for AI products. + + +## Quick Start + +```python +# pip install langfuse +import litellm +import os + +# set env +os.environ["BRAINTRUST_API_KEY"] = "" +os.environ['OPENAI_API_KEY']="" + +# set braintrust as a callback, litellm will send the data to braintrust +litellm.callbacks = ["braintrust"] + +# openai call +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Hi ๐Ÿ‘‹ - i'm openai"} + ] +) +``` + + + +## OpenAI Proxy Usage + +1. Add keys to env +```env +BRAINTRUST_API_KEY="" +``` + +2. Add braintrust to callbacks +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + + +litellm_settings: + callbacks: ["braintrust"] +``` + +3. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "groq-llama3", + "messages": [ + { "role": "system", "content": "Use your tools smartly"}, + { "role": "user", "content": "What time is it now? Use your tool"} + ] +}' +``` + +## Advanced - pass Project ID + + + + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Hi ๐Ÿ‘‹ - i'm openai"} + ], + metadata={ + "project_id": "my-special-project" + } +) +``` + + + + +**Curl** + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "groq-llama3", + "messages": [ + { "role": "system", "content": "Use your tools smartly"}, + { "role": "user", "content": "What time is it now? Use your tool"} + ], + "metadata": { + "project_id": "my-special-project" + } +}' +``` + +**OpenAI SDK** + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params + "metadata": { # ๐Ÿ‘ˆ use for logging additional params (e.g. to langfuse) + "project_id": "my-special-project" + } + } +) + +print(response) +``` + +For more examples, [**Click Here**](../proxy/user_keys.md#chatcompletions) + + + + +## Full API Spec + +Here's everything you can pass in metadata for a braintrust request + +`braintrust_*` - any metadata field starting with `braintrust_` will be passed as metadata to the logging request + +`project_id` - set the project id for a braintrust call. Default is `litellm`. \ No newline at end of file diff --git a/docs/my-website/docs/observability/helicone_integration.md b/docs/my-website/docs/observability/helicone_integration.md index 7e7f9fcb6f3..d8f3e5d0a5f 100644 --- a/docs/my-website/docs/observability/helicone_integration.md +++ b/docs/my-website/docs/observability/helicone_integration.md @@ -1,4 +1,4 @@ -# ๐Ÿง  Helicone - OSS LLM Observability Platform +# ๐ŸงŠ Helicone - OSS LLM Observability Platform :::tip diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 204c273944d..8d77bd85fe1 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -196,9 +196,10 @@ const sidebars = { "observability/raw_request_response", "observability/custom_callback", "observability/scrub_data", - "observability/helicone_integration", + "observability/braintrust", "observability/sentry", "observability/lago", + "observability/helicone_integration", "observability/openmeter", "observability/promptlayer_integration", "observability/wandb_integration", From 92b1262caa29e4cfd18d04d4d286abb18633bcfa Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 18:05:11 -0700 Subject: [PATCH 27/99] test(test_braintrust.py): add testing for braintrust integration --- litellm/integrations/braintrust_logging.py | 144 +++++++++++++++++++-- litellm/tests/test_braintrust.py | 53 ++++++++ 2 files changed, 187 insertions(+), 10 deletions(-) create mode 100644 litellm/tests/test_braintrust.py diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 8bd813b69f9..0f27bb10222 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -11,7 +11,6 @@ from typing import Literal, Optional import dotenv import httpx -from braintrust import Span, SpanTypeAttribute, init, start_span import litellm from litellm import verbose_logger @@ -20,6 +19,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import get_formatted_prompt global_braintrust_http_handler = AsyncHTTPHandler() +global_braintrust_sync_http_handler = HTTPHandler() API_BASE = "https://api.braintrustdata.com/v1" @@ -107,11 +107,143 @@ class BraintrustLogger(CustomLogger): self.default_project_id = project_dict["id"] + def create_sync_default_project_and_experiment(self): + project = global_braintrust_sync_http_handler.post( + f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} + ) + + project_dict = project.json() + + self.default_project_id = project_dict["id"] + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + verbose_logger.debug("REACHES BRAINTRUST SUCCESS") + try: + litellm_call_id = kwargs.get("litellm_call_id") + project_id = kwargs.get("project_id", None) + if project_id is None: + if self.default_project_id is None: + self.create_sync_default_project_and_experiment() + project_id = self.default_project_id + + prompt = {"messages": kwargs.get("messages")} + + if response_obj is not None and ( + kwargs.get("call_type", None) == "embedding" + or isinstance(response_obj, litellm.EmbeddingResponse) + ): + input = prompt + output = None + elif response_obj is not None and isinstance( + response_obj, litellm.ModelResponse + ): + input = prompt + output = response_obj["choices"][0]["message"].json() + elif response_obj is not None and isinstance( + response_obj, litellm.TextCompletionResponse + ): + input = prompt + output = response_obj.choices[0].text + elif response_obj is not None and isinstance( + response_obj, litellm.ImageResponse + ): + input = prompt + output = response_obj["data"] + + litellm_params = kwargs.get("litellm_params", {}) + metadata = ( + litellm_params.get("metadata", {}) or {} + ) # if litellm_params['metadata'] == None + metadata = self.add_metadata_from_header(litellm_params, metadata) + clean_metadata = {} + try: + metadata = copy.deepcopy( + metadata + ) # Avoid modifying the original metadata + except: + new_metadata = {} + for key, value in metadata.items(): + if ( + isinstance(value, list) + or isinstance(value, dict) + or isinstance(value, str) + or isinstance(value, int) + or isinstance(value, float) + ): + new_metadata[key] = copy.deepcopy(value) + metadata = new_metadata + + tags = [] + if isinstance(metadata, dict): + for key, value in metadata.items(): + + # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy + if ( + litellm._langfuse_default_tags is not None + and isinstance(litellm._langfuse_default_tags, list) + and key in litellm._langfuse_default_tags + ): + tags.append(f"{key}:{value}") + + # clean litellm metadata before logging + if key in [ + "headers", + "endpoint", + "caching_groups", + "previous_models", + ]: + continue + else: + clean_metadata[key] = value + + cost = kwargs.get("response_cost", None) + if cost is not None: + clean_metadata["litellm_response_cost"] = cost + + metrics: Optional[dict] = None + if ( + response_obj is not None + and hasattr(response_obj, "usage") + and isinstance(response_obj.usage, litellm.Usage) + ): + generation_id = litellm.utils.get_logging_id(start_time, response_obj) + metrics = { + "prompt_tokens": response_obj.usage.prompt_tokens, + "completion_tokens": response_obj.usage.completion_tokens, + "total_tokens": response_obj.usage.total_tokens, + "total_cost": cost, + } + + request_data = { + "id": litellm_call_id, + "input": prompt, + "output": output, + "metadata": clean_metadata, + "tags": tags, + } + if metrics is not None: + request_data["metrics"] = metrics + + try: + global_braintrust_sync_http_handler.post( + url=f"{self.api_base}/project_logs/{project_id}/insert", + json={"events": [request_data]}, + headers=self.headers, + ) + except httpx.HTTPStatusError as e: + raise Exception(e.response.text) + except Exception as e: + verbose_logger.error( + "Error logging to braintrust - Exception received - {}\n{}".format( + str(e), traceback.format_exc() + ) + ) + raise e + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") try: litellm_call_id = kwargs.get("litellm_call_id") - trace_id = kwargs.get("trace_id", litellm_call_id) project_id = kwargs.get("project_id", None) if project_id is None: if self.default_project_id is None: @@ -188,14 +320,6 @@ class BraintrustLogger(CustomLogger): else: clean_metadata[key] = value - session_id = clean_metadata.pop("session_id", None) - trace_name = clean_metadata.pop("trace_name", None) - trace_id = clean_metadata.pop("trace_id", litellm_call_id) - existing_trace_id = clean_metadata.pop("existing_trace_id", None) - update_trace_keys = clean_metadata.pop("update_trace_keys", []) - debug = clean_metadata.pop("debug_langfuse", None) - mask_input = clean_metadata.pop("mask_input", False) - mask_output = clean_metadata.pop("mask_output", False) cost = kwargs.get("response_cost", None) if cost is not None: clean_metadata["litellm_response_cost"] = cost diff --git a/litellm/tests/test_braintrust.py b/litellm/tests/test_braintrust.py new file mode 100644 index 00000000000..7792a084121 --- /dev/null +++ b/litellm/tests/test_braintrust.py @@ -0,0 +1,53 @@ +# What is this? +## This tests the braintrust integration + +import asyncio +import os +import random +import sys +import time +import traceback +from datetime import datetime + +from dotenv import load_dotenv +from fastapi import Request + +load_dotenv() +import os + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import asyncio +import logging +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler + + +def test_braintrust_logging(): + import litellm + + http_client = HTTPHandler() + + setattr( + litellm.integrations.braintrust_logging, + "global_braintrust_sync_http_handler", + http_client, + ) + + with patch.object(http_client, "post", new=MagicMock()) as mock_client: + + # set braintrust as a callback, litellm will send the data to braintrust + litellm.callbacks = ["braintrust"] + + # openai call + response = litellm.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hi ๐Ÿ‘‹ - i'm openai"}], + ) + + mock_client.assert_called() From ff768e75719b368570cefd657cc549876176a4ad Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 18:11:59 -0700 Subject: [PATCH 28/99] types - AddTeamCallback --- litellm/proxy/_types.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0724867aa92..190d4be2714 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -884,6 +884,26 @@ class BlockTeamRequest(LiteLLMBase): team_id: str # required +class AddTeamCallback(LiteLLMBase): + callback_name: str + callback_type: Literal["success", "failure", "success_and_failure"] + # for now - only supported for langfuse + callback_vars: Dict[ + Literal["langfuse_public_key", "langfuse_secret_key", "langfuse_host"], str + ] + + +class TeamCallbackMetadata(LiteLLMBase): + success_callback: Optional[List[str]] = [] + failure_callback: Optional[List[str]] = [] + # for now - only supported for langfuse + callback_vars: Optional[ + Dict[ + Literal["langfuse_public_key", "langfuse_secret_key", "langfuse_host"], str + ] + ] = {} + + class LiteLLM_TeamTable(TeamBase): spend: Optional[float] = None max_parallel_requests: Optional[int] = None From f4a388f21787477d9c2d80141a2c10634e955547 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 18:10:33 -0700 Subject: [PATCH 29/99] fix(openai.py): check if error body is a dictionary before indexing in --- litellm/llms/openai.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index d1e0d14ba99..25e2e518c55 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -968,7 +968,7 @@ class OpenAIChatCompletion(BaseLLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - if e.body is not None and e.body.get("detail"): # type: ignore + if e.body is not None and isinstance(e.body, dict) and e.body.get("detail"): # type: ignore detail = e.body.get("detail") # type: ignore invalid_params: List[str] = [] if ( @@ -1100,7 +1100,7 @@ class OpenAIChatCompletion(BaseLLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - if e.body is not None and e.body.get("detail"): # type: ignore + if e.body is not None and isinstance(e.body, dict) and e.body.get("detail"): # type: ignore detail = e.body.get("detail") # type: ignore invalid_params: List[str] = [] if ( @@ -1231,7 +1231,7 @@ class OpenAIChatCompletion(BaseLLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - if e.body is not None and e.body.get("detail"): # type: ignore + if e.body is not None and isinstance(e.body, dict) and e.body.get("detail"): # type: ignore detail = e.body.get("detail") # type: ignore invalid_params: List[str] = [] if ( From c34c123fe35fe61846045b135e971f2e64b02c43 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 18:18:09 -0700 Subject: [PATCH 30/99] feat - add endpoint to set team callbacks --- .../team_callback_endpoints.py | 106 ++++++++++++++++++ litellm/proxy/proxy_server.py | 4 + 2 files changed, 110 insertions(+) create mode 100644 litellm/proxy/management_endpoints/team_callback_endpoints.py diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py new file mode 100644 index 00000000000..0ed463992a7 --- /dev/null +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -0,0 +1,106 @@ +""" +Endpoints to control callbacks per team + +Use this when each team should control its own callbacks +""" + +import asyncio +import copy +import json +import traceback +import uuid +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +import fastapi +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import ( + AddTeamCallback, + LiteLLM_TeamTable, + TeamCallbackMetadata, + UserAPIKeyAuth, +) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_helpers.utils import ( + add_new_member, + management_endpoint_wrapper, +) + +router = APIRouter() + + +@router.post( + "/team/{team_id:path}/callback", + tags=["team management"], + dependencies=[Depends(user_api_key_auth)], +) +@management_endpoint_wrapper +async def add_team_callbacks( + data: AddTeamCallback, + http_request: Request, + team_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_changed_by: Optional[str] = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), +): + from litellm.proxy.proxy_server import ( + _duration_in_seconds, + create_audit_log_for_update, + litellm_proxy_admin_name, + prisma_client, + ) + + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) + + # Check if team_id exists already + _existing_team = await prisma_client.get_data( + team_id=team_id, table_name="team", query_type="find_unique" + ) + if _existing_team is None: + raise HTTPException( + status_code=400, + detail={ + "error": f"Team id = {team_id} does not exist. Please use a different team id." + }, + ) + + # store team callback settings in metadata + team_metadata = _existing_team.metadata + team_callback_settings = team_metadata.get("callback_settings", {}) + # expect callback settings to be + team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) + if data.callback_type == "success": + if team_callback_settings_obj.success_callback is None: + team_callback_settings_obj.success_callback = [] + + team_callback_settings_obj.success_callback.append(data.callback_name) + elif data.callback_type == "failure": + if team_callback_settings_obj.failure_callback is None: + team_callback_settings_obj.failure_callback = [] + team_callback_settings_obj.failure_callback.append(data.callback_name) + elif data.callback_type == "success_and_failure": + if team_callback_settings_obj.success_callback is None: + team_callback_settings_obj.success_callback = [] + if team_callback_settings_obj.failure_callback is None: + team_callback_settings_obj.failure_callback = [] + team_callback_settings_obj.success_callback.append(data.callback_name) + team_callback_settings_obj.failure_callback.append(data.callback_name) + for var, value in data.callback_vars.items(): + if team_callback_settings_obj.callback_vars is None: + team_callback_settings_obj.callback_vars = {} + team_callback_settings_obj.callback_vars[var] = value + + team_callback_settings_obj_dict = team_callback_settings_obj.model_dump() + + team_metadata["callback_settings"] = team_callback_settings_obj_dict + team_metadata_json = json.dumps(team_metadata) # update team_metadata + + await prisma_client.db.litellm_teamtable.update( + where={"team_id": team_id}, data={"metadata": team_metadata_json} # type: ignore + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 79f25c6e100..3ab8643813f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -170,6 +170,9 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( from litellm.proxy.management_endpoints.key_management_endpoints import ( router as key_management_router, ) +from litellm.proxy.management_endpoints.team_callback_endpoints import ( + router as team_callback_router, +) from litellm.proxy.management_endpoints.team_endpoints import router as team_router from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, @@ -9457,3 +9460,4 @@ app.include_router(analytics_router) app.include_router(debugging_endpoints_router) app.include_router(ui_crud_endpoints_router) app.include_router(openai_files_router) +app.include_router(team_callback_router) From c9e2f977dd92ab1803d2c56e6eb4173c629f4acf Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 18:21:50 -0700 Subject: [PATCH 31/99] feat - return team_metadata in user_api_key_auth --- litellm/proxy/_types.py | 1 + litellm/proxy/utils.py | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 190d4be2714..0cd4af7e18d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1256,6 +1256,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): soft_budget: Optional[float] = None team_model_aliases: Optional[Dict] = None team_member_spend: Optional[float] = None + team_metadata: Optional[Dict] = None # End User Params end_user_id: Optional[str] = None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0f87e962abc..5846dd25a86 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1313,6 +1313,7 @@ class PrismaClient: t.tpm_limit AS team_tpm_limit, t.rpm_limit AS team_rpm_limit, t.models AS team_models, + t.metadata AS team_metadata, t.blocked AS team_blocked, t.team_alias AS team_alias, tm.spend AS team_member_spend, From dcd8f7ebf2fffaef060cfabf4b31c499b73c6e44 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 18:29:21 -0700 Subject: [PATCH 32/99] control team callbacks using API --- litellm/proxy/litellm_pre_call_utils.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1014a325ab3..642c1261696 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional from fastapi import Request from litellm._logging import verbose_logger, verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, TeamCallbackMetadata, UserAPIKeyAuth from litellm.types.utils import SupportedCacheControls if TYPE_CHECKING: @@ -207,6 +207,29 @@ async def add_litellm_data_to_request( **data, } # add the team-specific configs to the completion call + # Team Callbacks controls + if user_api_key_dict.team_metadata is not None: + team_metadata = user_api_key_dict.team_metadata + if "callback_settings" in team_metadata: + callback_settings = team_metadata.get("callback_settings", None) or {} + callback_settings_obj = TeamCallbackMetadata(**callback_settings) + """ + callback_settings = { + { + 'callback_vars': {'langfuse_public_key': 'pk', 'langfuse_secret_key': 'sk_'}, + 'failure_callback': [], + 'success_callback': ['langfuse', 'langfuse'] + } + } + """ + data["success_callback"] = callback_settings_obj.success_callback + data["failure_callback"] = callback_settings_obj.failure_callback + + if callback_settings_obj.callback_vars is not None: + # unpack callback_vars in data + for k, v in callback_settings_obj.callback_vars.items(): + data[k] = v + return data From 5ed82ba5ceabc5943abf009bf788dc2edcf91f2c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 20:35:27 -0700 Subject: [PATCH 33/99] feat add return types on team/callback --- litellm/proxy/_types.py | 1 + .../team_callback_endpoints.py | 148 ++++++++++++------ 2 files changed, 101 insertions(+), 48 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0cd4af7e18d..6ca8c8a0b34 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1702,3 +1702,4 @@ class ProxyErrorTypes(str, enum.Enum): budget_exceeded = "budget_exceeded" expired_key = "expired_key" auth_error = "auth_error" + internal_server_error = "internal_server_error" diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 0ed463992a7..b73125b6536 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -20,6 +20,8 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( AddTeamCallback, LiteLLM_TeamTable, + ProxyErrorTypes, + ProxyException, TeamCallbackMetadata, UserAPIKeyAuth, ) @@ -48,59 +50,109 @@ async def add_team_callbacks( description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", ), ): - from litellm.proxy.proxy_server import ( - _duration_in_seconds, - create_audit_log_for_update, - litellm_proxy_admin_name, - prisma_client, - ) + """ + Add a success/failure callback to a team - if prisma_client is None: - raise HTTPException(status_code=500, detail={"error": "No db connected"}) + Use this if if you want different teams to have different success/failure callbacks - # Check if team_id exists already - _existing_team = await prisma_client.get_data( - team_id=team_id, table_name="team", query_type="find_unique" - ) - if _existing_team is None: - raise HTTPException( - status_code=400, - detail={ - "error": f"Team id = {team_id} does not exist. Please use a different team id." - }, + Example curl: + ``` + curl -X POST 'http:/localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/callback' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer sk-1234' \ + -d '{ + "callback_name": "langfuse", + "callback_type": "success", + "callback_vars": {"langfuse_public_key": "pk-lf-xxxx1", "langfuse_secret_key": "sk-xxxxx"} + + }' + ``` + + This means for the team where team_id = dbe2f686-a686-4896-864a-4c3924458709, all LLM calls will be logged to langfuse using the public key pk-lf-xxxx1 and the secret key sk-xxxxx + + """ + try: + from litellm.proxy.proxy_server import ( + _duration_in_seconds, + create_audit_log_for_update, + litellm_proxy_admin_name, + prisma_client, ) - # store team callback settings in metadata - team_metadata = _existing_team.metadata - team_callback_settings = team_metadata.get("callback_settings", {}) - # expect callback settings to be - team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) - if data.callback_type == "success": - if team_callback_settings_obj.success_callback is None: - team_callback_settings_obj.success_callback = [] + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) - team_callback_settings_obj.success_callback.append(data.callback_name) - elif data.callback_type == "failure": - if team_callback_settings_obj.failure_callback is None: - team_callback_settings_obj.failure_callback = [] - team_callback_settings_obj.failure_callback.append(data.callback_name) - elif data.callback_type == "success_and_failure": - if team_callback_settings_obj.success_callback is None: - team_callback_settings_obj.success_callback = [] - if team_callback_settings_obj.failure_callback is None: - team_callback_settings_obj.failure_callback = [] - team_callback_settings_obj.success_callback.append(data.callback_name) - team_callback_settings_obj.failure_callback.append(data.callback_name) - for var, value in data.callback_vars.items(): - if team_callback_settings_obj.callback_vars is None: - team_callback_settings_obj.callback_vars = {} - team_callback_settings_obj.callback_vars[var] = value + # Check if team_id exists already + _existing_team = await prisma_client.get_data( + team_id=team_id, table_name="team", query_type="find_unique" + ) + if _existing_team is None: + raise HTTPException( + status_code=400, + detail={ + "error": f"Team id = {team_id} does not exist. Please use a different team id." + }, + ) - team_callback_settings_obj_dict = team_callback_settings_obj.model_dump() + # store team callback settings in metadata + team_metadata = _existing_team.metadata + team_callback_settings = team_metadata.get("callback_settings", {}) + # expect callback settings to be + team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) + if data.callback_type == "success": + if team_callback_settings_obj.success_callback is None: + team_callback_settings_obj.success_callback = [] - team_metadata["callback_settings"] = team_callback_settings_obj_dict - team_metadata_json = json.dumps(team_metadata) # update team_metadata + team_callback_settings_obj.success_callback.append(data.callback_name) + elif data.callback_type == "failure": + if team_callback_settings_obj.failure_callback is None: + team_callback_settings_obj.failure_callback = [] + team_callback_settings_obj.failure_callback.append(data.callback_name) + elif data.callback_type == "success_and_failure": + if team_callback_settings_obj.success_callback is None: + team_callback_settings_obj.success_callback = [] + if team_callback_settings_obj.failure_callback is None: + team_callback_settings_obj.failure_callback = [] + team_callback_settings_obj.success_callback.append(data.callback_name) + team_callback_settings_obj.failure_callback.append(data.callback_name) + for var, value in data.callback_vars.items(): + if team_callback_settings_obj.callback_vars is None: + team_callback_settings_obj.callback_vars = {} + team_callback_settings_obj.callback_vars[var] = value - await prisma_client.db.litellm_teamtable.update( - where={"team_id": team_id}, data={"metadata": team_metadata_json} # type: ignore - ) + team_callback_settings_obj_dict = team_callback_settings_obj.model_dump() + + team_metadata["callback_settings"] = team_callback_settings_obj_dict + team_metadata_json = json.dumps(team_metadata) # update team_metadata + + new_team_row = await prisma_client.db.litellm_teamtable.update( + where={"team_id": team_id}, data={"metadata": team_metadata_json} # type: ignore + ) + + return { + "status": "success", + "data": new_team_row, + } + + except Exception as e: + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.add_team_callbacks(): Exception occured - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type=ProxyErrorTypes.internal_server_error.value, + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type=ProxyErrorTypes.internal_server_error.value, + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) From 0b9e93d86322e03a02a218be8156b4a8c1a05711 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 20:36:35 -0700 Subject: [PATCH 34/99] fix(main.py): check if anthropic api base ends with required url Fixes https://github.com/BerriAI/litellm/issues/4803 --- litellm/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/main.py b/litellm/main.py index 8cb52d9459e..4e2df72cd8e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1491,6 +1491,10 @@ def completion( or get_secret("ANTHROPIC_BASE_URL") or "https://api.anthropic.com/v1/complete" ) + + if api_base is not None and not api_base.endswith("/v1/complete"): + api_base += "/v1/complete" + response = anthropic_text_completions.completion( model=model, messages=messages, @@ -1517,6 +1521,10 @@ def completion( or get_secret("ANTHROPIC_BASE_URL") or "https://api.anthropic.com/v1/messages" ) + + if api_base is not None and not api_base.endswith("/v1/messages"): + api_base += "/v1/messages" + response = anthropic_chat_completions.completion( model=model, messages=messages, From 447bab4d46537d3799aa0e1dd04225dc937060df Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 20:43:42 -0700 Subject: [PATCH 35/99] only allow unique callbacks for team callbacks --- litellm/proxy/_types.py | 1 + .../team_callback_endpoints.py | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 6ca8c8a0b34..25aa942e50a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1703,3 +1703,4 @@ class ProxyErrorTypes(str, enum.Enum): expired_key = "expired_key" auth_error = "auth_error" internal_server_error = "internal_server_error" + bad_request_error = "bad_request_error" diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index b73125b6536..a711c851679 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -103,16 +103,48 @@ async def add_team_callbacks( if team_callback_settings_obj.success_callback is None: team_callback_settings_obj.success_callback = [] + if data.callback_name in team_callback_settings_obj.success_callback: + raise ProxyException( + message=f"callback_name = {data.callback_name} already exists in failure_callback, for team_id = {team_id}. \n Existing failure_callback = {team_callback_settings_obj.success_callback}", + code=status.HTTP_400_BAD_REQUEST, + type=ProxyErrorTypes.bad_request_error, + param="callback_name", + ) + team_callback_settings_obj.success_callback.append(data.callback_name) elif data.callback_type == "failure": if team_callback_settings_obj.failure_callback is None: team_callback_settings_obj.failure_callback = [] + + if data.callback_name in team_callback_settings_obj.failure_callback: + raise ProxyException( + message=f"callback_name = {data.callback_name} already exists in failure_callback, for team_id = {team_id}. \n Existing failure_callback = {team_callback_settings_obj.failure_callback}", + code=status.HTTP_400_BAD_REQUEST, + type=ProxyErrorTypes.bad_request_error, + param="callback_name", + ) team_callback_settings_obj.failure_callback.append(data.callback_name) elif data.callback_type == "success_and_failure": if team_callback_settings_obj.success_callback is None: team_callback_settings_obj.success_callback = [] if team_callback_settings_obj.failure_callback is None: team_callback_settings_obj.failure_callback = [] + if data.callback_name in team_callback_settings_obj.success_callback: + raise ProxyException( + message=f"callback_name = {data.callback_name} already exists in success_callback, for team_id = {team_id}. \n Existing success_callback = {team_callback_settings_obj.success_callback}", + code=status.HTTP_400_BAD_REQUEST, + type=ProxyErrorTypes.bad_request_error, + param="callback_name", + ) + + if data.callback_name in team_callback_settings_obj.failure_callback: + raise ProxyException( + message=f"callback_name = {data.callback_name} already exists in failure_callback, for team_id = {team_id}. \n Existing failure_callback = {team_callback_settings_obj.failure_callback}", + code=status.HTTP_400_BAD_REQUEST, + type=ProxyErrorTypes.bad_request_error, + param="callback_name", + ) + team_callback_settings_obj.success_callback.append(data.callback_name) team_callback_settings_obj.failure_callback.append(data.callback_name) for var, value in data.callback_vars.items(): From 2e5da5ea1877d2c2e7a74b8b1729881edad1a102 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 20:46:20 -0700 Subject: [PATCH 36/99] GET endpoint to get team callbacks --- .../team_callback_endpoints.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index a711c851679..9c2ac65cc89 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -188,3 +188,92 @@ async def add_team_callbacks( param=getattr(e, "param", "None"), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + + +@router.get( + "/team/{team_id:path}/callback", + tags=["team management"], + dependencies=[Depends(user_api_key_auth)], +) +@management_endpoint_wrapper +async def get_team_callbacks( + http_request: Request, + team_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get the success/failure callbacks and variables for a team + + Example curl: + ``` + curl -X GET 'http://localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/callback' \ + -H 'Authorization: Bearer sk-1234' + ``` + + This will return the callback settings for the team with id dbe2f686-a686-4896-864a-4c3924458709 + + Returns { + "status": "success", + "data": { + "team_id": team_id, + "success_callbacks": team_callback_settings_obj.success_callback, + "failure_callbacks": team_callback_settings_obj.failure_callback, + "callback_vars": team_callback_settings_obj.callback_vars, + }, + } + """ + try: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) + + # Check if team_id exists + _existing_team = await prisma_client.get_data( + team_id=team_id, table_name="team", query_type="find_unique" + ) + if _existing_team is None: + raise HTTPException( + status_code=404, + detail={"error": f"Team id = {team_id} does not exist."}, + ) + + # Retrieve team callback settings from metadata + team_metadata = _existing_team.metadata + team_callback_settings = team_metadata.get("callback_settings", {}) + + # Convert to TeamCallbackMetadata object for consistent structure + team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) + + return { + "status": "success", + "data": { + "team_id": team_id, + "success_callbacks": team_callback_settings_obj.success_callback, + "failure_callbacks": team_callback_settings_obj.failure_callback, + "callback_vars": team_callback_settings_obj.callback_vars, + }, + } + + except Exception as e: + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.get_team_callbacks(): Exception occurred - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type=ProxyErrorTypes.internal_server_error.value, + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type=ProxyErrorTypes.internal_server_error.value, + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) From 548e4f53f8a759db419b23f9d8d2bb797d2ec6dd Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 20:58:02 -0700 Subject: [PATCH 37/99] feat(redact_messages.py): allow remove sensitive key information before passing to logging integration --- docs/my-website/docs/proxy/logging.md | 14 +++++++++ litellm/__init__.py | 1 + litellm/integrations/langfuse.py | 3 ++ litellm/integrations/logfire_logger.py | 12 ++++++-- litellm/integrations/opentelemetry.py | 5 +++- litellm/litellm_core_utils/redact_messages.py | 30 +++++++++++++++++++ litellm/proxy/_new_secret_config.yaml | 4 +++ 7 files changed, 65 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 680264ec759..5314395ccb8 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -48,6 +48,20 @@ A number of these headers could be useful for troubleshooting, but the `x-litellm-call-id` is the one that is most useful for tracking a request across components in your system, including in logging tools. +## Redacting UserAPIKeyInfo + +Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. + +Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging. + +```yaml +litellm_settings: + callbacks: ["langfuse"] + redact_user_api_key_info: true +``` + +Removes any field with `user_api_key_*` from metadata. + ## Logging Proxy Input/Output - Langfuse We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment diff --git a/litellm/__init__.py b/litellm/__init__.py index 78043b90604..9982097b13c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -76,6 +76,7 @@ post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False +redact_user_api_key_info: Optional[bool] = False store_audit_logs = False # Enterprise feature, allow users to see audit logs ## end of callbacks ############# diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 0647afabcde..0217f7458d9 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -8,6 +8,7 @@ from packaging.version import Version import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info class LangFuseLogger: @@ -382,6 +383,8 @@ class LangFuseLogger: mask_input = clean_metadata.pop("mask_input", False) mask_output = clean_metadata.pop("mask_output", False) + clean_metadata = redact_user_api_key_info(metadata=clean_metadata) + if trace_name is None and existing_trace_id is None: # just log `litellm-{call_type}` as the trace name ## DO NOT SET TRACE_NAME if trace-id set. this can lead to overwriting of past traces. diff --git a/litellm/integrations/logfire_logger.py b/litellm/integrations/logfire_logger.py index b4ab00820ea..fa4ab7bd512 100644 --- a/litellm/integrations/logfire_logger.py +++ b/litellm/integrations/logfire_logger.py @@ -1,17 +1,21 @@ #### What this does #### # On success + failure, log events to Logfire -import dotenv, os +import os + +import dotenv dotenv.load_dotenv() # Loading env variables using dotenv import traceback import uuid -from litellm._logging import print_verbose, verbose_logger - from enum import Enum from typing import Any, Dict, NamedTuple + from typing_extensions import LiteralString +from litellm._logging import print_verbose, verbose_logger +from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info + class SpanConfig(NamedTuple): message_template: LiteralString @@ -135,6 +139,8 @@ class LogfireLogger: else: clean_metadata[key] = value + clean_metadata = redact_user_api_key_info(metadata=clean_metadata) + # Build the initial payload payload = { "id": id, diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index bc58efad318..c47911b4fd9 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.types.services import ServiceLoggerPayload if TYPE_CHECKING: @@ -315,7 +316,9 @@ class OpenTelemetry(CustomLogger): ############################################# metadata = litellm_params.get("metadata", {}) or {} - for key, value in metadata.items(): + clean_metadata = redact_user_api_key_info(metadata=metadata) + + for key, value in clean_metadata.items(): if self.is_primitive(value): span.set_attribute("metadata.{}".format(key), value) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 378c46ba0b1..7f342e27116 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -87,3 +87,33 @@ def redact_message_input_output_from_logging( # by default return result return result + + +def redact_user_api_key_info(metadata: dict) -> dict: + """ + removes any user_api_key_info before passing to logging object, if flag set + + Usage: + + SDK + ```python + litellm.redact_user_api_key_info = True + ``` + + PROXY: + ```yaml + litellm_settings: + redact_user_api_key_info: true + ``` + """ + if litellm.redact_user_api_key_info is not True: + return metadata + + new_metadata = {} + for k, v in metadata.items(): + if isinstance(k, str) and k.startswith("user_api_key"): + pass + else: + new_metadata[k] = v + + return new_metadata diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 81244f0fa09..1b5c1724673 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -3,3 +3,7 @@ model_list: litellm_params: model: groq/llama3-groq-70b-8192-tool-use-preview api_key: os.environ/GROQ_API_KEY + +litellm_settings: + callbacks: ["logfire"] + redact_user_api_key_info: true \ No newline at end of file From 09dcf4f3bbed98e251474bcac93d1d22df1ddad4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 21:13:39 -0700 Subject: [PATCH 38/99] docs - control team logging --- docs/my-website/docs/proxy/team_logging.md | 79 ++++++++++++++++++++++ docs/my-website/sidebars.js | 17 ++--- 2 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 docs/my-website/docs/proxy/team_logging.md diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md new file mode 100644 index 00000000000..892a5140e60 --- /dev/null +++ b/docs/my-website/docs/proxy/team_logging.md @@ -0,0 +1,79 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# ๐Ÿ‘ฅ๐Ÿ“Š Team Based Logging + +Allow each team to use their own Langfuse Project / custom callbacks +``` +Team 1 -> Logs to Langfuse Project 1 +Team 2 -> Logs to Langfuse Project 2 +Team 3 -> Logs to Langsmith +``` + +## Quick Start + +## 1. Set callback for team + +```shell +curl -X POST 'http:/localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/callback' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "callback_name": "langfuse", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk_", + "langfuse_host": "https://cloud.langfuse.com" + } + +}' +``` + +#### Supported Values + +| Field | Supported Values | Notes | +|-------|------------------|-------| +| `callback_name` | `"langfuse"` | Currently only supports "langfuse" | +| `callback_type` | `"success"`, `"failure"`, `"success_and_failure"` | | +| `callback_vars` | | dict of callback settings | +|     `langfuse_public_key` | string | Required | +|     `langfuse_secret_key` | string | Required | +|     `langfuse_host` | string | Optional (defaults to https://cloud.langfuse.com) | + +## 2. Create key for team + +All keys created for team `dbe2f686-a686-4896-864a-4c3924458709` will log to langfuse project specified on [Step 1. Set callback for team](#1-set-callback-for-team) + + +```shell +curl --location 'http://0.0.0.0:4000/key/generate' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "team_id": "dbe2f686-a686-4896-864a-4c3924458709" +}' +``` + + +## 3. Make `/chat/completion` request for team + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-KbUuE0WNptC0jXapyMmLBA" \ + -d '{ + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello, Claude gm!"} + ] +}' +``` + +Expect this to be logged on the langfuse project specified on [Step 1. Set callback for team](#1-set-callback-for-team) + +## + + + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index cbe4c204646..110adaab83b 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -44,19 +44,20 @@ const sidebars = { "proxy/cost_tracking", "proxy/self_serve", "proxy/virtual_keys", - "proxy/tag_routing", - "proxy/users", - "proxy/team_budgets", - "proxy/customers", - "proxy/billing", - "proxy/guardrails", - "proxy/token_auth", - "proxy/alerting", { type: "category", label: "๐Ÿชข Logging", items: ["proxy/logging", "proxy/streaming_logging"], }, + "proxy/team_logging", + "proxy/guardrails", + "proxy/tag_routing", + "proxy/users", + "proxy/team_budgets", + "proxy/customers", + "proxy/billing", + "proxy/token_auth", + "proxy/alerting", "proxy/ui", "proxy/prometheus", "proxy/pass_through", From 4465675d9aab419a802d6813bf46a3892ac132ee Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 21:20:13 -0700 Subject: [PATCH 39/99] doc - team based logging --- docs/my-website/docs/proxy/logging.md | 5 ++++- docs/my-website/docs/proxy/team_based_routing.md | 8 +++++++- docs/my-website/docs/proxy/team_logging.md | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 680264ec759..52a0d8ca7ff 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -202,6 +202,9 @@ print(response) ### Team based Logging to Langfuse +[๐Ÿ‘‰ Tutorial - Allow each team to use their own Langfuse Project / custom callbacks](team_logging) + ### Redacting Messages, Response Content from Langfuse Logging diff --git a/docs/my-website/docs/proxy/team_based_routing.md b/docs/my-website/docs/proxy/team_based_routing.md index 6a68e5a1f8d..6254abaf555 100644 --- a/docs/my-website/docs/proxy/team_based_routing.md +++ b/docs/my-website/docs/proxy/team_based_routing.md @@ -71,7 +71,13 @@ curl --location 'http://0.0.0.0:4000/v1/chat/completions' \ }' ``` +## Team Based Logging +[๐Ÿ‘‰ Tutorial - Allow each team to use their own Langfuse Project / custom callbacks](team_logging.md) + + + + diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index 892a5140e60..a6b7080dd97 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -5,6 +5,8 @@ import TabItem from '@theme/TabItem'; # ๐Ÿ‘ฅ๐Ÿ“Š Team Based Logging Allow each team to use their own Langfuse Project / custom callbacks + +**This allows you to do the following** ``` Team 1 -> Logs to Langfuse Project 1 Team 2 -> Logs to Langfuse Project 2 From ac3c6a4604f92fe60d01a1583cf67f27600aee00 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 21:23:57 -0700 Subject: [PATCH 40/99] =?UTF-8?q?bump:=20version=201.41.26=20=E2=86=92=201?= =?UTF-8?q?.41.27?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b14fb819a23..5dc8ab62d32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.41.26" +version = "1.41.27" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.41.26" +version = "1.41.27" version_files = [ "pyproject.toml:^version" ] From 34575293a5a193ec1309064f421bd198c025a02d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Jul 2024 21:30:24 -0700 Subject: [PATCH 41/99] docs - team logging endpoints --- docs/my-website/docs/proxy/team_logging.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index a6b7080dd97..c3758a7c713 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -75,7 +75,10 @@ curl -i http://localhost:4000/v1/chat/completions \ Expect this to be logged on the langfuse project specified on [Step 1. Set callback for team](#1-set-callback-for-team) -## +## Team Logging Endpoints + +- [`POST /team/{team_id}/callback` Add a success/failure callback to a team](https://litellm-api.up.railway.app/#/team%20management/add_team_callbacks_team__team_id__callback_post) +- [`GET /team/{team_id}/callback` - Get the success/failure callbacks and variables for a team](https://litellm-api.up.railway.app/#/team%20management/get_team_callbacks_team__team_id__callback_get) From 1a83935aa437cca63a10af073c15f5b9dbda18d9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 22 Jul 2024 21:31:21 -0700 Subject: [PATCH 42/99] fix(proxy/utils.py): add stronger typing for litellm params in failure call logging --- litellm/proxy/utils.py | 14 ++++++++++---- litellm/tests/test_custom_callback_input.py | 1 + litellm/types/utils.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 5846dd25a86..cebd79aa29d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -25,7 +25,7 @@ from typing_extensions import overload import litellm import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging -from litellm import EmbeddingResponse, ImageResponse, ModelResponse +from litellm import EmbeddingResponse, ImageResponse, ModelResponse, get_litellm_params from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching import DualCache, RedisCache @@ -50,7 +50,7 @@ from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter from litellm.proxy.hooks.parallel_request_limiter import ( _PROXY_MaxParallelRequestsHandler, ) -from litellm.types.utils import CallTypes +from litellm.types.utils import CallTypes, LoggedLiteLLMParams if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -602,14 +602,20 @@ class ProxyLogging: if litellm_logging_obj is not None: ## UPDATE LOGGING INPUT _optional_params = {} + _litellm_params = {} + + litellm_param_keys = LoggedLiteLLMParams.__annotations__.keys() for k, v in request_data.items(): - if k != "model" and k != "user" and k != "litellm_params": + if k in litellm_param_keys: + _litellm_params[k] = v + elif k != "model" and k != "user": _optional_params[k] = v + litellm_logging_obj.update_environment_variables( model=request_data.get("model", ""), user=request_data.get("user", ""), optional_params=_optional_params, - litellm_params=request_data.get("litellm_params", {}), + litellm_params=_litellm_params, ) input: Union[list, str, dict] = "" diff --git a/litellm/tests/test_custom_callback_input.py b/litellm/tests/test_custom_callback_input.py index eae0412d391..9c18899a577 100644 --- a/litellm/tests/test_custom_callback_input.py +++ b/litellm/tests/test_custom_callback_input.py @@ -234,6 +234,7 @@ class CompletionCustomHandler( ) assert isinstance(kwargs["optional_params"], dict) assert isinstance(kwargs["litellm_params"], dict) + assert isinstance(kwargs["litellm_params"]["metadata"], Optional[dict]) assert isinstance(kwargs["start_time"], (datetime, type(None))) assert isinstance(kwargs["stream"], bool) assert isinstance(kwargs["user"], (str, type(None))) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6581fea5f8f..88bfa19e90c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1029,3 +1029,22 @@ class GenericImageParsingChunk(TypedDict): class ResponseFormatChunk(TypedDict, total=False): type: Required[Literal["json_object", "text"]] response_schema: dict + + +class LoggedLiteLLMParams(TypedDict, total=False): + force_timeout: Optional[float] + custom_llm_provider: Optional[str] + api_base: Optional[str] + litellm_call_id: Optional[str] + model_alias_map: Optional[dict] + metadata: Optional[dict] + model_info: Optional[dict] + proxy_server_request: Optional[dict] + acompletion: Optional[bool] + preset_cache_key: Optional[str] + no_log: Optional[bool] + input_cost_per_second: Optional[float] + input_cost_per_token: Optional[float] + output_cost_per_token: Optional[float] + output_cost_per_second: Optional[float] + cooldown_time: Optional[float] From 1a33c407137f0c07558f8f039a4998b3d2da66f0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 08:38:23 -0700 Subject: [PATCH 43/99] add endpoint to disable logging for a team --- .../team_callback_endpoints.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 9c2ac65cc89..d51ca9ea1a2 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -190,6 +190,91 @@ async def add_team_callbacks( ) +@router.post( + "/team/{team_id}/disable_logging", + tags=["team management"], + dependencies=[Depends(user_api_key_auth)], +) +@management_endpoint_wrapper +async def disable_team_logging( + http_request: Request, + team_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + try: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) + + # Check if team exists + _existing_team = await prisma_client.get_data( + team_id=team_id, table_name="team", query_type="find_unique" + ) + if _existing_team is None: + raise HTTPException( + status_code=404, + detail={"error": f"Team id = {team_id} does not exist."}, + ) + + # Update team metadata to disable logging + team_metadata = _existing_team.metadata + team_callback_settings = team_metadata.get("callback_settings", {}) + team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) + + # Reset callbacks + team_callback_settings_obj.success_callback = [] + team_callback_settings_obj.failure_callback = [] + + # Update metadata + team_metadata["callback_settings"] = team_callback_settings_obj.model_dump() + team_metadata_json = json.dumps(team_metadata) + + # Update team in database + updated_team = await prisma_client.db.litellm_teamtable.update( + where={"team_id": team_id}, data={"metadata": team_metadata_json} # type: ignore + ) + + if updated_team is None: + raise HTTPException( + status_code=404, + detail={ + "error": f"Team id = {team_id} does not exist. Error updating team logging" + }, + ) + + return { + "status": "success", + "message": f"Logging disabled for team {team_id}", + "data": { + "team_id": updated_team.team_id, + "success_callbacks": [], + "failure_callbacks": [], + }, + } + + except Exception as e: + verbose_proxy_logger.error( + f"litellm.proxy.proxy_server.disable_team_logging(): Exception occurred - {str(e)}" + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type=ProxyErrorTypes.internal_server_error.value, + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type=ProxyErrorTypes.internal_server_error.value, + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + @router.get( "/team/{team_id:path}/callback", tags=["team management"], From 24ae0119d1e7e60b5b65a69e33524ef6ceae0b6d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 08:41:05 -0700 Subject: [PATCH 44/99] add debug logging for team callback settings --- litellm/proxy/litellm_pre_call_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 642c1261696..8909b1da3d0 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -213,6 +213,9 @@ async def add_litellm_data_to_request( if "callback_settings" in team_metadata: callback_settings = team_metadata.get("callback_settings", None) or {} callback_settings_obj = TeamCallbackMetadata(**callback_settings) + verbose_proxy_logger.debug( + "Team callback settings activated: %s", callback_settings_obj + ) """ callback_settings = { { From 69091f31dfd412de274bd713879962779e40638e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 08:43:01 -0700 Subject: [PATCH 45/99] feat - add success_Callback per request --- litellm/proxy/proxy_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 2508a48a1df..60ddfba32ba 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -12,4 +12,4 @@ general_settings: master_key: sk-1234 litellm_settings: - callbacks: ["arize"] \ No newline at end of file + success_callback: ["langfuse"] \ No newline at end of file From 3d390a79f3c2f777f6596b1825eadc0950bb4193 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 23 Jul 2024 16:08:13 +0000 Subject: [PATCH 46/99] (docs): Add OIDC doc. --- docs/my-website/docs/oidc.md | 223 +++++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 docs/my-website/docs/oidc.md diff --git a/docs/my-website/docs/oidc.md b/docs/my-website/docs/oidc.md new file mode 100644 index 00000000000..4d4c0d89e2b --- /dev/null +++ b/docs/my-website/docs/oidc.md @@ -0,0 +1,223 @@ +# OpenID Connect (OIDC) +LiteLLM supports using OpenID Connect (OIDC) for authentication to upstream services . This allows you to avoid storing sensitive credentials in your configuration files. + + +## OIDC Identity Provider (IdP) + +LiteLLM supports the following OIDC identity providers: + +| Provider | Config Name | Custom Audiences | +| -------------------------| ------------ | ---------------- | +| Google Cloud Run | `google` | Yes | +| CircleCI v1 | `circleci` | No | +| CircleCI v2 | `circleci_v2`| No | +| GitHub Actions | `github` | Yes | +| Azure Kubernetes Service | `azure` | No | + +If you would like to use a different OIDC provider, please open an issue on GitHub. + + +## OIDC Connect Relying Party (RP) + +LiteLLM supports the following OIDC relying parties / clients: + +- Amazon Bedrock +- Azure OpenAI +- _(Coming soon) Google Cloud Vertex AI_ + + +### Configuring OIDC + +Wherever a secret key can be used, OIDC can be used in-place. The general format is: + +``` +oidc/config_name_here/audience_here +``` + +For providers that do not use the `audience` parameter, you can (and should) omit it: + +``` +oidc/config_name_here/ +``` + +## Examples + +### Google Cloud Run -> Amazon Bedrock + +```yaml +model_list: + - model_name: claude-3-haiku-20240307 + litellm_params: + model: bedrock/anthropic.claude-3-haiku-20240307-v1:0 + aws_region_name: us-west-2 + aws_session_name: "litellm" + aws_role_name: "arn:aws:iam::YOUR_THING_HERE:role/litellm-google-demo" + aws_web_identity_token: "oidc/google/https://example.com" +``` + +### CircleCI v2 -> Amazon Bedrock + +```yaml +model_list: + - model_name: command-r + litellm_params: + model: bedrock/cohere.command-r-v1:0 + aws_region_name: us-west-2 + aws_session_name: "my-test-session" + aws_role_name: "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci" + aws_web_identity_token: "oidc/circleci_v2/" +``` + +#### Amazon IAM Role Configuration for CircleCI v2 -> Bedrock + +The configuration below is only an example. You should adjust the permissions and trust relationship to match your specific use case. + +Permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream" + ], + "Resource": [ + "arn:aws:bedrock:*::foundation-model/anthropic.claude-3-haiku-20240307-v1:0", + "arn:aws:bedrock:*::foundation-model/cohere.command-r-v1:0" + ] + } + ] +} +``` + +See https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html for more examples. + +Trust Relationship: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::335785316107:oidc-provider/oidc.circleci.com/org/c5a99188-154f-4f69-8da2-b442b1bf78dd" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.circleci.com/org/c5a99188-154f-4f69-8da2-b442b1bf78dd:aud": "c5a99188-154f-4f69-8da2-b442b1bf78dd" + }, + "ForAnyValue:StringLike": { + "oidc.circleci.com/org/c5a99188-154f-4f69-8da2-b442b1bf78dd:sub": [ + "org/c5a99188-154f-4f69-8da2-b442b1bf78dd/project/*/user/*/vcs-origin/github.com/BerriAI/litellm/vcs-ref/refs/heads/main", + "org/c5a99188-154f-4f69-8da2-b442b1bf78dd/project/*/user/*/vcs-origin/github.com/BerriAI/litellm/vcs-ref/refs/heads/litellm_*" + ] + } + } + } + ] +} +``` + +This trust relationship restricts CircleCI to only assume the role on the main branch and branches that start with `litellm_`. + +For CircleCI (v1 and v2), you also need to add your organization's OIDC provider in your AWS IAM settings. See https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_oidc.html for more information. + +:::tip + +You should _never_ need to create an IAM user. If you did, you're not using OIDC correctly. You should only be creating a role with permissions and a trust relationship to your OIDC provider. + +::: + + +### Google Cloud Run -> Azure OpenAI + +```yaml +model_list: + - model_name: gpt-4o-2024-05-13 + litellm_params: + model: azure/gpt-4o-2024-05-13 + azure_ad_token: "oidc/google/https://example.com" + api_version: "2024-06-01" + api_base: "https://demo-here.openai.azure.com" + model_info: + base_model: azure/gpt-4o-2024-05-13 +``` + +For Azure OpenAI, you need to define `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, and optionally `AZURE_AUTHORITY_HOST` in your environment. + +```bash +export AZURE_CLIENT_ID="91a43c21-cf21-4f34-9085-331015ea4f91" # Azure AD Application (Client) ID +export AZURE_TENANT_ID="f3b1cf79-eba8-40c3-8120-cb26aca169c2" # Will be the same across of all your Azure AD applications +export AZURE_AUTHORITY_HOST="https://login.microsoftonline.com" # ๐Ÿ‘ˆ Optional, defaults to "https://login.microsoftonline.com" +``` + +:::tip + +You can find `AZURE_CLIENT_ID` by visiting `https://login.microsoftonline.com/YOUR_DOMAIN_HERE/v2.0/.well-known/openid-configuration` and looking for the UUID in the `issuer` field. + +::: + + +:::tip + +Don't set `AZURE_AUTHORITY_HOST` in your environment unless you need to override the default value. This way, if the default value changes in the future, you won't need to update your environment. + +::: + + +:::tip + +By default, Azure AD applications use the audience `api://AzureADTokenExchange`. We recommend setting the audience to something more specific to your application. + +::: + + +#### Azure AD Application Configuration + +Unfortunately, Azure is bit more complicated to set up than other OIDC relying parties like AWS. Basically, you have to: + +1. Create an Azure application. +2. Add a federated credential for the OIDC IdP you're using (e.g. Google Cloud Run). +3. Add the Azure application to resource group that contains the Azure OpenAI resource(s). +4. Give the Azure application the necessary role to access the Azure OpenAI resource(s). + +The custom role below is the recommended minimum permissions for the Azure application to access Azure OpenAI resources. You should adjust the permissions to match your specific use case. + +```json +{ + "id": "/subscriptions/24ebb700-ec2f-417f-afad-78fe15dcc91f/providers/Microsoft.Authorization/roleDefinitions/baf42808-99ff-466d-b9da-f95bb0422c5f", + "properties": { + "roleName": "invoke-only", + "description": "", + "assignableScopes": [ + "/subscriptions/24ebb700-ec2f-417f-afad-78fe15dcc91f/resourceGroups/openai-group" + ], + "permissions": [ + { + "actions": [], + "notActions": [], + "dataActions": [ + "Microsoft.CognitiveServices/accounts/OpenAI/deployments/audio/action", + "Microsoft.CognitiveServices/accounts/OpenAI/deployments/search/action", + "Microsoft.CognitiveServices/accounts/OpenAI/deployments/completions/action", + "Microsoft.CognitiveServices/accounts/OpenAI/deployments/chat/completions/action", + "Microsoft.CognitiveServices/accounts/OpenAI/deployments/extensions/chat/completions/action", + "Microsoft.CognitiveServices/accounts/OpenAI/deployments/embeddings/action", + "Microsoft.CognitiveServices/accounts/OpenAI/images/generations/action" + ], + "notDataActions": [] + } + ] + } +} +``` + +_Note: Your UUIDs will be different._ + +Please contact us for paid enterprise support if you need help setting up Azure AD applications. From bce56c1356d1596b188e6c406f3dac983d90d054 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 23 Jul 2024 16:15:21 +0000 Subject: [PATCH 47/99] (docs): Make it more obvious where the group name is set in the example. --- docs/my-website/docs/oidc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/oidc.md b/docs/my-website/docs/oidc.md index 4d4c0d89e2b..936f0b91d4e 100644 --- a/docs/my-website/docs/oidc.md +++ b/docs/my-website/docs/oidc.md @@ -196,7 +196,7 @@ The custom role below is the recommended minimum permissions for the Azure appli "roleName": "invoke-only", "description": "", "assignableScopes": [ - "/subscriptions/24ebb700-ec2f-417f-afad-78fe15dcc91f/resourceGroups/openai-group" + "/subscriptions/24ebb700-ec2f-417f-afad-78fe15dcc91f/resourceGroups/your-openai-group-name" ], "permissions": [ { From 25dc0877d9316588f48629324c9d73fbae2c47b3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 09:16:53 -0700 Subject: [PATCH 48/99] docs Debugging / Troubleshooting --- docs/my-website/docs/proxy/team_logging.md | 70 ++++++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index c3758a7c713..3e6594efccb 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -10,12 +10,14 @@ Allow each team to use their own Langfuse Project / custom callbacks ``` Team 1 -> Logs to Langfuse Project 1 Team 2 -> Logs to Langfuse Project 2 -Team 3 -> Logs to Langsmith +Team 3 -> Disabled Logging (for GDPR compliance) ``` -## Quick Start +## Set Callbacks Per Team -## 1. Set callback for team +### 1. Set callback for team + +We make a request to `POST /team/{team_id}/callback` to add a callback for ```shell curl -X POST 'http:/localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/callback' \ @@ -44,7 +46,7 @@ curl -X POST 'http:/localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/cal |     `langfuse_secret_key` | string | Required | |     `langfuse_host` | string | Optional (defaults to https://cloud.langfuse.com) | -## 2. Create key for team +### 2. Create key for team All keys created for team `dbe2f686-a686-4896-864a-4c3924458709` will log to langfuse project specified on [Step 1. Set callback for team](#1-set-callback-for-team) @@ -59,7 +61,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \ ``` -## 3. Make `/chat/completion` request for team +### 3. Make `/chat/completion` request for team ```shell curl -i http://localhost:4000/v1/chat/completions \ @@ -75,6 +77,64 @@ curl -i http://localhost:4000/v1/chat/completions \ Expect this to be logged on the langfuse project specified on [Step 1. Set callback for team](#1-set-callback-for-team) + +## Disable Logging for a Team + +To disable logging for a specific team, you can use the following endpoint: + +`POST /team/{team_id}/disable_logging` + +This endpoint removes all success and failure callbacks for the specified team, effectively disabling logging. + +### Step 1. Disable logging for team + +```shell +curl -X POST 'http://localhost:4000/team/YOUR_TEAM_ID/disable_logging' \ + -H 'Authorization: Bearer YOUR_API_KEY' +``` +Replace YOUR_TEAM_ID with the actual team ID + +**Response** +A successful request will return a response similar to this: +```json +{ + "status": "success", + "message": "Logging disabled for team YOUR_TEAM_ID", + "data": { + "team_id": "YOUR_TEAM_ID", + "success_callbacks": [], + "failure_callbacks": [] + } +} +``` + +### Step 2. Test it - `/chat/completions` + +Use a key generated for team = `team_id` - you should see no logs on your configured success callback (eg. Langfuse) + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-KbUuE0WNptC0jXapyMmLBA" \ + -d '{ + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello, Claude gm!"} + ] +}' +``` + +### Debugging / Troubleshooting + +- Check active callbacks for team using `GET /team/{team_id}/callback` + +Use this to check what success/failure callbacks are active for team=`team_id` + +```shell +curl -X GET 'http://localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/callback' \ + -H 'Authorization: Bearer sk-1234' +``` + ## Team Logging Endpoints - [`POST /team/{team_id}/callback` Add a success/failure callback to a team](https://litellm-api.up.railway.app/#/team%20management/add_team_callbacks_team__team_id__callback_post) From c28697133fa101b0c66a8a66b6edfd27d024ed54 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 09:42:12 -0700 Subject: [PATCH 49/99] docs - slack alerting --- docs/my-website/docs/proxy/alerting.md | 42 ++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index 08030f47846..4df8b9f0655 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -119,8 +119,8 @@ All Possible Alert Types ```python AlertType = Literal[ - "llm_exceptions", - "llm_too_slow", + "llm_exceptions", # LLM API Exceptions + "llm_too_slow", # LLM Responses slower than alerting_threshold "llm_requests_hanging", "budget_alerts", "db_exceptions", @@ -133,6 +133,44 @@ AlertType = Literal[ ``` +## Advanced - set specific slack channels per alert type + +Use this if you want to set specific channels per alert type + +**This allows you to do the following** +``` +llm_exceptions -> go to slack channel #llm-exceptions +spend_reports -> go to slack channel #llm-spend-reports +``` + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +general_settings: + master_key: sk-1234 + alerting: ["slack"] + alert_to_webhook_url: { + "llm_exceptions": "https://hooks.slack.com/services/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", + "llm_too_slow": "https://hooks.slack.com/services/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", + "llm_requests_hanging": "https://hooks.slack.com/services/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", + "budget_alerts": "https://hooks.slack.com/services/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", + "db_exceptions": "https://hooks.slack.com/services/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", + "daily_reports": "https://hooks.slack.com/services/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", + "spend_reports": "https://hooks.slack.com/services/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", + "cooldown_deployment": "https://hooks.slack.com/services/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", + "new_model_added": "https://hooks.slack.com/services/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", + "outage_alerts": "https://hooks.slack.com/services/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", + } + +litellm_settings: + success_callback: ["langfuse"] +``` + ## Advanced - Using MS Teams Webhooks From 4559970a443d5ab402b1e2c964ffe07c6fc78662 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 10:06:18 -0700 Subject: [PATCH 50/99] docs - alert to webhook_url --- docs/my-website/docs/proxy/alerting.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index 4df8b9f0655..7c2e5a06966 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -143,6 +143,8 @@ llm_exceptions -> go to slack channel #llm-exceptions spend_reports -> go to slack channel #llm-spend-reports ``` +Set `alert_to_webhook_url` on your config.yaml + ```yaml model_list: - model_name: gpt-4 @@ -154,6 +156,7 @@ model_list: general_settings: master_key: sk-1234 alerting: ["slack"] + alerting_threshold: 0.0001 # (Seconds) set an artifically low threshold for testing alerting alert_to_webhook_url: { "llm_exceptions": "https://hooks.slack.com/services/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", "llm_too_slow": "https://hooks.slack.com/services/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", @@ -171,6 +174,20 @@ litellm_settings: success_callback: ["langfuse"] ``` +Test it - send a valid llm request - expect to see a `llm_too_slow` in it's own slack channel + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello, Claude gm!"} + ] +}' +``` + ## Advanced - Using MS Teams Webhooks From c1593c0cd1f2698ec8751fb2ce2559f18caaaa4b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 10:07:08 -0700 Subject: [PATCH 51/99] update alert_to_webhook_url --- litellm/proxy/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a982c6cd782..5e693deef57 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -202,6 +202,7 @@ class ProxyLogging: redis_cache: Optional[RedisCache] = None, alert_types: Optional[List[AlertType]] = None, alerting_args: Optional[dict] = None, + alert_to_webhook_url: Optional[dict] = None, ): updated_slack_alerting: bool = False if alerting is not None: @@ -213,6 +214,9 @@ class ProxyLogging: if alert_types is not None: self.alert_types = alert_types updated_slack_alerting = True + if alert_to_webhook_url is not None: + self.alert_to_webhook_url = alert_to_webhook_url + updated_slack_alerting = True if updated_slack_alerting is True: self.slack_alerting_instance.update_values( @@ -220,6 +224,7 @@ class ProxyLogging: alerting_threshold=self.alerting_threshold, alert_types=self.alert_types, alerting_args=alerting_args, + alert_to_webhook_url=self.alert_to_webhook_url, ) if ( From d116ff280e5b73ac0f808628b051b652414dd335 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 10:08:21 -0700 Subject: [PATCH 52/99] feat - set alert_to_webhook_url --- litellm/proxy/proxy_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3ab8643813f..04034827583 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1623,6 +1623,7 @@ class ProxyConfig: alerting=general_settings.get("alerting", None), alerting_threshold=general_settings.get("alerting_threshold", 600), alert_types=general_settings.get("alert_types", None), + alert_to_webhook_url=general_settings.get("alert_to_webhook_url", None), alerting_args=general_settings.get("alerting_args", None), redis_cache=redis_usage_cache, ) From 0ec2e9aa27ec04b18b1b16b74e20a0fc6c771cfc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 10:09:24 -0700 Subject: [PATCH 53/99] feat alert_to_webhook_url --- litellm/proxy/proxy_config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 60ddfba32ba..0e3f0826e27 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -10,6 +10,12 @@ model_list: api_key: "os.environ/FIREWORKS" general_settings: master_key: sk-1234 + alerting: ["slack"] + alerting_threshold: 0.0001 + alert_to_webhook_url: { + "llm_too_slow": "https://hooks.slack.com/services/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", + "llm_requests_hanging": "https://hooks.slack.com/services/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", + } litellm_settings: success_callback: ["langfuse"] \ No newline at end of file From 2ae5a936ea808cd77bef821ca4b7ded0a5afd921 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 10:10:01 -0700 Subject: [PATCH 54/99] docs alerting --- docs/my-website/docs/proxy/alerting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index 7c2e5a06966..b8e5ebe207e 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -174,7 +174,7 @@ litellm_settings: success_callback: ["langfuse"] ``` -Test it - send a valid llm request - expect to see a `llm_too_slow` in it's own slack channel +Test it - send a valid llm request - expect to see a `llm_too_slow` alert in it's own slack channel ```shell curl -i http://localhost:4000/v1/chat/completions \ From a8c88dad64b2d707c6660f908317ed0b943a3928 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 10:37:06 -0700 Subject: [PATCH 55/99] docs(sidebar.js): add oidc to left nav --- docs/my-website/sidebars.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 54df1f3e35b..c3f7e924984 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -184,7 +184,14 @@ const sidebars = { "scheduler", "set_keys", "budget_manager", - "secret", + { + type: "category", + label: "Secret Manager", + items: [ + "secret", + "oidc" + ] + }, "completion/token_usage", "load_test", { From 8845bd4d76605c4eb78a0a7d95fe8b29e6cd8b39 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 10:42:17 -0700 Subject: [PATCH 56/99] doc - using anthropic with litellm proxy server --- docs/my-website/docs/providers/anthropic.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index deb640b1715..496343f8792 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -56,7 +56,7 @@ for chunk in response: print(chunk["choices"][0]["delta"]["content"]) # same as openai format ``` -## OpenAI Proxy Usage +## Usage with LiteLLM Proxy Here's how to call Anthropic with the LiteLLM Proxy Server @@ -69,14 +69,6 @@ export ANTHROPIC_API_KEY="your-api-key" ### 2. Start the proxy - - -```bash -$ litellm --model claude-3-opus-20240229 - -# Server running on http://0.0.0.0:4000 -``` - ```yaml @@ -91,6 +83,14 @@ model_list: litellm --config /path/to/config.yaml ``` + + +```bash +$ litellm --model claude-3-opus-20240229 + +# Server running on http://0.0.0.0:4000 +``` + ### 3. Test it From aba600a892922ae99290d09e20c7a78962a6371d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 11:03:34 -0700 Subject: [PATCH 57/99] fix triton linting --- litellm/llms/triton.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/litellm/llms/triton.py b/litellm/llms/triton.py index 77089894955..7d0338d0691 100644 --- a/litellm/llms/triton.py +++ b/litellm/llms/triton.py @@ -1,24 +1,27 @@ -import os import json -from enum import Enum -import requests # type: ignore +import os import time -from typing import Callable, Optional, List, Sequence, Any, Union, Dict +from enum import Enum +from typing import Any, Callable, Dict, List, Optional, Sequence, Union + +import httpx # type: ignore +import requests # type: ignore + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import ( - ModelResponse, Choices, + CustomStreamWrapper, Delta, + EmbeddingResponse, + Message, + ModelResponse, Usage, map_finish_reason, - CustomStreamWrapper, - Message, - EmbeddingResponse, ) -import litellm -from .prompt_templates.factory import prompt_factory, custom_prompt -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from .base import BaseLLM -import httpx # type: ignore +from .prompt_templates.factory import custom_prompt, prompt_factory class TritonError(Exception): @@ -143,7 +146,7 @@ class TritonChatCompletion(BaseLLM): logging_obj=None, optional_params=None, client=None, - stream: bool = False, + stream: Optional[bool] = False, acompletion: bool = False, ) -> ModelResponse: type_of_model = "" @@ -220,12 +223,12 @@ class TritonChatCompletion(BaseLLM): ) headers = {"Content-Type": "application/json"} - data_for_triton = json.dumps(data_for_triton) + json_data_for_triton: str = json.dumps(data_for_triton) if acompletion: - return self.acompletion( + return self.acompletion( # type: ignore model, - data_for_triton, + json_data_for_triton, headers=headers, logging_obj=logging_obj, api_base=api_base, From b137207ae6ee4c71d4197bb44eb530eba9d2ed4b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 11:04:15 -0700 Subject: [PATCH 58/99] doc alert_to_webhook_url --- litellm/proxy/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 5e693deef57..df3b68593cc 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -188,6 +188,7 @@ class ProxyLogging: "new_model_added", "outage_alerts", ] + self.alert_to_webhook_url: Optional[dict] = None self.slack_alerting_instance: SlackAlerting = SlackAlerting( alerting_threshold=self.alerting_threshold, alerting=self.alerting, From dcb974dd1eb00b2f214984265fddd8b3d44589d3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 11:30:52 -0700 Subject: [PATCH 59/99] feat(utils.py): support passing openai response headers to client, if enabled Allows openai/openai-compatible provider response headers to be sent to client, if 'return_response_headers' is enabled --- litellm/proxy/_new_secret_config.yaml | 1 + litellm/tests/test_completion_cost.py | 3 +++ litellm/utils.py | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 16570cbe13e..a1af38379ad 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -7,3 +7,4 @@ model_list: litellm_settings: callbacks: ["logfire"] redact_user_api_key_info: true + return_response_headers: true diff --git a/litellm/tests/test_completion_cost.py b/litellm/tests/test_completion_cost.py index 22e82b29f87..6e4425fb634 100644 --- a/litellm/tests/test_completion_cost.py +++ b/litellm/tests/test_completion_cost.py @@ -896,6 +896,9 @@ async def test_completion_cost_hidden_params(sync_mode): assert "response_cost" in response._hidden_params assert isinstance(response._hidden_params["response_cost"], float) + assert isinstance( + response._hidden_params["llm_provider-x-ratelimit-remaining-requests"], float + ) def test_vertex_ai_gemini_predict_cost(): diff --git a/litellm/utils.py b/litellm/utils.py index 97eb874d68a..0beb041e938 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5678,6 +5678,14 @@ def convert_to_model_response_object( _response_headers: Optional[dict] = None, ): received_args = locals() + if _response_headers is not None: + if hidden_params is not None: + hidden_params["additional_headers"] = { + "{}-{}".format("llm_provider", k): v + for k, v in _response_headers.items() + } + else: + hidden_params = {"additional_headers": _response_headers} ### CHECK IF ERROR IN RESPONSE ### - openrouter returns these in the dictionary if ( response_object is not None From d1ffb4de5f53870678bd9ce69130da7c38b521bf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 11:40:26 -0700 Subject: [PATCH 60/99] docs(raw_request_response.md): show how to get openai headers from response --- .../observability/raw_request_response.md | 84 +++++++++++++++++- docs/my-website/img/raw_response_headers.png | Bin 0 -> 119648 bytes 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 docs/my-website/img/raw_response_headers.png diff --git a/docs/my-website/docs/observability/raw_request_response.md b/docs/my-website/docs/observability/raw_request_response.md index dddf75e9828..71305dae692 100644 --- a/docs/my-website/docs/observability/raw_request_response.md +++ b/docs/my-website/docs/observability/raw_request_response.md @@ -1,10 +1,16 @@ import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; # Raw Request/Response Logging + +## Logging See the raw request/response sent by LiteLLM in your logging provider (OTEL/Langfuse/etc.). -**on SDK** + + + ```python # pip install langfuse import litellm @@ -34,13 +40,85 @@ response = litellm.completion( ) ``` -**on Proxy** + + + + ```yaml litellm_settings: log_raw_request_response: True ``` + + + + **Expected Log** - \ No newline at end of file + + + +## Return Raw Response Headers + +Return raw response headers from llm provider. + +Currently only supported for openai. + + + + +```python +import litellm +import os + +litellm.return_response_headers = True + +## set ENV variables +os.environ["OPENAI_API_KEY"] = "your-api-key" + +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) + +print(response._hidden_params) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: os.environ/GROQ_API_KEY + +litellm_settings: + return_response_headers: true +``` + +2. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-D '{ + "model": "gpt-3.5-turbo", + "messages": [ + { "role": "system", "content": "Use your tools smartly"}, + { "role": "user", "content": "What time is it now? Use your tool"} + ] +}' +``` + + + + +**Expected Response** + + \ No newline at end of file diff --git a/docs/my-website/img/raw_response_headers.png b/docs/my-website/img/raw_response_headers.png new file mode 100644 index 0000000000000000000000000000000000000000..d6595c807efa387c2fe2008941334fde75a274ec GIT binary patch literal 119648 zcmeEug;!MF{yrcG2ojP?gA&r+A>D`w3|%7S&^4rVgGhHNNJw}0(A_C9v@{Gu{)YG7 z_pZBsSKhzi%UZMMaMqr4_SyRr&wifIJ|S$#1V>zNVnntxS9gwJ}N08yoMj6As{2ZKtQ=~0{@60lKkgb8j&91 z(O>705D*SuF2{KG3cK-tCqTm#^o9lA(H%_7fjuBT57aN8LWmb zIOsLV}l+u zB9X%zv9R+xKBWpRIp#jIlSTmJKX_{*4&qNAI%K!Nu%j1=D={XMe-3iA(qdt!AzA57 zzLoSDO%BEpe{S~ISwmmsO4@YFJ$52U9Vw?_MZ%vt`Rt1rIwJF^Lgqk90wLtYxx1+3 zAL8q&ycpxUzFp)i&f!r&X>$GpO=;!D`cFbu$&1;T`-0X?KmEgV13n^GYNV&(?bIcv zIT$HD@HlJ?NZQFNp^i%Y+OgBE?0=lWe`kbvQN*8#bS{ym!u(V5FaJD?J`i!SuoLuD z_`J{|%u+va)B7xA=!|X-S-7i3(C4{7{ zVm51UR=P0HyKnIGx?5@H`qlqt?nHb<=3p6`rUe)?^n5b?Cax;$vAp3eI=$oY9e*Xt zANqSLVfa4nLj>WwD}Iy>jzVPV8@mC^hMgVm5@7nY+jJU6d%RUc@*J@CkJj00jMT3l zoH6Y@Xw^hypw1jsNpFMRXWoSSmiXZ5_vdndz!g5GQkg$UeyWQ9E*FS#m`Pmi;tf*z zQFPo`3Dn=wLKZGLW2x>i`J<8hK%@-yQO-g6(8dXeABEas$62<|L1JP)3;ln%(Y?XN zqx*YRF`+DKJr*5y3Jd#~KL57ps;J6hYKL>TG4$*N@W7bIP_pf|UlvW;z#EQnmm7bZ zpiOr{s$f|i7Btvp{iCI^i6T$*ZEWhE>5oHfkIjBQoyvpuPhkFGlJ1i%n-fyS$2UMv zx5k&LIgK)jobi)I=;xe);GGAsm?1qn$)Q z(<8BvSJ$0n1&&^h7w>-SV~tl6vc_n(s;`ETs5~%wR@6Sl;;!s^N3Oa|bn43LH+gzq z>P>{O5`g1+TM_Qtz4R(hqE71l(v5qU5R9H23}N%iE#=mjUZB=en)>%$C}xC!B}p zLFLVV8t_b6LnQ*A`D(${E}jd5sFZ7(#^2Si@2a;Ox>Gdjs?H{}8wphod_tQPqWc~S z4z2doBN}%}a)?f$bV$}5Nh9dy?$OFW1VHRD#cBo+b%@%jO|q5Hs!StaByiqg6u*U` z5inG{M%4eHNt+@E5=31^E_1iY5;??!?X*aa!d!9n30nF@$Ep_3c+_ zN%@Lu7jj#xK|EbnPgtkEzxImrW^V%j>0#l1x8)}mN6u}LRg=%`IUVE-O*2y?nMyn? z2($c(kgFcl+{e73CWO_0X!v5K3)mU~Tz@_v>p$g2e1cCa@Pv%VbNXWZHE)ScaFrh8vP`|j zWLAm&xX0LyAzr%+sZ8oBPgD(TBgLj^C6GkK76-KHgVVhBtHR=?;{!H`h(18)<74i1 z`|G6Y&2N>at?V$7qfOB}SWThMm^wWj=rv2;Rbj<`-WOuTEhoS5jJg_yAIyatsjXscEuRiy z$E{=69h({6w_ujW!}dGOy#|Jm>0&rI&)V|W|Xwpr>E z0E>R4a1!>Io9W3075sKDRA0s7DK8jb%T7BBT3%K|@Yj^SFW}gO#Xa9V;_CY*41Z`O zmuRJRodxMeS#ataAK_cQtK{j%4a7$n$9YeYj|c4yS{_WD-0`hqOn1Aj?bk}1>F)bF zs5m(NtXfZV=u=U1aAL#e(V~E9ndEjm)zRL1sdoNKl?ym7;L$GyLV2pRPS3yVj%j?y zzFXy)Ymud=U>XADolP6uRXy^#md0)S`Nv%M7Y;$gL_W#5zFNK$u`RvW)YQOxu1+sw zOD^R|{_6J*t>c{H{cQv%9eX5Wn8FIq)b-I8M;DBn?Q|EU>zy>{77SB>)u8QA0>(tk zSF6(L)(@lycxGQmeryc4aXshRXtNB@bA`v`ce)Xf)yer%9Bit(T0Jsg2p{=2b>l8i z-)^$kj3XX{%~#*wFZ+?4DIn1@>=^}XhfzQ@$gXu+7vc1hq2JRb&JnPU$ey~P$Af+~ z_>yg;zvXV&`}Tz0`+ALH_yx|zf#nk_*Pze33y+HqRieVYaCru*ql7*dH{v)2< zrsT>&8A^WJp>0_i>}pTDjnwZJU+uq{dph@ihVOxZ1Apv`Dr3$_m9%`&ZKFYoABa z*Mojxyw}vlIu8%71Rk+@M6Ek!=dnRiHB*bD6K#%myo{dsX80Pnr65)oY3BH6flqZ^ z0@9Z=gsjMt2V6>>I%dDH!3$qK`k8AQ=iPRHx=j4ap8VPn`h8m#=OG{a7L$03>Jq3*YuxY@p+20CW#U zXIaBNNH^ zNL6Gz-M1RToyI4CzOyO`h$TAe_?n6$sA=7FHl?Bs_dbc&=p_HUg!vnf8Y&_`)`m^; zEZBB<>)W=Wm|=#Bbtp3Tt3_~H;t+1wk6r9LSgQN!&g9nZ};F(Wk$=r_nSzt=o)r^eLzpu>tG#n67%2>HlQ z@8Z{n`V@~DPl;1^D=o{ro2NZ&KKC%#fS?493kN5h)BTS*Y+KV~75cTU*bavMXw^$0 z4=hpOSkRohRJ%g=n+ebNZFV8%8t^B&m`OHq{-;W!G5JPL48XQLr(y*>;n!Wyj`B85 z(=I-h_tH)S%8a`ryMoCT{=A5TpG_o0R9Et@R=7nbXpv9)Iuk<(+z9%64EbiDym zA0KBB_|13mHB3FuqCN?zBKOpxm8F7CQ=G@xb&%Zj`=GLbM@H$oPh54+iWzBI%pViY z1H090Ve$`-!!)tt`3(tJ;|sqoXs6MykKnp^8pASZGI8@m!UBe9G$Gy{ zC-Pb}^18PHc#9hqAfy?Q!?E!oWh9%4ja0}zfTC1kn%=SXow-66pLwUE$3X)tG4G?s zV(G8I5Cg)REuu8Ew&1lk?&{U#iYEhZ52qKvPor!zW8nx>{{63vJX(OSLFC$%rQI!C zH3Hk&O6r$V(N9o;CoC^X?u>%l)>tiSpCKLF=FfHQe{~6lSm}AeAS6|c-GGOq7_Ew7 zRp4_0bMusr_bxa91)p_w4p9Jeu#?u~-h>Q)q6}_5hn+y<^ zT7ncrW-X}Kkkl^co#)1)OgaO_`BhKFFi&#A!&AY5WS%7DS;4f#=Cw8=1CHKg1NN=u z5<(>C_8H!2;y7)iF$RL|KIL4(^J1;5@mKe27{2>E>ZU(KL!algrQ5kF8&!gL*C%GR z@fu1J1!lM2p)k0OkGF~RvIQU4ueVTW~A%OOS$xCz0}i9?ZB%<*@plQ zn`nzxE8HrlSdkM&*te^nj3$x~&$Jr#7t7@B+aHtLYJL&*;WWcN4R&Ghp?%F3X&=;E zLCnh%duT>-_1Rw?kmOMG4`HehIaK>2JWqiF$+b%I_Y$WU?Y>8{Y z8V-c4cArHQL~yP7*uiPVkI80}UKx<2f0%C(^!MAz`Q8SQ2%6#MovGIu7H|N?$ivjh z@(+PQigJ=uV(r_Wqv+56A9Hluy@JR2uj!a$Zj9E5 z#-a8?7GcNcjudeRoF&GdV~w-$6`tM0G{Z|a_yk;T@Kp_ooa}R!bUSd$g&m8K@W|_r z^1mIRiJa|5{Va7S{X_b15cj{CjLj5D>4n)=y6PYw)(zJNWz=D{&as7+(o(Eh252&# zAfh;e%LG13v|2wcV)=W|wc;Ogic%qTMTz-T=_H!8COJT?$`g5Y{uP6T$F;BTxP(^y zGyZJ)fBF517y=wbEMvCJ1QNKC64l_610xT!z5St8X|nt$=Bi6Xzj$Dsa+KEf%d|i3 z9(Hxb7_kD2+^g}Tam9FH^swWCKx}v8V9`%6`igfjgBy)W&cf~6MXT5LpNAYT{zF># z6EjcFvI{&<+DHprp|GG<-YQH8l<8;Q?kP+(P(S`5Gk>8Db3Bfcc9vhDR^=*|n&_tY z+?ae1h$YZj&AU?Ce)@nMbdk@9n&UzMdvE?zPf4mZYEdZvnlG6)X75&?I|nhVMg zOP=T;X<+|@_r83aXXZT0e>J7kkh%TtKm+24`&Gwu!Q@*RcXMv!IDJK~z~hIL-&Q~? zSLHB#GVzNn;ACuWHV~mYL_({}wQJ2d4{^mJ)v-lsF@om7fNYqI{7#AUbSUwW(CJ9O zh9+A?m?o*rb%8S`UI{}<0Y~>f1=}SH{jcz^HWZ-ncg+?Lb1Jf#qDY?R&7SLJ-=x+` z=?@CCPg%Z9m%`4-S`^6QKTkkOHYm z%=x1p^4ML4#>6M>%iNlJ;cjR=e5E2B`f(1m7Dw-QPU+G~Kj(O`!j5x&=UdJ3@&L-; zLV)Z-ku;5#BuDMp!jUvyqz!QoL)U)f(BzM!Pf`wp)fNcXqy^`@!!Ej*w+oMHt=A2R z@)9?Dz@9p>GBncp01A*JvnX?eWsB9)KU96NSsleg!P5l)#Q~?zCeEUdS}g5UvH>@l{V` zB^+Pk_kl-8P|gZ%CWFFJJ%MA+}1?@b}j zPw;#)4x+k|5I>He-fh}K+~P`l)PLQ7P2H@6KEUk*=ORVtou>6F9EOE?1=qqkH|m)+ zyqeJl+N)}&w5?19_iIh6kGe_B=rTO)osYVRhA}(gBXc>6CAjw`&2da_7Pgro8s`RV zhHcEUH0*u#yxBDv9un9s%AaTx|1bZzqDpAD#SEe#Zr<}{m~y- zIVwFHK%_sTU~)B3nDB(TM6s_@={j`Pm0h2dXia_8BGiB&(tDm`r;nT=O2q%Tps(r- zTMW1s#ks^ZX`1?bFBYnxfBbdNxu!vHssDPNP86g{?qn23?Z)PSj3skqC`amSA%H?zv@Hlrw5gVjyf@^yi1zS`}SR0 zu9*FmEX^Q&aiFGRy9lcfcQ*&#Cs+Y3*AUj2*Gu;Q73SS&-{SX12_5%H`v?`___`(s z8G}ahmx}>q)b8zg>0f@s?Ghvz{#O6?IANNu0PTClYH0^n|1w_iaW+vC{ne05vl?#0 zl6#udRPWMRv5P4&_?O6WbkqkK3Qx!8xeO{YL-tCkGRdA zN`Jusdue8+rW+Jvlg78#&%UxvnO~qKGP^s?6OCw+$ocn(Z^L`(hS58I=}(7k?b5hL zcG6)0f9lx-FK7eWj;VVG$r9VVZO79J=9&e2RWD2%sh40qI1}49rDM~y2bVvFv-#FU zW`)h?@fYM_02=Mflk)$(3=KjL+)OpZ+cyZh{P z%Vx2eU))jIUu(sNcx#>k8!-FWa`W4!0^4&-!ip_~J8!$h1HyO`q|Fo$;!BQC39!P( zc*2wmYV}y$)vrPB%1yCOHnNBm+{*MThc-i-6jsAK{P-ytcH?UrO#{CVf0MGGJDqO1 z<#yjzFh~|&#f|Dw7CxKUb6>t%`kE`U!5-CnzThytXXAaP6NnOQ4Z}5jIh(S4z0qg{ z_n4+~gX>kI{*vhGe^-T+4xr;p*{A!+IITyALKGLrxIFuRm=3AA74<-8%`*13RlMB- zN6{Y#8KI3=k)eTIYCFj4X$@kr5L@~iC=XDhyiRjY8rfc6PUMNi>3rn`zD%PTAbR!-ZG@2bX`(43_&E#QsmGnkUw(NKz-1YmhFbjV_%Sa+BS8${Iqu6t4O6nVk7+Cuy&R3QB#AHu zF`b4GFVh~(VJ67^f}82@JtZ4AHRpA*8h#=xZPrk$vnZ8$1l zuC#W42WQXVC{*)9c)IY_Ud2gt4p7oVnl?ywsOqRkM#SF8+CDc1H0-_kP5YZ7GDw-ON=37VY=?!qCE*Xteu4A8}1J7Zdpw#8P%vb z#QAh*Cu(susyr_`DAK9iV$SP^J*@+yv5Fa3rDIl!L7^jh)t~7ca=q6`mV4HyzE$k2 zj;_cMN7h#U-0Df;pvM5$E7eD%PgtZ_0P02Uv&c9TwaMc3%gA)td_EvDROb+C4FoNP zW*cZr>j7R2-yk6do*vS*4MOoUE+om_zBB4EH2enIq(CWe_sm55-7$wn@2;g7wnsnc zzYp}}WC~hYGJy4I%G-2@J#*%xFQjp>=8T$wPuzI;<=FTogw>nhM1S)&m_k#C)C2A% zeKtD-4N@fgj(LI*iDG@Y10>Lz@!?`q56BFpb}0rN`%zp|-o?I}fd|nForP6G5Kequ z3p`BsXER^mCUEf)b_T=RvwvTVE4agRafQxKVpL)^+W{9uwT0sD0$gPw&-udj9Ser{ z$WjQ`yS07WL2>(*pejxj zBmX!PL8jQQ*u8HCw3)`a5$t|7znpI1-ZfB5``oz(&h~!L0amG>!$m9_1M`F+144Z! zfz<_g{2XqotF}|*Kh8uIreEINU5`;ICVc_6eN(JboCW_G6NLs09$$s7;CXoYLageo zm{H#TTv(~yGVj>(r%S=rbSW}l0`<*CWY%19c%TwdGgoa1vIAbjy;R$tBDl2Hnm%dQ z9!%ykaG*IrG~`>~6{NS%x!`WG4DV=yC$wRCu%xLV*&%}6`*Qj27V5!HwrT`=78cem z9Kb)}_tY*i*K{JQ#&o(>z1Tj^gPsnGRZFT@HiLtGd+lwzIIg_UTIrMstJ>Xn*>=K% zy3Iu>bc|W;gpPDAE1I_3B!OP15E}V{OQ8dmT>H$rG}n3Tyds;YsSZQEo5T@!eq179 zoOVr}cbG=G%CL<9$ zt(IznHI$1l6SGhJLzRg27*`nKg3{4EzV1K{KsrT;mOQWUt^TLIPR1oQ z9}B^;oAh#C5;V#ScFEwbC7iyx-qHTVb0&Mv?|E@6qI~)zTBSZ+>+ir^=nE4`Ed?r? zMKr$DNUwZlc5o=Q(39)p`a5MMmlHP=VI8ieuJiLt)ohJBV+yQ;q2m11zu{(a{Rz70 zVi24VAL4pkp3w9uYxpV76(iRoI#y{C|`5##9SumgK|Hes-LA;2cZjq2HNcp!#luufBBQ=xs40-v=y zg}zm6fQ^$++AVDnuUwB<4V?viCfl!Fi`01;>(F#a;NKIc z5@&;3wCWauP&_;l)uq|D@x|oFd`!mmdbre0=zfsswUITV7#+AY(uy(=o3Nw_jRfjR z)0BI@=&~Gum4_;F=mSf2vq-`5mB8%&YO){>(j^IQlWNB8cIxP$o_^|Z@g3{#9&HjC z;g;7C#og@7_UroHT~4=*t;25D$f32@0h@WEH`~5%hqx2&OE9`x&nIOynw zA#=bZEIz}PW}icC{5}vRnEDkVp)!di9qEDy9M$G7DV)zQ9MlpaKiWUr4ilf`uKJeg zgL)Tmy>y>?G@d|m7C3VTE?DIcm~p$SJhrleJ}y|jgBF8#c%P{H7bOo+TJn&C|KmYtdYk5l3L!u%a3*QI$y=pQeOrzUB~0G zJ4V!jC9suwJ)G)BAjnD0Qs_ReyfcVXRjwQmEVxIQbreh!*(1g2g=dB(z%?WcU3ld? zW)wsg??Ol=yMi;VLdI0t!@uHTyGY&KwP2@;T_mFd$Mx3|LZldMffVzHQCy$4C&VP5 zMw7l_YZ56_h9rvv$P-e!J^gTBvV|-Y12IIe!`jbk=aynwvBP!ouCeM`Y^{waFraee zFYAUW^~VrHoVcNtj>y`g5PP!R(i;`OT6h_nD@eY+=Kn(Ezbx1I}}Mqpt9Nqd2}TkC7h zJ^{keXa2`QLhDpv1>M2OL$9XZuC$kIsqihR@U8fxM5o?y3bcwq>xLoIWKc&!N4LpB zeS+D#zqt&<0pxdCjdEYY3k=yhD-bg{*0O>VtksN!q!IE~tmZ@x>^+0mZB5wUVzRqd zLxgSc!GfOQm{J}p%jn6+-?8}8yGEIINP(MakBGL%E>>hxX_aH8lGH)pVfD+QaQ>bS z6rsjR#(khXcgDrkCj&Guk@Kf;+N$wOR)?%U*RTud?YQspJ7POYjiT)8*m+ zimAVl)Cg0_Ebe3Orx1^^i%uf}RML3o60fsDqS{|FS|#2$yC>TeIIg7^N_l`cjJu;S6L-n#DWxb2s}LL`VGyn$!*?V@L9@`R+~ZcKB1 z_>7LJl+;r{9fua2MnC3$`K%<62EL~iSl@NL7m5v%y@J=Y@w1kclvphC@_{`r7QqKk z7(E3>Kla)!veo-X;$EyJzChWvVp>lse)J|*6=Q)@^m+xGwKMCHuS4kPJ`X>p#17%Y zUaI2ms?JB}@Y1a$1GAOsSg#pdICcm!rKOM|P;nFa_eGg6L(t zfXHIVZf-$#X!QUJ4gSuT$S-gG$2_QSl?8vtE^dlId$DmbaU#`NkSWcnF;{TvLxp>; z>jH3ag*!m&>tQ@n#1O;p3scdN{C0K|C}poD?XPjn%Y3_ch!r*jWWXH->V@`%sN}AO z)~_h2`L}T@d+2iCiLE!ZJ+*$DWQw=k=g*A@*ayqc1>bG|KpevMeR=(Bq()*vvQs=Y z!a_Cfvj=nq80vx;A^U{GNk)TiqXC~627QMT28v| zj`(pQ>k~=ts4Y}QN?Tlp0L$QY|C4#c9$GSCJ^{4Q9F4%buIoX%w1pU%T3)p zYW}m$?=+nbomPQhXS>p<1S@HFTi|ynyFdo)ZZ-W4|F&nL0x0+Iasfj=r2H-yb$LlA zY4*GfoVmiWEm0AqMpUC8UC@gIfeInXy`Zq&fYB?nv!HHO^)caPdv&Wnof?LUB~HIfO58ehD#1ob0-ll%)o7Vk0ws)`lcsXnZd`*(9aJa)=ny8GMh; zC%~0>MI3~q$CKbd&eu+a#FvKa6d9nLsPF5L=FIWU0tYFF(+||!Ah7WXdorfXgxk5@u;Q?V;`IBX$7whVYFk1-u7+Rd zDnunM(FrIN=B^kg)89fbSVQiPxv-&suYR{O#1va*y@wPWFxkDnnzH=tIzBS$q?5)L zn+|jC43S-Y~-$Dj!+jbBSOWjhv%d1kA?sk92&G|F!M)f^Ib)m^1K0LW}e! zkd}=#d9!o3?!7umGH@4p&01Ieb_HI}@C0uOKkxO=TV`~&5~m~ye~C^p50TAOhraA} zFFP93OM)UqwXn86Nr}g7307$x@-gWuOix^vK${*zr{xz+UYd|CGU?N9BZ394Ma1#% zrG1QUPPdk-?N-raYCIGo#QwQg*$PVYBEJjcEK+O?DQZw@ktvd@G?<` z2;c9o+wocaZ^nWjspLLs+ljUiPQ=RWn>>;_wRjeS+mJM0w^a%-jr7R9HA*BY>L`EF z79XJ1M>>v0NjS-$%OsoUAh#Cy-D*d(%3x|1-wwp~RAE+?Nz7Qvf!x&uUn*90A#kSe z2P(OKvVc~;lC7oBc@etwgW%?E{ zecnD)IZv5SKwy`2`WY+9!g6AKpc9*|=hDGWpAE^>v@j>6TR39ZFwtFU{?ZuIamRTl z5zet9(p&erO8IXF5JrhZ*xx9@mi(Sq7>T%^{Rs(pB8vT8r9#9A03MP19Sl=9jZU8A zG*I8``SI>oxn7TraRbXkDc}FX0p@NOAb+n^MoUM780Du0c6r62u&kgJ1c%lZ!1PaA zCk-r8aZ+hgVL{tQQtQS$2!V}3Yr|~0F8tfyVvYkSXo@Vte&I@d8#^u>gVVuU>Tkkw ze;Acb&@w*CjcJ|iDqK@AYu|4;(AT0s!a?izw{OSkGYa|A?MQ%@K12GpALxq z_8Mz5Q=T51@vxv$%6sUkLmj8fB|JzU8#x{tAGKUC6O3azG{P74>!V2=J`6!Sq$gbg zISx5CSsqgdB)TP62QA)|KNdLN<=SL57C&Na!pda;L|(chn=jHwmL}(#e1?HjNL#pb z)mNcp#lu&6>->E@bCTrnWk^}E*^2VM*#hHCoI^gDD% zu5v=8w4WVIYtk!b+JEba#gsTFflZCt27jo5_S37AWjkA3QmxW}kPJ<<3Od zc&uF~VfUXF4|_HC9}W!UMJTL&KM301L$pY(7oyQmR?+k52&DZ?%!`Zbwv~JqWBB9z zDNSt5(T`#!a$LGX0dYk#Ch*h>gN%Ky3$;X|e zT+ARty>Q|(sNs-f(dQ&o9*xM4^?UFOqK7rw4yQWpj> zl(y4rie#n3`&mv%Z61erFn>(Lt{p}k@!YDVv>S7L+R2G{wripcEM zC)?;9OML65XY{l#?x$PLGWkZM2V!{&q*0$Hiw=9xa5N{}VL{1kI$ zRVJbYfvSS1K$tYKtl0}InP}M(glRE~6?;Q*JS&c9hEFx=vfmY!x)!G(9cDLf!bAcKUk%0Rrhcob<)?-ve75E+#Kj&YsCM`Zrz=SS z$tP?GUqCNW+ZXmjCar~e=sAod$WBLd+Gva?3KRye#ACNNS=tc^xja8|FDf)!z}xqP;Bru%YZgW1vm-?FH`i&G?^V!i-VU}f+*5;;!ReDZzAr9!G?Ad7|pc{R*J#oX>qU3ueH8g^D=k@YsFmsvi&6d?ag)?>Mi_?2 ze?U~0zKyG!kp2D34_lvjc3@5$E`mUhU*1bZoR}|dh5X8vG~!I&Q$}tp&^)Ci6w z-bCA|B?pTV%@Z(NJ&&+??zxfk{kI%+_w^Ewz8-ntAfivM6lGmBc%fR40*TlSUtE?X zOUNH|$o^X)DNIy^{9WQOX21Q$RJRmBtNyiU=L|roN{K89TGaRbWa2w8KU5Uxy)i45 z_)&lTe$fru?4?1g#%pWP0R!BVb?%Fet;$H7S8{PtO?WT^~88yqZK*?AFhmX zbPNLpk%Wc=J8+nD+nB7!&px@{MjQfABZS_udyXsd>ILK3jRRvtAkG&=KbzEc&vVVB zb3ZTI_cNb-GO2WrZR5Fi)u6RbbN}^-q82B0%BXETtr~gGGTEZhu?5IqL0Y2jK8FVR z$ze&!u1MYq#A^XImKS_)BHI?R>pupnWt7{Wrbr+~fd))0i6lOZN_WaG+;7AhV!np! z1u1z=@QQ*2opW_le~5ul5oM{SQkL%|U&mQK$%!U7gNV>qebr4pX@$aDvZnd$=Lrg= zy`>hS65F*}Kq{^=;RCYAO@lA_ub&eVG5-u*uDP>r_#MBvt?bfQ(dBz}uhZtCw?68Z zWzxu7d52hPjb&S$2iNjz)CtB>%Zz>TRD{@}tB60hvtG2>z`65Jde&t{ApYVtZdM?% zrZ5PnWiJYw)SfLcL^S34N=;29;Q zJelj*e4p;S8D6R#Ro-W^GcWns>^m_cpvLrr9Uq}pRPA;$Kqw~Adz>fz`*UG3JN5!u zG8&{ctw@)RdWH{cNni5c_fN@pd|}Pi`b!KVtT4}j&f)YiqzG(DQ@_PG?mHcRPoo44 zVyI!>vEsZ}=(^8U124#dl2X)0P+VZ`K-8JaSCXdIV7{gILXIU6t_dAh=E-jsU zmHJypNIClErHS%7Wc3!#DQ~nmNt<=rSk_hkjB*}{_RsN)w0d3C;4qBtX}4&{`mumv z>Pf3OUM@`djAWS^o@wM?}s`8`B>6`tp3M{?Tn7^cA6VP&rjedWgVAte@ zo(sTNz9;ZuF>5~L+7&k(P17!rnO#s>y&XJdk|Nq9_FnC_UE(5#b|n^vo+mWn9Fb&K z;A(0pCudy!T~c@|^Lf)PUBOkO10f8$z)@(k8VBQ@T)!`;5XV0raiVK=H%n7}zdj(q zhV3@ucPufL$e@#lN)rIz$HyK0&ADM%8T(F@}m7yjSK5BGC z{c{*^P6cnR9{bm z1VVcZm|6zesbL>efblwc&4&T3128FpK+-XO@p*%@0!>q0q8LXs-Fw~tggQwfF({PS z9{hz{Kzqw0i!Fi&b%qtn_8m9NuQT4ltdT?8xa?FO0Gbi-0y z*!qR)t^KoJ3p`)T@YU95&)L6N#aU@4m>OJ;B@nV5QuZT7hFfl#TxJA>>+Fr7gKrX- zU!5@6N3GPf9uVx>NzXRI)v&>XvJc=UdV<~~k`1o}7G(Lk>@ZZ%qCMQcjH`vskAeq{ z4lF}0@EVOlI15;M{fa)f=5_97ZInB0S|Q(l+4U4|KR@q87>aPl={GtE)Oghph2n@G z3vAr1I&3Dqt5rA2cc-2&^@7#B41s|xFPt>=3QuR_RnQ|cOTJAy?E1eeyJM)nJeg6a z8N}%g(L7@)v=Dd|Hnnc*AD=++n8B#kDaw6;#+#@!G2BHm*UJ;G+SVUp^(Jc`De6l5 zlmZ%_nn;d)MFmk+;te&RL~@wa$Co#r?#7AfL{tJB^%;3X258|sl0uvTfyiz&pK+W| zM`g_151L$V+nz;HiEdXSEmY#7$T*=X1QQ0Moj&n~OWQ+I+I^UB(m94m0l;IN_S9J? ztU`G=Gu}D#9HBWl3-h&%7(yQ@%Ctdbav_3YKXMY5=}W*tjUTCR)0u&@)89uUB0LmC z;pNIwdIIi4_4Z%fqNHT@oi?Gknu!*m?%R308T6L4={PlLH9HOPbBt!#m?8Mv1a`FR zSFDv6W!Potj5egJo&!?D?6;0Q6N3>0m7){glbI)ve1vlcSF)0ltc3IrwNs;~N zuH%A8P|rsC;7tHZVD0pw5sUS{+V;&AohVF>1t$SAdF~|x{$WBOnHtZq{FGseDKYsX;;s|h{Ud8%Qf$%{vy42x}cU&tP3+HBj7{2 z%4n7asckDFh8hZChj2W6%bmt)+6o$Mlw>qlCYwQDjZgg(iCO$b_~)r_9XJ>8-o;{& z-q#G+zaf6ZCP^qsF7xX8C(=Qj(O{cQN$>3K$<}xL_ z>--YQJZ&r6sraY0c7jTZ&<}}uSEsYz%tiSePEW0MLV9Sb)wkjM;na^pSDiYefb5NX zd@N7KBn1>rW}_~4i$wHoO7+|iXaB0vt~&k;{c;y-Ty;hhntHAY4-Rp-Y=>uOlV0Wv^OTC zx#_{&XoKr0X1N4DW!}EO={YPv*X_>QlN(M$<~|fWA-7*4Pa{(2ZcUDL2o$@9w%iFv zdBf>Hbt!}RTSW{1u192e`D~3$B=cFSx)hGtN10(=y_KX;{2@FE-h9+)9%!PGfWywd z3t*`8#bd{pR7lhOUYyq&`4ZFa3vlLO%%LR!eVToj$q{zLAxwb5Hy*$Bg}ypXl@-3M zIv?M=>ke?AJ6B zV!B<;ymiC1Cs*z}Ei_X5py;ed6}HRr9e#%%rU(&AwhPY#z8PJC3*Ab$upWPiS!1bQ z^(Ehi?JFh`|06lQNjpGHH_sUU8XV=(czx<>>36531lS6h*vqwBQOQ{uNOKQ%ERAj z(U-|@W-YsCx%YIu@)c3l+hTnrN6zu z9kPl4r|J{xSnPu2=>#B=Cd1d5=f<{a5^VcB(sJ`v10rMam)cyJU+HeNEzD>(AIAk1 zG&IFXtZeO*=YS$RN9MLl6>Q1}*LfTW>5_uCykMpsr#$g1C5 zrIR?UsO?3S$hW9rV@dKsZl8OZ&_U0e@qag%By#!S43v>ZVLPFfk@g?io|wNOcXENh ze8Tbnarf4LRW0rRKO)LTl$25$B&18aOQZz^r9n!hYj0AzHYF`35{e*5NJ&X|OXsFT zV$)LJ+4p@s=Y4<9@%#bb-;U?8S!>p;nOQT}^?JT8hF6iD1>_-vS&$*kx7O~1hd(T} zt-RI)kaa8r^4`4Jy?lYAIebcgtfv2?t1nQ;NRl#}$yI-vr4k(x-|&*~!|gO}?XS$9 zxzDsy>%SIt-lEeXtWZA76;PhnvQg|R>Y|I#j!(S!wH^FB`f2Vf5t~l-%u=JKx?F}* z%i&F<0{_bs8)U&p)f^xFd)34`@r9p!v}*3Fiv!e;Ts}*~r!g!8(pH`R8R5XXTs1)b z@&`ylkHEyPQzxWMLKh-R^YxbQ!%BMvS5h~TU5fHXtQF>yl>dnyfm_n{B$~!Xl|LO0CMuK<{n+i?6bhWEp z>-`>HjqcTBxlA>yv&+!9G8-gEI8Ow3*Lr5gUk~vAsQrEULIFN8uHMP2tP6zbh3$!Y zWSSq!9tTz`?@lbLw>x$i>r*N(adasC3&Fu7^;TiFU`THAK_+`tE;0*VzDw&pYLj=U ze=ly7ufY16Io~vJ3VAAuIr8fx{J)QZVsI&a`4s5bY^;;j4DJ*!3EsJ9(?uH;Cg&yh zY3RdR@W;lZ_k z<_^?a4SL;JiVfnt`MIQW@5yY*gVldj@&6|a44|&P+~mV2vtohBaz_8o3M+|8sMbm~r9jWCy;NI(hes zi%2Z*W!7lsM`f)G<%5kkh0KOc|I(=nZ}StL#^YQb??3IcTK>{MqREtD9vH;tk$I(w zfxMaOo=q@9iT3Y}{9}X}6++h0yfd?;E=u*FeuVWze~< z$doa(XZGdyZ37lajGrB^Ux80t?dKC+tIkc6gg;UE{j0j?t6ruyXkd`+-&@ zMIrr#$qtiU#o+5$>+An#PW<&qoOdyXALynA1N@-;ZRqF&+whNxa#o$d4T*J#-KrBg z@H>ktg6IZbi1UCz${=rG)PxGsPFHRc8t-x=({mAiUvZV+D9%0db@}X*q?F&!%yCA} zuw0=>6AG_8Hprw-lzK2VB#rfqI^6VN+Pt!MzY-z8bOpc0vjjSiq*3b6Rgu-gst1%( zCnB2AJK7O?xsub|zw1!nI;;;a34}dW0cr-6W_6w$D$e>kS}b3)^EwrMet(}KhJM+- z0!0my2S3Vjv@QS}vw$WXDH81W^7-%o$VthF{LY2|auZIP=^1w;;&r*snmQ89_F~F^ zAt|FJgweAsOS7`sMr`A1M&9s+BTW`We(P6+>JxZ;Luc`_x9oBj)5$^s<-z@Y-S|s^ zjX~X+6x(VmJ6LkeKXtv13dXQwWvR)u09lxYUq0aj-eMpc`46lf@ONu` ztvCJLyEmCHO&Hy>Ocwi~3}1QPYASCt%0#_TLNs2(JoERHhh9zUZK=6vn3}<;wA$mg zVkjS94YMa}Jh#U0-$fT(B6eirwip(98s7$xy+5^tQcOK9GSr{G^)HaDHHNvq(+|D+ zr%72gPD{cD_u}~1xP_m5fByT%F+8F!8|obymmYB)T#A2UcRL(8c=O<2`VTli5nc&I zUZ#Cotvb$#^7#Fw6F9E$4SZj59SL+)9Y&txPjLV4d5wB#IES9uX=i#8-_gr+DCGsD$p^4_ky-~IiH2fkvf5e@x!TuCxL2lLYs2FR*2 z&63M@QZxnGKAPPAUArXE)3B5tZ4|J)XD@JSdPIQp$=D6Qlkt1f>%)jmEv=*bmoeD9 zSAYN4JW6jL-Tj2jn9pU^v{|hNe@pVndMuM$TKfY{--B;jW`|3&xj3!AG$;SS%vxjo z9DJCMB(v~fc6zxKsli0Wczf?(C)d7|kr_vobJz2i>pm6hNsoTJ^uEH#m7(58WUi2G zXVpE6JCYm_`TK(P=Q}X6#bSy}%&1Tgj%9c+To=9c#OP0 z1#NGmt4^jT&3uwu(2M4z4U@y)-;}-|*Ef0C+d_+JJFMm=x3}3vochx27tf)8fCc|> zs7aDX*Lr3ZU-yg%By!W)_%l7(RsG`sU+2M_%}%b=c!x$Ziy2c1gCi;?&cZ_E2YVTM9LV8GbBZ9T*GXEi&JI@l~I z?Y+ayLHJ8R4Z2(@Lt-F|+<_e1A@@n1Q?{b)-e;N7>YB)jmqEV}Ndd>DCxv>+yCF)8Pkwl z14vpmH@nr1iMxSglpdQg6prmQFk%#zM^ZSijwEOJ;m}5rX!H5o_3_;{stgK;W_)ag zB9G|}bH;S2Nl2-uhbMB|*ek>N{|G{-@8D_>#EIe3H~;oCqFQLpszI4@4aMJ)F* z8TMc2Yf(a`utj-F*z+t2N-c<_NRM=}$m+12c7aZe=scEcJ78ogL}=xhIDAgJSP1`< z7E8Z8ThR_8xk`C9hdCPdeWRdM1VAs9pWKR?(X@*mCmt+WU<3G{;WnaSQR& zag&7jHk1zeKPlfufAu~9QKXy$GO=e5F+Gndek^G=VbaiJ>sp%H0f3H;mmr(aH!D~N znBL{Zb0OW&B`CLWrjU@weHugx9xc|W+@v&WuTRcV5+0Y@5^JClh zgn4RAV;tXSPYA8vuv^_X;Hk9el3Z|l96!U25{F}#?}JM)Qz;QJyU; zw~*GLK#NylTN94(%NSRC6pZC_gg6gZxpp(?jSkV?L8Uf&O*A_Zng^T7A$LlUns-ae zL=b@X5PE(SY6mjCD?4i8ZahE+8Y4-nOAK(0zeN6Zd z`^feP#g#5Sxu-}AU;W0Ei;@Ds*#9IS>6R@W|NQKHUd9JwUF2%idOICaS8{cZ)z1X= zuGIJYA*0}|uX+fEqbR@7w>i$6#&0HkX>Q=q#Q^q4RC$2qebgdFup8M7v%TT@y7pex z5*Jz{ZK`^{($R6;dy#uzk?}>TeY9#?q=<7MNT`G7ktNeE1rVyV*8vuJMMH+Rk>?SG z;z1LxxQJ=M38UC<=*NN*6tV(a`3Ahg((WQ;3~D@N4M^9>JvNKSv#8EjH^IYmvMYAARWv0aDmvwbsun(rf z&Rap|)$>7*d}J8t+O3i2?1muOa^&~$0w@VMJwW=oGWym~J@q>ukm7Iu^bJ2-{2))& z431!Rx;*l^OblOyT=aP#mXcu2lYWoBln3ne&Egd1r(}5s5AMWSF=ENnTTtYp{x?~p zmjH$<#xZK~hQZ>x%D9BEl)@wqj^PU5-)12)-cU~5({MW1HcPWyGqag}?xW~n5qdh{ zU7<-Ib-bu5DA1|=z6&m3D@;iGzE!ZdMwlNu7Eo%gUdQ?t$Acij!*h8Np^A>(QJKko zG;Cw%F0p?Rh{1WrTT`{^{&H>)l?wd)4E*1N3)Hxs|dr10;BOoKPl z^I;n>x@ku*xr?0QvyONvaq9MSIF@___DP4X6E7O2&!RjJayrP?oBbiu@TAftaarQV z56^D;6SdhngzGNCjFV4+*J&N^><&G6g!HKV zmK6O60RQ#Q(|)`*-@T{5kQgu2*=3Jv{sJ9s=Kca5zn2-DEleDLFY^AfD28Y3Fs(*C zx)dqAl{8zF#F2W}jY(_mAcFY1n7@ z+=K%EHa-*nz~f-1Mtj=jC+viD_+F;zro=6C-w>#|(TGljk{!OjD}oMTdA1GqPuRPI z-1R{I@SMYAyek1>c4jQ|bRU8FmDi@a51m$|=5zG{*d*)*zO@vl3@I>^67)&vwpN~c zmxY-`R5K~$-e!dhBU-VP_I$3wGxjIFNKE6oYt-=N_!XweTTPKg_Xb+D%p})639gR| zH3M9YCzo zVjA@xGkfVdQi8ZVteuUGVX{Yn{N=1r`LN#4+^QN?b=~CHaQ6=?*ZKues_3uUo6OXp z2sJAgCmMlfBjPXQ6>5~DtpoB@wTnko#(MlB?T*u&^+xME7LpV}FKM4Uaoj(hj!CF$ zzdR`N$?4cTZxh{eIKjqc8P5Pat}3cgBlqdK4!q9c?Ry-sj)zhwrRfI+fwCfdQkaG{ zmI3}=$!fid*N#YpT*8h>C^y&Wvu%y}2v5FD)4p~ROInHXxc7w9jP``7KG~?3{553h zOfCsK9p}sNfcnAIikMr+mzdVj%eVXK1oL`Ls6B40sNApMYXg!VPwrM(;V1m7Qf@l5 z6-2e6ZC@E8xhE6i8FhjTNso8o`iCl$!+N~YoFil{??h&FIRjn@MWJ$z7m*##Or(p= zOuiqi7hCq$z(#3hhkg1r$xye^^Wel5 zs~=@A3yFXici|GpqY9IOfqh)`!+mR$Tdh*BjCrQ!@Hl(_2~Lzzv@59DEH&Y2{833x z3}<~8%oMg7@|}~G+T=d5Uw4$O0CBBLq&OOJrnDw4+mlQbmnfXvg{>a=UoKIAptcRd z&00d1fTqrHV%!sM%A~D+I}G}=xDk$-s7bH7p!tKRenK8lpn67hI7&$s@ICpjcq>qd zgov#p$lZGZ+XOGxEEJo_>F_6;$iCQA-TP%;H?fY?wOo;FJ{3p5{SU^+EAs0dK$RXEGY zs(Qsm5v`%NNuV=t^YINqHioIn^@r)mC^J; zJlEwY$$Kiho#C^0&y`CLgI0dqxD|A)k=~xP)Fp7+iA9)Q{-A<3o3J1ls%=SMFdN z1R=i>1{<~7-DG(j;IrNr$12d=s8bft{A5+Ycs(ddo;^zQ*d54l>iW$w z_r>efSn&Ty+~Og3XXc3>qq0){>s&V2uNXgZ(C%V9FE2oQCIc7>~jtSHqs3Q_-A_#%PBcNp>YV|>Uu+(bp3rc66$ zpxy_4_ojTW$PkaH<^9(SeGhGNKgz~3If>LGUxry*x8J6d(%?$jUvp^wR^PcB*~i1f zPBcDb+Ni-s9i@kiPe>feqW@@6Ds8mIH?c$=cjO@0eIB}bwUHiabR&uzpOf`L-qlm9 zKI36*-^i0{2N!gg=Iv@Ts47IMMFfutkHr4$Y0EFSLBQ0UGILG9UajzBP*!{Ig z^x6VextB5)##8=Xh(IdharmH7jAnqFkRlnD#&YX!%D2}w-Rg-+x~U|Yz|G1;y=-5v zl~*~O_9o8iF~Y}R{hG_AR*UnWc`}eQGWpq~Ff7mvlB+;js zBaNHPjSV>dO(p7Sc@jOiDiIegPFQxeWkjIRq(5rXX9E7ceq~(>Ti(K3e}b< zb$kXTQC6IC6I(0!Rz`(xqXxy*8E)Qg%Q7WC-%*hp_ z&A(~fZMX|+kVL&M!qX)?Xzkn!$KaB9I<0J5%hvHwrI<7BszJ$DV8;+Sh*8o5L?=#6kG z5^Plzgh~cez*jjxuG>?4&(@P&Y_GGv(#hr{KN>BgSUZpLAY1WC@=#jb_~qf~%NLAh zT|5U3HsZ~*6~g#TyW$nMqC3+x=q5g+mE1|(gHCsUrv5(xtUeeiJV5c*>Uw$vo!DeZ zVs1-@1;XB#nX*l5@?#C{Hl*kg)rA$zN|bi$6Gp4#+fQm@p$35paa=e-%Yqob)q**A z<_-{oA!jYW$IlosR0EKohgF(P>O0XB?J=}2hV8YSww8VK>jwpB1mutN-lxAAI1O)a z(PYk3Q)!b{+IAdNU|hl$m#fZYYZOiht1LVF=$vP%0grvgUxKC(8!uUa)F&y5gr_ z!^4pfsoUP=&qHZr;y%fSY)ZwvE^ANtB0HEHSN4G^C-9-w`Q?oK>;_0I@VKRa73r(( z+qDYU#cq%ka%zC8Y zb{mz!&8BdVyqX>vZ`5>E?Fbz;B)SynW6977?2VAbUdl4Cv#;$<7ZsL&d_qCv)Gr0o zz|-^887yURjEL@0+Np@&0A3Y9xR`YYP>};U#2yngU%V z>~YG?sXv|#;@1>uJoAFeCJ<>QIqe@UaTnbt=q;TTXUDV2 z8+A3ap1JD}XnAqz0yLWkJav&d!DK}ih}ijohva)n-{08NsLP4;F`5)pO!A6m)Acdf z!jd--A7sT&EQ)RkQNF{k=kMdVq>R;D?m`GF&9{Fphqy>X{i^L5U&?Qb89TpWl@jC4 zSu2l>3?g+Wp&&bX055s#Pw&35=R$ZwxpI>$y}mri(vavz_K(Abq{xk&$%9r0KCFuc z=#)*@)b{z9jkv?7*M-XZ@uO8s(f5iu3?4Wx@YCcva-4gL&SsJOS6lr2F-5IMd@z-u zYnvR_DU*F>49#DNXkIKE)_?P7MrburqG;O|m^$X#_6d{NRFghUcY1q^^rz&7LB@M# zpD1S<^R}OQn~}W5h3kZj??~rU?Gz{QzCKkI62f@HI%_Ccl>0-E(bpRS4_Av^FH$gi z44G3Hqks1}WCz+CEUBDre}ea}gwaS#EF_)Ab9qwknN%sYght_aZNplg0lfhId0*!r zUqsDTUraCZ8K|UOnwt<_{47QW6hnqxX6806 zKyGcr!tGP}y?|?6Q>J5N?*!7XLiC+;snG~>19tG1#vDJnM&Ao_lB;XxwzXEKV6fRE zt#F23w6;e$u$(@RbHh6_)ezhhDpvM-+}p28ehHz1#!hBIU78x*E^40&<{UM z^VE<7A2qm(NY`%Udjo%b4oz|rF1An`vfbVkfv3jVZp@x9d@Cvfw%?=YTm+;^Ux=TvUo8}*m1~--@VXdC!$kp^0cNS>l zRQVW>#Qh<7D))*S@^R8KH+3p9ceN%K=nl3wBaJE%1nR zK~7OQufog8@s6bE+e*`PoSmZ=Del>G)78xe4hnRxcggrI z0s7suCk{U)m<){riI$&^cFg%0R?0OFg#B=o-fQ48yEs|lAT93wu)$!u=%piAbjA;J z`TB&G<8U_rkicwL1KkFzd-Q>Exbyza9e&#fk!F8lw3P^D@wX}&grr;)2n z@ThjH=?6V01Ra((maD;0Xmw`_g3xN6!sQY&0g?KxVPnXWbzznHcZM`Ph8Q0+Aew&& zG+fAO*U3*~pdQs@tJQ!Uf2}_QjX&|*XSJQUzOFw6E;*-uM{rMC)w)u+i*iD+e#g;x zS;WRiths_+kf_Pg0T5|E;d4d00?HpFdi0DPplI9c$F#5L9`TP`gA`crdKdfGg1s~j z_?wzDWY-XPXTV7R^B9syH#0^a?_gkGi~;uM4x{wqHzMgHj@tdf^l*0&lFqTw)^liCoYzjor+hV_BWYBGo#{6! zfQ#FC7^YqGj?QzAISE_AbzHt{4YBXc$vqSXhm|2ahL2MqkQ!WdI#IJ1DwSgL%1KmKsdA$G zqT*d%%pS^qFvlcYqC$=@O_8Oj#&P4^m1Xopp&f$nqgi@^BffjyS{1N5;`gJyuV2#V(bVyNNXoLX1P?802)B&yiSRrWRSwefDA>+TLvG zGcZQ{(pzCC>CL02nB(#U$!=#P2K5TsdMmQsk3@#0WoBRGe|nsRmT-hn?sY}k zA0~>Cnag_4+|RsWCu1@tH?>(N*-~HHE5yPE3Z2zQ+x-=3z{8 ze(V4w47ULas`mrtYcV_-U@`DZf|HbvVVQ-eg5Pu{Uxk&CLdxyCVU4&ZKCOs8Da`U5 zA7UYbh};HSx%lLPQP{)q+y;aTv*x}8Ss_r}&JQws#i zQ_jWcUi`>&5FSz-l`TkttKO*qi;%|z5mzADIG(|$o9~;lGI~jboZc`n(O(s!;k+*p5ty&OY?5ZS`7i3ySYDNsf&?FP(Pgma~$ys zs^9me^GiNai5T8=hBW`=Uuuk)_@jP(gSAH93AUJRYDe2z;uA+Ikb!t1VVFTR!MkwT*($p|i@| zw}5mMb@$Oe&tZbmqz@&whCUL}QP+(DmXqDQF*Q%ieS2OtGB&cwan` zsN)K=oGDL?$o3q3W+^R7eFknXb~NRLE%%7_bruFwDpwr2eh1# zYNoh7(M{>F>0+#i!MB5+bBJM&T&Md@hls+KwIddFK$`c@wHB0r`^6T(Wp6)5_oab{ zQK)}5l-7|_YrL9jpg{>0*&MTgAU79A1n0#q1MqmhcI?={UlFJoDyl?;`E@Xza#7~) zn}fg&KBRUK!V*U~K-#qM^FvnTqgcg}AL098PHPb4Dn zD~b%Mb{uAtXF}D|9DLC$Etr0%co=UMM{qHEAML%TNCV>#Ug0iYP6Z{0-OXJwlYzaI zCFcjt&skYn?i%|riQgi_IE17~X93FYCjM=X>q1x%i*Fe3NJkmb^=zGA*o&&lHKV<=|(Zu-yHz zfLcq_)tF&I6KBmTZ4LqWZ#=fDEPi$k?W7INmx-p_+)Z(1p1`Jg-cQ~cM0i8Rzsgia zFruC@0?lO1htbAPqbBPl7kas6fKDg>YCJAf!>fmHl>R?#oudrReJ2rj80UdCr1_@4 z)Q1Kq0#iep*K~deSj4MI-YK@uZ%vDehCd&lk%!1Xz~v0Gq9JH&z8hd72{rKv5E%os zJwbkNq;=gKS(v^5m+V>>}zw9x? z`Si?G5nWO;Lg`O=*3$IBL>$65n{3|fQH!(J|B|9{&S2n`G;Q21Cbviu+D7Km$pVIs zvvDitJuQr|IJC&rnQwbcfOhxdY2-mqyIpF({m#01z?d{^+dLa?MO*vRnI*hBtG<=( zKo=P8vpums!9EJ)Sb`hF4EER1#TfpzhkdRmbsdRk<59G*exwsORzFL>94n(ff4-uI zfx!jENh|UgzT{a_@Ir9o-C41iSbl=M*vZP-bM7`o><^u&eAW!yjJ-DQ+(UgMV1r!m z-uwQ66b332&l~4SD=Q@sLKZiUV`qbL4L&ao87B3kc1T_0%Hb!@G{ZxOb=DxLn%*as z#pujIxHfUOr`5SIeh3~(+*y`Ad(qgoFLcWL?v@l!+f9qcum%C2hmoBQQ6_?QR>cogk+mc07kNH+gRVO zy%)985QKw&^|XV({2nhsET)Dbr+aQM)76zwJW1yrwRD5+m};-Gr(!pbZo04?uu8wI z@&;8ZmsRAYQcO7`ejQtf!qE^=O6^p)1<2Py=j8I>Z#6dp!0MAyD|)R;rxXO!8_|qb5+EV{V7--LHrIwTFE8me03Dhw^TaC1pK+Td^Mb z3jf*;tsT#L@_v@d!u`0Y8{%DD=@_>=SA^{J109=b@c@^~EKh$Y&n6cSj~*#$~?+AkxKSrZz$(dyUmWz;~Jc&RM< z?|-gNgXbrhzOXAlF2&mBB=VA6v^{In8ztelR5Y3jbd<{0d^(N%SjPVAwDRl5|B=@6 zyhfwZY-NLY!bQ|6%P{1^(I%|5m}0A!vb27_8I&R4e)HY5N+-sXRW&8>pS7?6>}U;W zC>t_ro6hp29qlO-h7k%vhaM0f1JipgJ+P@{9soA5L#bn5{?_>as(X{jFj^-RJ!#*6 zh)^qR5W#iy#itlGy!cDa+0`hO<~{{l`3NUx#m z^KCtls-j%VvcJ#f|C>vqww?P@Z}VHH<(Rh3+H8M$V*ew5LyC=lW69Lv4QuwlTc<(Szs1Y-oxE~*^MBaJzcmGZ7cdwO!lntom*T&|xc*2Y)AOxB&p!g8fAm;u1lZ_Trg3%s zwzxU?;njQKFsGJk{%b$_x8{J~EXaYIP4y-%{iPWG+vEEB5~wjzF{$wU<9z(@&wU-} zz|Cwex{?2Y;{1Bk4HAGxz`gxG`S-H@=!QysaI*>?bJ*|TDllw}-h$DD88+1YZ*P4H z!pdoTt-R)M&-2F|hWMgl>a-oopbvrY`^M7HW;O@-`VDP{h>-ukL-UWh38%awY$Ugx zy01YiU9>OVXY+53+{9W{7dw1!zjgI9qqQyN#Xn+kT+$!`&R-`>LGMd0syULdKC=f}2=kqV!)X+`K85`*6p=>_#WozKKKkUIUFEAI14MZ+0 zHvaDO^#Au)qhlmTZl*5sJc#^#@ITIkXM{SQyDi~~U~iwckJ6P49ig;lYkNRHD~s=w zCgTm@E%Q59%D!K!9Qmg+OPx52M^XMG)NQGZ=HLk;)gP6q{9vB1caU8i9Kc(kXxMh) z25RywD1nii+LvzjkHedg8jo_Z3axj!3KRBRenNzzQOE}c%cz1iOb9G=87dY-rZEz@ z7?dcn3qWLS9l-gBh=0mn8dtC~6nNio-WZ*crbI@YEX8ZHQ|=|T-A>eZeY)G{b5UC1 z?~w0HsACd(FN~yiRko#t#87VvROh+yAWH;!H$=2Fj>07B@-52z)598Kw!F0v-tV{ks}FzDl(!J;-bRTHI}v3= zDjRxD=ZkCQ=Y-Ia(sV6P5aVaZD+SeyH%x$^z_$3%W1RHU|1+OBe_+MQB0+$xDqn(9 zO$x7XPmL8lRdHrR9hhBZE&JHjCSGUAYx-1m<2L zP;oBhbxacR{Y+do1K>eMQd3bsca4K7eHU(#9gnIX6V3sQj9P$ZLI=bGUm7)S79$uRu|=x?@z0md^)M`KU|>0yw9d-Y0o?in<5_6rr#Z%Ebki zMJ;J=GlECgFWzin z?$&B&o()8KJ6DUfhhwHbsKx*#+Y)a_c`njvWSNdp+);LEdRtiQ{SZJoeOoWn6B&sI z-+GRB86i(J3yG|EUn;San_K;kb6uFr9TnbG{)VSoXpXivR$c5otY(Ajc)*z zdzwwwauQWi41LZITEL(hwSHpzha_(`S%TfZ;()ejg3M3Q2c>bmf#O(X*S?w6n4(I3 zv;Zaa z>UbLvc3=fU*{FLQND-ci&9hInhZ8LzE~5sXk7`ZmnH$P7*}N}3%|fFa(8MiH>3gaB zN1(nPts8=4U2rk7TP@<g9R2Ufs|W$eF>m_&U@!Ey`=)Dd@B zVIEj7#)K~g*sc-R<;;3T4jn>cwm}%Q1@K_%XS0{*hK>R^<)aOvOo8hnn*neIQ?c9w z*_INZY~=}K6kSuW5++|LtXh790L^jOt!xuz56qOIUP*(98g;Q%6o^I%;Cbz5Q7O|3 z{@4SN>+;H<>-n))NABkrJf?of0TeJTsOo}^`*C#mi`P~}d1a6&b$&~{wm~woN!<#B z$C#dg;zYN@mR7T(*omK6W+G4qiTp>WP{Jg~qFn~hEcQ75_-PlwEMy=B_uHPp^69+jM0i?pkC%5Wq_=7V>yF~m*Wf;Q~ljSw_58$-m?ati|kmH zb@;yLP|A$vju=T0;Rqe&Zb%&N-p%=2b>FVvy-i{OHKt$lB=8He|1^fOdNBbD2nRs) zW?e>nehy`*jDD|LN!PWX0?viPvC%3}5@qxz2F}^fZbrXxT!~dhyCMu%_n3Zp@LgNA zxFFP`=ZHZX&3vCK$S8d~-|FjH9gdx)jAP@-AnY?(-u|=n8wN)}dn~nUipXflpAg;Q zA7QIreMf=2ugG5YoH9ma5LF#=yP%1urgKL~9@2(F_^M=(5Bp?NAwRfE(4v0c4_|Vd z^>PbSO%Gp;N#BtO*@YGo2F2`ff)s)OE6Sx zZbbp|s@AHlam=Qk)e{Tu+l93y4$9n{Tc~1qcV-U3HsPLfnc@P8oo@*jhHXy#lSkoI zd^OTNjbgz@8 zD0RIGHQuZU(jEO%eufI&WmiI?oFEy06ggO2>UOwvXG>0P=n12Y?gncwdhEPQcPBzi z9F>;bIY(N&nhM)ftj|zsX4L*6XJPJVL20#<>Q7Jo;Uy4b3HB0MVpdh?2K?`~Ys5mk ztMOE6_kkp9`-eRNF{rlwUa@{rh8E26#|~2NysHjX-==djx$Bh$rwisvD$;R=k-+fl zTC#duQsd$V^0egR+W_uNG5U1#iqL{Nu7MSmI@NWHEs}|A&Al~X6EoPX1s!VP;ZyC~ zG2KdoJ}JeC5-U`PM+1;j>$+Rb$#E8CvwzrW>8ypy_d31*9#4l-B%rbKdPrM&MGp@6 zw(_=Iw)E|LcIh#j;VbW+&@(j8<(>~04Y)Q5-xc2PmyYf2XHcCnLlxG{%?3F<*9C^M zl->Q-yXi;G$SI{FXba2WVHq~eu7JlmZLd$fehe&!k9A*{bG%X2={9b$Yy89u3*~t? zuRHtmiM#eVurbmY8$)_LTHJV#aa|Vd>D1_Z-Y1~>QAr1Y;pTHM(-XL3tMKs9;RLCV z4-EYTmG(H;5KBmDs-J4RV{Kv&UT3S`-#`_xY$<$WEt?_UX(j`ae7k5S%8${m>b0(s z=T}HThwn$*%eC8Nse{+V5~R)L@AQ<(Vj7jtH37`iMt0XxNBAWQFM$QnnxwbXt+wF0 zo|K=sG2mtHHgbM~>Xe8@rfr85fW55viBKYq^MJMTLgvlUgli@MEWsJOrAkJ?>W%?Gkg^l)Q~Dz)B& zMo|=-dGD5Vxbv<;(;}$5$c(B+sS_uLuM28-5$R=|a5LEF5p*U8rXD*gp=*@iwX5N} z)x(0FyyUD^kAh!}PA0-#nhN=m)-S-Pe$!c?7LcvVls92j_#M(rZLy|!QYSKf;xo60 zNdgqZ-0yMFjnWs-ZV06!&uJR&=qFv;yD2rfG}jw#G!+t~~aiz82iPW?|y?iGe>-3aP))OW#3s7&PKro(q` z%udZgcnn^2UY8%!zLnWCR1Py&%aA+2YAVFszBl_5GV*2%4Ut@sYwzN5qdz!mZ4`_vxxMALks_if>h%} z$_mZlnhsTIi($J_DX8O z%(4zli#ro^Hmsutoj7Qg{LTndfP?9eZrU@2pCdRmh&$>0MVu8`S$tlTsEC$PRJB$M zUXQOhs=#1)WK{Sob-{+mGL~)~I_+ej< z1ug+%Y2uOCF%fcT2nT*sS6lbQHArSn$YAR1T$dma3QfmG<8ClmH&s}jBCGvzgqT8` z#=v_0-%f3S7x%W!cDC{Qhoh#ilqvbT8AP|uG$%R9xsxPiKks@ zPA%Z+Y*>^AYV`^Y6}jj3K0}g${i+5{yj$u*HAdfR`*^aaxPnOVDbT@BwC%lO5=szQ zm-Dzj0$5*M%qSLlvTUgcpNfU7;A}#B10Sj|5~e*Ku7!?iP|?NNm2@yT5&ZzAY#$v! z3|x(i;qLT?z3!E${oQuRVj0$|Og2xTNAwDE4fwqA@6)*C?JBF5mOLfgXYuLJL)!2Zpt)HKg3k zGEh$%F2X+?q79u*-#GodS)G1S_810#cw^WG{WR1&;MS2PEF@6Ky z$@)#zIRSI`Q)B;j!cp$nGERkCO)Fzm2r6T~>sA_CcJ`4c3w2SaH#k5&upnbFn&E2# zO-y$Y_tTh-Qw)QU5;>(f*xAVi43*Iz*&qMRk4M1y;MDKJy*84OGwcI zn}^BON@L28mx}kjpWPn*4yG{QD1M{c_?S3HbM_K+*K>ISR~5ESvQFMB&ZQ`bBXr9# zrJW}$&%1ALbLI4Hy71>Ya>T6`!R%N%xQa}ngOD8zCOaH?sOeHYs7yFs==S(1pdOQ`H*Y}p25&z7A~6e3HuhQ!!Kma^~0F2>lGdXA~C@Ataz z`}$qaAJ3o9{b#8$bIy4l=W%{MpV#{(%Vj#CZ)BYc`55c4w(|o5Jw?qznqwC})Ab_K z{n&V;M`feq-78FMKQWbkPv(uQQ+rG(F3a@eJWqK{srB72uWp2JyjC|GBxPNoVca(9 zG$lT-l0>%p)iVpk_)Rs5JkXmv$%_$pqUyxR`xr4vKsVfKvNkKEx2k2FwPM%uB(|is` zCOB!7L>G8oG80AVeef~u~-oy%mEC!Klg}Mu?Eh=NkN+&|A1qG#~A0!G0_Xk*|0Ng z?TnmKwy;Xhc(QpGhc%PPEcZ_Ss42bAXlnTNjYkYItWM7#1rR3Qmc2|YjZmuX>2OM@ zTzv9_tPyH*(5a>!a(&OFc_!fzB&R1d$?kd`|2#nW3kil>#=Kk<%H7z-eGSq^5>9RC~*ZT!% zr;77NfN@vW5{^bEUP@e0T3~^>bB$w3NIGch_R~-`CdzfT9f5Q^Jx-|)SwC>n*pW1< zv%%ZJYF(a7bO@0zhQU3C0m!rwB~nKp zTT_r+>KR9fd{Tb^MU15GoEbj{ZC#wri$tII*AMK|Pf#gz8~!0QTy=>B$Me$GpxO@m zo?iM`iW)Lsb^kC9R^%aBW1pf__jcmKm&^VN5FB-KnadARGS>^&Akl|S0^+T;*}EoG zI*QWRprqFY>Mx?;UPst)I4awj70t+@tdrG3`V8lKh;WTuzo@rti56EX@LpHJ54pIs6Ik;i9n3yN*RLlFhL472FIxyp;KuA&IF0&V@;#2ecHM@w z%T1KOeUH;3NYa3HPG4(eGu08+^Li>}KJ7LG%U!fIJ%@C#`OHm+O|Q|hF5iASq!~)` z-g|Avnx3IPS2^gAOALHtnzeO#M&KzvyyRh#b^`*Huxwd$PC72h6_Qm|WFNkt<{7?D zUrV1Y!0D2p^OJMi0w=8hs`y^>)*Xp9XoEY&)vIeKrbJ9=)-k!Xh3%7pAvam_^6fkg zo8^|UdWaxF9g*hh%U;bbKNeaM>n(%1qsdaO)`*p|%x(p^`v!rLqqyBFn>N5RxhpVt z{;9`@LkWQZ{KlOD+lKcD<9dSR_e?nTU!x=Kq%}b^m-E)<-O>2C>h7oQGD{=E7BN&_ z!{~_h!bdHdo02w}!k1#7bPzp{!)=l$dY{`e`e?NpP_Uq4vhYO)w`qsnnV@O67nC?cGl0*Btec($KGLRI3qB5 z+uJRGbVSJfd~k~eg_Sn>sa@*gP97!YmJiGhRbt-^xSBMIuWP4`O3x_xEktrWobnjD z4_8CJ;K01=HnKsenYxd;=|S%Qh3U0m#IM?1{)~3Q$QXE+bf}-^XFR&To>+n zx2C_Qvqm&orMJq+waFvcn@aPwP3|r=u!((5cJhhgO>fd{-fF7GF?9RtA?ncLT_0P4S zI?5O#m#<9zL0UtGqaGFO%M4VNR41kHHMZ7K*R|Dt^_5J?Bl&e9=-unen9@xw!yOHy zUxd9zyV07r+bTM0F?H{SI0m;!Se$DdcgaeN3W46?uiKrDnzwvBvf0+*sZrjO8^NKb zhfDUFhcZvC6RwNd|D2~O4&}CFR34FC1c_1Z}AryQ->-@TH!@$Yao}%Ws2sjA_ zmRE+#QlYQhL}`{$8@U8+A6UTUM&=Lc(a;xCA+AMwZiS+5-nap6J+A6y|K~K0K90sT zCsJ$t9wi7jiy=0=GhcmX^3@l^5Gb=VUYLKD^OVZ4A8=XWl74ORvy%n(Etd*6q{|!M zSnyd4(PEN*b6k{$XURrdJwfhfaIa#hmUBI`+4SvyxBdGPuVE?ITKpemyAETO zGHo(MuF7)Y9jWe%UUb1y^g?`F+1AL)J776vhOHkiOsT@&?HPZymje!!Sl#o4pU^px zxwNdSXUE4~f6P#757LiE7Yz4C#Ev}C;=4EJ@#z!g=YWsCAl}SH(=j^ml$CkimBLHK z3Z+8D3`)n|dX1}-{&e3RLv1i9wVg<~2DK4+oI+>K^Ljz7`5|jx!CSuua^Y#4x~oxJt6TF90la=imXUfZ`}BOTvD&5?eaH{U8oWj^_PX#);{6*NW^tL`V6>n<(gBM zjUayD8+5fm(X)K)DRZAwt?P$z$s(Rl9~Y~Vp~4X{d_s&Do`3amW?@IT*l7g`r34{F zw{9jtyN+mBf^~K3Dh>)g)ose`zT2A8wR3m#W)AY1cDXNQhz}SPD?T)`q;RsLX{%^H zVigB#`|MD+b-(#+&zeifOw`+L{SjbU(ts%HGxSL;sH9<*h5cS!21v1(8l{?${+7!OC@fBT8o@n7`ZXjq= z-0iH4gwyAupwPy6v~GmT;5k+;TAz}x=(qeo3|2C`y{9|8r6yb)WK1@dS_Seh|D@9X zys=^b(W8|x5S_pqq4J`LaLQVqSaFyC@aX;3NFIx-zJq9vFxu^7PNy7hOB*k^@fL}4 z+}tmbw4S|jI9BnkpQwCux}mBhgs`W`S=p2A=1$TKc0B6TJm49*YGr_JGsQyx4gP7j`E z-HtQTAXpHwPcvmOyFDKJ#2k)VN#Tc~)}-AI@I{X{E~7KFaUwkrlzd^0DzdWY==G{uE!c$Gb z;wM+#=OC7%3!sIfcQL+9zL!QiNM{h4Ig_AukWulp{IpXnng0h{Qoc2ar0t}mf+U@s z17NX4O|{BhIN7$}-lRHdHwdOTLNo^tTcVCl-LBEAhYi0-UDt76-)qcb)zs_oOUWP}Cz?ej_IOWr$7XVXm{R z64s(uALD0x<%?z1{8QtF2=hd`wFYRL?M=Z&G8XP0dWOv()d-j$y>y$SgxwNkc8h{Q z0hMr!kF2IqKR0*IbDtCmBxKPByDaB$18KMxH^BBGQ%A_%K7zr;f`defVnwf5v+>Nf)kA9Zz)!qRqjn6AcgbSjbbg4x_&>zm{N z<&<@{dIaJOzBB0<7yPWH@%1koY!+9jxe7HILhbaetFOhg-7wyNW1f-62T-=A!&uMKdTgXy6+SA3uUxO|&b2d=5I8<;VJ?}iLf^SzEk{Rg zS7~5M^+;{Re_Jw3W@M1qhlwwQ?@*CDQJGn|%*`!5hzXX`Z_j$*VS0h>$CoQ95$@{{ zpOyoM^0%oKPU)okU?6Q=K%p}83272C;7oPb>&-W#LJyvP|H=;lmy41xy-OPk z%#qD!NW6?8=P=C-54DESrYr)>LauN4f9B$Z`om2|d0-(1@32AzBFah7U5_tfcq^7N zEA0e#TriV;QgE8n6{O>YH17i@rBNZw{(42)nU1D{T>42PZ9U4=eBpV@k_}IIq7b9a z7~@y89uihL=HOvEv+*Hx;9WGw68ZO4o~dWflP48h^!j|nJ2#rS(^0Ea#a;S_RtYtF zeHK#bSsMFj*mFxjsJ)ft&yWH2ShLlA^I|g_2MiJ$t~n;hU>k7SP0!=>Rwp}37Zqsh zxh-m}6zKVVvZP6HHrMkReMXPd(CU!hooM#b8K0*2x2#9CoiyD>w2fJR&q?_wtFkv# zP8nv94+nC75)uBxf92vDUE~nOVm`v9k`bZJTy`E>SH^oFZxoSLub$t%B>v{`zAJMM z?p=Nt74s8lWwMPc=w{`21_s%jfNaup;M#+Er8z9x!gp5l#jPGUj%t*9e3634=Bb+r zgW)ReQ&u}d`lQI#fvuz&^{&8EjWd)^S$AYW$2=BVkEO%*MFH;X5a}nla;AX5`G6%` z1HC}eK}Fqcf@y1`1`rIPmpASqG^B9pWM-llD`sjnP5kw)JFyk}3UXZLdzGb1*y-~^ zwv2FkPca6kkA#59P)+8$5%ldu-`IYXAPc_Dx;w*IBj}C;n(ORq`~DLo-<}?x{xu+<`e(5#IgU-4gvMBY4fo9h);-l|-EdPi%aV!GIJ|;U!N`#4hqPmDEgqyB~=B zp61%C-o#*?ro#*n0v6Z{jP}xO6)#!$Epch1#-$~PjH{SU)+dAJD(RJyIoO4FP5Ilj z2kW}O<^#B&(Y`Qd<9h2@pHJJNPDjX&m!CP}=$5E!@#6j1rnH=XY%QG_;yUmVe#l73 z!HQ$gbYGq0w`Aa$UJ^Y$PR3B%44_iGn#~#CpA9i#nHZ5fwW>7YrxciaWO+ba0=*VP z$LV406}7!UjHN6m8~Cb`Mt&K&CshsZH@u0JUQ8!QG_(+p1u$ra2GlbalVRRWZ)a5q zH^phC&t77|>^+3=DgV&H-PlCTOG0Tapd2ss#3fo&^c*XxY0I`w4mo4uh^Evl)04o( z6kEQ&4d=H0oT8WJx%8&3jM*tIgl*~?Ix#{*O6f}~1;;t_lS&lUGRiK4v2?Y)u8Jsh z)Q-12r?0Nz(l>m)DRL?(&g}iq9|ycoW1e$_KysO76i*E_-Div(0TUdutFZ_wlT4vm zDr(EFY;E3*^<4v)V#GTIo6p;)#%cm z7ZyJsOQch*1ughBgg+nLGAc0o5NC_RqX%a(5tC94H=Baa-{lj#wYYh0a@HG>bI!@( z(v>2C?+Ja{^0FQgj;1>N%5uSWx7gl2tzjCV+JQ75UH8#Q_gw#UI*&>L{w{P%>xei} z<$c|W8H|lACL_jzUu(;1`ZRb4(FcIF;Tb4JHzZIa&rpYT5070N=H)S|~JA@}|io?BszM zPsQxbGL&EOb@4FUqWiFI9*Mvm(kH<>_!ei>*m|?&J!y7y&NppUWj%(J=NxAp zK${*Bo`Bb|Zy?rYNaP;^1zsz~6n1a>{& z8`-?kaGs@AFf8hJugFq}m(Wi9rFjw1<`F=_531Ja^_)9}T3#;if&9hG8}jZ&Fp=3eipM9SRdt*d*pv*AZx1WdrBW+R{q}^w_4gn5#zQ<Dq>69;8JuM2 z@`FS@i)>Vhc=3JSIJj%vxi!HkqWj6}z(t;G2cg@iIFysQAiR;%N~g0ZY0`IauH_SL zfjb^)1T|A}=Dfjqu>m7pldpS`4-^>FM4&BB_0z?VF{#t{X{ju6TR)!H3xbVWgvMuG zFmyQyE&vUI`P(a)XPl7WYuD23P_BSd!><^`;bTN~`3(7i# zK2Fi2v>QftlB_AfvEc{(U6nN_Y+PjNZJ%kKkU>8+D85f}9NNYuKV7gReb4>t(+^hz z-O)7hFj#i``ee8>ho{Pwx%~1*gJv=H6P%gcg38WCBY?nrRa$b3pL{;%Q5c7VAa%h; zY;&rgPpo*C6Mrn%2FLeFpW)AOvfCo|nGw&;Zryp{AC2H*foZ108YL0C!b``U1lL+a z2@2`T^Yj@%byw}$Z5Y}aro5`ULC{tl{c+Lnugu>uzK4@dD72i}qR42(sNJe0re-b< zhwVG&cYGG>VuoQh*GU3(EbX6MQgjePk6Bm&(DzJ4*v}wUGhNs9jHC9fQg%nyBO(lD0p=buE@y)KVgY_I zPwCa0z$_-}fnQVV^_>^49dKdlU>%E{#RPtbY?NBpfjMiUQ4A(@u)iqhs*;WF_>gUx z@&R1n+^Wm?qGL_iPP(^`cb;r1n3;U_1Qw-Kjh72kI=^slilu;gyM-`L>sH0 zU{}yc?yB{^w6bb&6~4{G&sbo9$Nap_Y(s6TK}aIkJca)*n2No=xCSO*8Qd;GY?bLr z83SO~&GU~rvO@eDU!Nt_C_3KN@tT2l|E8*L;+Qw#bLiOX1gEp@fxP0ZBBSAO_uS#v ziPhi8Y5{t@bz*B%SsI&?!GNL)(}B4*bC5_d49IM;SS@Tx5laxO9uGdDYW$X~6Q!Ts zR34>czTv7CADtefRDOCF;}#`xZuPmz(ynveFG>rb1?*aLPE8)o%7Y0!(tjX06(k#P zOxg~^V_{YPJeH_I`Ej-+ck;Xw{6dQ29-+E~IopSyns^SoB=6J?&V zQ95JxT}QRUX{ASxy4Xy3ekKz_Dw!L;eJtyiGVODaQ*YegrGQ0r@1_#OeCFd^4KnzD zrt3dg?pU_?kndV@Wg<6CPCG8|vOu@*z+4P@)7~k(EEczGyft)HvoN~bnH7iITdP@{ zg)M_}uJ`<3-?nGahLm7)Pw>U~XfEoGE59CTa#Z}fy6wliDP)boWO85l!yg;ccU6U} zdNpqIv5jVY{>x}Yq~eqnMWZLsGY0%ogj83^tQ|r&?&)ofEbrT0C|r^TnCwp|e-z|w z8GmQPc}fz-E-p?xtchKMSMkJ#*#4Ph)FfR$^}v}EV@Dr+cb@FC8jppn1xmm7KTt}& zAQ`(e;m-L^7`7uDSwCOd-L1D=*Gee?EVbIIrmeU-qgmwH>~XbQ!umA8KT*^i5TW>5yJ9B zZQM}9vAX}(4Yw!P88MwZAo_e7{LIa|+VnriTf#~6?A0kPyA-~0KM^J+Y z$yQ-`va7aDL{t;r>I)`sxz{#o260C#kNPWrd9EdIb`eSArlXw+`6ZPK@BbV?spGr^ zW&*7Z-ivecen!IJLlt@1_RAjCj=8WZkhXdTbjp4WGSt6=jDiudlfHLjc*KuD%=Cp{ z!~cK14)`}*O6-_o)!b|6mH!-{|MDh(NS=VKq}Kmr0f56Gm`vPy$5kBsBNz1hrTt@w zSe0`gmgL`%sr!J>%G4PC{y)0lzyvZP3J_zh=``fr|Hm66;v&D_W1Kqpl7($Q2>kxk z|M~mBeNGyBDR63WxnIBX?63dz_n-cGEe#&P{ItDXt^41<_rJkEX0zatGkWx5$+drY zfj@K8|MwvM--G0TlE=mR`=edb^G1%Jrq-~sf1MkDuB%_CoWBtF3wz;UUOQB{_s*Dm zwBzEHVPztJr(==(k7UlD>&sT|Mq#H8M8|~J~==drE!nM zGhTZ{ciI16E=Ujv2UcIcWW@ip6o9v!dlg(Ewq6fq1^?w*vrz)18s&w)fBM9KF5~)g zBIRkP{r2>~VV-v2Qmf3m&-gFDD~U+x=B>Wjs{fa3_5~$)*W&RS{{|ZOJqIgIx$Jw^ zq<>i;$~izOFgVusUw)U_FmYL{$oqEU-?&q8@UBO03jYW2|A(~%d}jde-0t@@`-dO? zb!fi?`zd-bUC;fWtzPW@wl~1?IGk~p`rpu8fcx<|D7A_H%kL7j103xCpHCwHlf&Hq z!&e4EbKdPvhtm;#L^7whiRu|2FqP?>A?J$^H$96VP@knEp2dC7z*!~=f5ZAujw%`( zPCv=IZ+9PkAtT zAiR{zE`!QEUH&A2eKrr=w9|=o(!8?8615knNr(_vILHpGtXi!Y2?H9TtPti=Ws+JL zebA#d4r&jzl8yGubkQE2mSMz*jJ{v!xX#dAQyfK7g4Mdt*c_#5w`GYx{OH zupSjj&OTG>tsvGOoB(&U6(7x{DM0dvBSYJjG@WnC3SmttB~JE94>-zu)Y$zb1-=36 zbEZH$MUGton(Z0`fQcP+m&XSJhxq+#^288OIwF)74r=2sB=K0<)A0pEe_d-J#?C6{ z5O_bnUwYQ^RO$ZxjnNVVH>?~1PBlB_OX|k->6FiYtvvclKf8hXD3m>jDDF!%j++58d z27Uun%y(XDA=iz-{p6Dx?Ao(4GhW4tdqj_qtIsn!+5X`L8f4oIlUP(CF$rqnzZl+p z(@<`omXtNz3$@Qg~%3GfjZdfnil){ zr!Qnu8#*&~vApMR2-Lb>`m{A*YB~y-9;K_FiOk!yPPFIkl7;MWDmVEDe<_RsfHh0v zeotT{UzRBJjhbbebyQjOt^krHJ?IGX5wb#zxZ)K+G(RZ=RhP0| zKrlSVv=Yk8%l7Vpygzc39d=~{GD2j|(svHMN9-rqcB)!FCy@HH$=YV;RtSRo>RKXL zGCz`znUGj&`JG+`gpA=RDAqRt546afswI8|iUh+0nT+p|E8j7ME<16C>nZ<0*=5XN zM@Y0?R$c^BU6KQ})9$$DlYzX~I&T;N$WDNN-IcLiO)K<6`oSl*ul=*FYJw)7?}&Qu zCZde(%|*1=e;!%dA@M5C!wN@?9w8PAKnZ#@$b?!V(9}p^fp4dtD@9Y2HxH-k;_?*+;TLq?9i6!3h zVcslS*jS14!I5s}{_RJqVZ>*1RCd@?TO;Geikb-yBiT zXXJ%Mi==5YT_qu|Zns*nf;Vv0Y&^#Q(L|cXLr?<}Ru8#iiBsv?O+SH(=?jt``MgCr z6OFgquBW~_3!M_Ri<&3b%*-XVWT#=#d|lGZg`yH)3V;F2B9Kx%>GJ({fCU<^ z4>S`AkoXm_Am6|@2|RTw(v;j5M|db^z9?ia_!T8KasPX#&p+>nG^mb~t_UJl-9#`R zrU0tVldqFtAQ8;@DM(82Hq{)_+;Icw=UtjMvU@k?KY=#ool!vQ`mk(&0REkTo+4kM zNh3!aGdf*lx>Rw9>^md zRc0ph3yLm*utE7(td>#cU-j;2Diu+|*XtjLpZ>^S{zdU5CCmZ^n@G6Hr}0`x;>{_c zIwZaMW;s!A<|9}Z=J3*rX(cOInt>h zIuRmAW=G~HhSmUyo*A_R*1Javht?ZJ1yCk$>av%-_k8>7&A=+_V$ZJyy|=m;=8L+j zcp4rBEN=t-rsCSM>o87rIBcb<(MyNLJ?lCgNtII{r(dVe!6j(UC69k z%^Kn6H1Uqb!zEIF&7=D=h#;$QWj4xm(|7-_o>?4TlZHcne;1txpt#UIq7;)%lOHR4 z<8iafdBc~w^G@FDMKyCn*2T?&cf0Pvd0S6A2=AJ~EhSI8HM@QCKGnZ78lthSoGyWu zJQ}_83pFcxQ(TqUKEQn0>#h5wMuckyuTdZU7QCA|3Y5LKR>2lwU};aKmV$ct1lH^$ zxG~dEV-}_jQ=UxA|K5_^35{&#$_VZ!JYs^z6-Nny!G*CH26p+Au8<7#EfTVKQYX z7qhC~fm-M2Cn`j>?cRsZnFHj@mUX-@bOkz**KqnR5)1l&|xuM|g@W`J-;g3&L+EsFblM?NxAL#uPkuqo&J zsBV-uiJj!>mLrPRqFAYHYv!%{FA@&BpIzc(Ucuy;U z^&>W1t4>Pbk-5=FJw>SoNjFgzA)GYhr?jYV#OvX$=@$!Me5hKN)P*1LZlUsa2!_zZ$4 zo~7Na!3B;7l${NBxhB!b5wOh_9M_?YavpX=@6mI56pzoA4%%iyjohh!8be__QVguBh(J0yN!G6QDjXltTV$4Rn>n|G=IvTA}J#~B!s8%l@#ZVm(GJlR5 zHrE(rIE0u5(*qiVb!TrYG~@zN2lejpr6o%2og}Pmb4io%^POIyFW9`#d-J0ipep<( zhZ}UaXc_B969@N~i!(7Q(@%>><_`(orjA#geg+R*rQJ8{FLtfdk?&b-I0~aX<7U1R z;)>qtNt$!Fk3WZhwlA#gjL2d)(k*tO)~RF{o_RA-062^?^lW_QBf>wCZ4cK8hyD@* za^B7qD{NVbNpWrn26P)er<9uvPWdJ7M{eMmtW3-)+X#xuyG@OkuRKpkY}v2M%&{b0 z^F`dfIj?K_MWKTb?3F=~(3z=jE7>s00Wktq`5cphh3*Tt?`I1wNkpYqobJ9$v9_-5 zJx%-G?#z-{h66`zU#PP6S-0;1G)$Df3~MLGbBAUN{^m2(KtBDfqQ||SRCm`5heJIn zYQ1xCx@Jq1n_f+FX+*^|yRtKK>P;l=X*-5-!Z4$vl$}I`dzF2&mz4qbb@arToQs2V zL(!&%q%KNAYK=vYilugTS)3+`&;PWsCvVNG#S4N8zhSnLsj+jl5fsbj0io-n2ybPS z=kmRY6fDD&MrjjQpivnSWey3WFK$<0vU+(1-v0zNxTnGB)B>`Gy<9_9z+>w z|BYdpepu^@;DZFj|3tD8w-a$MVnYlRTpxIOhT#ngy^^50$8soWVePQQDELNel$u|j zM1vtDAMTAGRT3?tP3()BKU5k7cjKAZ$*^mRo^V7kGv5<4ho}}lL_ zqrmbv`4W;XhfWhMK0)rJ={Ak;#OK+XArbUeN{%l@E<_N*^8_L_5Po0q26}7HGLCv2 zbAN+uYaq?xWX2ZixB|uf@)K^W;EyBfN-ZY`NTxYV*hGWhEIW5BQ!a^D37z4Jpi-|; zx;bwWC4K2z<mQ0u8{m16 zl7{VmhtUbw3L)EDX>`(RuiY0W`MkF0_ck|Vv0fXwBF6)1IRM@aOYNeSo-zyLm7AB9 zUY0zS(TIFSC`DulhHmM3%yl47Ia2F<>}5c!X|<>ioibc={ODM_u`=xmDfX>$kr%q_ zw`Meh4@gW5vBXv3Ur&$rRKpPS90TI%6_|2iVfJCMnC3rdX8mniNEUoeviPztnEYEF z30r!xMEN5j|Q8L$*R6@Nb7my)NS$AVtYjD#llEhqn8)T$G~Gz?9<+F z9Vd6x`sQd^{7Nl6<0?5Gq?p6L@YQF|J(vGoR># zA&rhX>-?$^=3pHJ>lt*Q=o!UMGgF8kMh)<5Ix!L+{Zn?$vkX|*4uU$M23i_XhOdNB zlBk(9@(vi7X>EZPw#&8>b2?OSs!+TUj=K1*B-6;C;CiNjr3|Y#y_Le@nV2*yNI6nr z=7^43?D?I)7`CQyueEgLSI{=~Xg>uxinbOa#U^rxUYk7EpEoF^{hew$FT|tC?-3OW&zf+&W)d zv6ZqX*06}I%eS&C)}&pSY6~IIqBov9vd7DO8@VcSS(fDz&WoY@P7y~$vtXRyR|X5| zft%>rmoxj>W&D_h>Qgo=z*uGbx(_-lBrR-HKOek~=F_T;+_^fS!XF#=Jc>tpOi((D zUbFXaC>v2bn!bOgc-7>pG^tBW8pbTq<+W$Ea?*n+%7`v<>C*%+v}c5txuoW75Vw`& zr0=@ou@?HTO*VN(q#OHcZ+d}F2l)=A|_4CX08e-u(-~eS3eFca<4isMYXg7 zqHuXf7n346n)C)vuSTqM&%@_Ar91=!IIu1VZLG(G^gp#{dNCrzW;_J_%FvYdGw6#+ zWn@Q)s^1Pf^OlW6-va=#Wj7x%BD`b~kUO|fbr2;+@t7x?l1%IN69p3BFjE~L9>i<0 z;(&qSuU1s?*@m3!q!iJbP<$^G{llq2g#f2kwF-~;i&KlPK@Rbv+JAv+`RB$%^+slnN2-n7 zCl-qC9)Sn)x@(igBJlYsjX*!Lo6Chv6E&+T&t0gXA=QY`;)oV5Lh%dbY6sCSUd11(0E zA#vD?%6vryBiV9!-C3q~~4eWW<)|LbL%0q9&>jGehS%l9HbwI*%=Y2e6eE*bAdBU$%I(i3QQT+w0 zDLnV>->|@Pvh0+@7>=IHsJ*{>nd$`n@JY@mn2ChSysLpqO7x2(^dl%d(O3u**48MNjX<$78C;d_m3 z6tV>7f0`-;H{0-v$n9R zk-G^t%p<*mxbgaZJkgY8v);@XR{mnz6rCI4$$ogFQY$X(3L{;lXRX5gx$&W~h^S}l zOMH2&Pkz9lC70(>N{c+N8_1)b8}j-M)5`L<_FA9UOr^W_E2kUJlW9t&GHxPcLq4a7 z48KftL|6$-v23!Wv{UNn%4>8`nXA?Eaf87q224kAFyk~+2|yTnoy33*HF(Zaje z0(ohmztu*5Q(!9kwSZ+QuQY2)7gP5m#a!r_utW*cx%@7>_7;(F50!<@Si%IKxyDcJpF2QMHA>3>+4ClqVAZqOqe0^K;N)PlS%S zec8Dij?J+*Ifc@#Pw5rIzY;MAk2aU-EQ=;YN%B$7G#v4bG+8n4XyF*xJ@hLhYt-^Z zaw}YUTA*|m98=#h(tk8J3hiJifyKl~{#b7ik@)_q#%|;aZ?gyj^TX!YS36KOl`Sd@ z`(Gqj9~xhLC#UR^2Klvzx~8bf_GfMO^IW7hT^x#5dM(#=TkbqPa;3#BM~RtsZGCc6 zr?6ilDbj1FAE37-_{@zwIT(KnW7{G%SzgIWoLPJ#+ai}8c&VMaLFD5|KLPHwgmOl{ zuWnu#d_)tR^BC8CSl#=U;BQ9|wepwR;Bh_Yc5zFnnePPRteBw@ZD78;MZj>4&sU)*ZpBgiS1SWxDJDt*6uxbsgT6Tn#H;WMU zFccdKT4B=gjeJMpk?La4#x7IEBegWjH~Q%8q})W}`n&uZ{jUKy$Z=uV>EviTX+;QXL# zZI0zYo;B;Q%2&IgsULN5EWW~?g@qF7JZ;X*!r*atdc9ZS*12F_?SWOVG!ZQF$1r2I zkDgj|#YnWJbWx)H08`6X?p+BMcUGDJB0~1cmg$=l3p?ETH$rww>VHMZR#c35*GBv5 z;BD4l>j~q!i3YuvO-OtO!>-<7V zo_PZu0EJYUwMxw8FHf+?{f;Jg&#%^a*HyLV9Wr#uU}pM*=&v~F z{?M#CH*P1S$n!0do8MvHfs7&!bddxXQ+5n!>dA}Lm2oc`W~$qD%RVcJBpVtG0Y%Du z<{uXqUV*QOM$M$(=yebsN$k_=o8<=CgNPH2Us{DihpOth}nwg4!10EWR|ek z%a~6?+7xul)|q1!uX>aPWK0_9vnBB;nC&`31(WThU~@w8#ipfGS8Jjq#_sK##+P=i zGtZ+C))Axhh#`Vk#ya=kVKE+{PeC_8JwqF{PNc~zn<5%SgQn1SEn6Zx`ngoAb7Rcu z&U!6g#7^KLWyi*&!p|ZX*Iqt4)7S9+dMiipL2u|bsIUtLE#%X_Q++!=g^SVBwJ4l1 ztZQaoLuOD4TJ)c5X+kw?sIlU^LfoP{|3o-{q4v*>sX3ZWbzmZ5VSCaaE4i-y!otqp z16bHFZol5!p}diPwGwR9a$y4ynJ#tKs_O)_yGY1n-3Xx3j@K0wQr({54i9%PK>S;p@~=SzHS~EiNDVL55rO zY4c0U&dM!2Aeizbpupb!roaN|rK>@|l0w>xkb2A?lVP6W4+;-XB=@wR-eT47s4&_lPN~U)`A0YH!91G@{pVH<@yDI2JZ-Mlu z32;n$E1=hDyU`1`^Je?%8B(nqCI4f0v!yISN)+0*$gU)6og~~n1&m+JaerJ${%^Sb z;%-3(F}hE@ns@GgTlaf#V7{6&P#W3ygxKwqu@E!AMPI9LRUJ3X5yITdFcdM5?8C@Q z9{<8$EKIujg}LUvt+RtOVqnVxl^D@8*Lk8DW0OLP@UQpIxe4iD_t)Y4Vn?IqdDk)) zW5RTtLY5jYNyoe-i4yGIui-e<5Do27HPKVb+p238!Dzj^P_-^eeT|Bzqxki>;5 z#*hQ+pmX{Q%=7_!MDeqSvNH{|z+0M0s4ypx#=oj<{iS(-`)$J?Mz(!Ij9Vlb3q<+X zA%jo@s_pf(l;6#|xs*rxcD-cgG<$b=Ql^9YKTzn5+f|N*ff^2PLU7`B={I733eB(o zn7{JVuwiquT$!zLYga6d<8bD8HhjeGW@+wtx$dHJF8j{8_)R^7Eq9nxE)H7X za!#YGF?^};uB&k+p%V0BzJ`H%9{<(vyyANxh~NuZ<7)|z)%-B zZYK5}JM$>)c-PkLjN5`0;Bx@tOZqS3%awmhW%S8MoKIxO8jne%T<~jws>%;RUpe5U zEO#__h6Fp9XoYDiLOx30vr5og($@^fxdsdGaUqaVDs;x>+f~T#&>W{heRoQNB9qV; zqVTJ^^vfrhc`cIFja;u;2zJi}B%f)FYZ)b?n%hm+r>j)ojDOM%w^jgx&e@ke=2Cjy zg=X_$nt(XL^djmS%Av6gPFKF%tL-Y)-M)S|+DB8Ax=;R^Uxj6a#iRZE_nqizqg01C z(hMYrfE!(E;rivbXHz=~Z3l;7p-+6C=`oi~DD0*Kg(dT|UXhwzMsg8@%|&(lOBb2m zr?VwaT!4ZpnLtz~%`V$<)(tn=Ibhm$ZYLfbdC2qUQM8 zViLR4rFKcf1Nio!Wna;PvW4^HB?miY=cT=MIX2fJ=jgSXnm`|7jnj4j49Xjvxo}I5 zO4)w;UNh#LPVt>rR|Rq!U5jhCuZJ_NVz-j1nI1IpeU%~FzdZMd#V=`rpE%0^>(TK4 z+P`wt{N^7;tTg;`e;Id%-}(`;6OupEq1qJ_@=fOWxJ)!Cs1^BYJY#K!Uo4??YH=aw zC4G+1xQA0yGIAdFN5zG=wG-eQYnVq^J~d36SGq2R!^W5Mn>^iQ2R>&(J2|?+@Erx0 z3UGUct_INwDJ22J%#G9#1&BXtiXEBKg41``PKhgc+*6(LAGaedAdb@F!^^?n@^xFsz(s z45Lnrp`3~d4H266`>paC;3BHL{vcYK&uhIxOSt9?dY)I-E z@n4M>9{pS6C6kPxk49^DbwDz>Ph*eFe;Wfu&ly%8s>?odZRjlTRJN{h_}P+ABp zLDKkrzpcKpswWFo^;M>CqTlFJ1A|sSN9$`Aiz?lZJB$-0OiDa6k%D(2Uy+rB!iyid z2@U7ABxz3*8dO?@<|F4PZ{4_>fL%?h3Jo>jT@rf&)Q7GaUSJ*{B!bOd`TxWeY4*%%R*i*fkn{$6jMBASM@FY-ssAjwXlU!nm71Pg=&6?l2L;mcqn{pUuLk5F z8i^Z_l&dIFPx>66XHv{t|A7`^U8BB|6WPUA{nL6)V{)~iSsugGxi7(fL5JoXL)a+} zC6vvrmyoO>16yI`;9DFQZKG}qE5CUy8Y2~Zz4;c5Aw`t>=fRcQJhvT(HJ>?`?&=*6 z`An}JDKBs;4|m_Srk>a;a>xIm%faPekM)Y%*nB6k-Q55b4hA_P^>t_&mra?b%?A4+J_JQMTWGMK$vs3_^o z&reDap|Vs8YWLu(5chomA{z@bSjh*!P-;CE{DRhNmuX@bv4P4<1Id+yS}^RA2gIaN zD*ND0B~Eq~va(!QrnPS=+hFeseJ9TXC5BZWETLatRvq(1O-G6k^V#LHw#Pl8;bNRZ zx7~uOUo?f7xp`1|%2M+qB{m%W6D5L?Y$XeOU7pfRDGe^r=D-aBk*|CH^{nPDNSoZZ zLHNm1aEr*TLZO1pNOU5$Ly_@J>5S#`LpE^II-Cj}TWawwvllbJebo@9KQ{pm}K*A_tw^k@+7L3YVk62G@pYau5 zLcxw}ng>sh8vRiQJP13+DXAz$64rUZdcz-*9|{wam9H&Rg@CT zUDY8+{YjQsb+P4${bs{sGed9hI5eXi%$&16O(&F7?3ezp4K zgtFX0DMog`YiMXsPGp-MURr%zSebh2 zGS6A=v*g)iEtW~=WqaN|v_^_?O0)MXABuKKC~vez?=ESUxmC(&RgO$)N|Hwi3*UXw zzFrmaU(iZ56&o(|3$z+u1v`zKdy~5SD)S8I55YfqSHlAX}I(NIvV(Raw*xK{3NT)$#f?Y9GVJyA=3{hJ+{M=bmGydHgHm zjELu1X|TGG2@lsvp@$)l9v?m*#)&ai5QG75HNX(uvccK*K0WlR?L8-1=5N)#)i-=* z8=YoUc;$Jr1p;IT4nTyUxSirCbxhjPBHSwWn~7)-3dH5n4PFPQlOfcc&qnDq1Dh$TE#W2N`F^#8E; zo>5U{ZTqJn3MfJW3Wem1fGMI?iyk|2m6 zAekZ*Ip?6WkD>c{-uCS^pJuI@`G4uk~DXI}fVYxY;l(sZ@E~xCT&Q&m5?(1%?6? zn6n-h=5&1yc&9+GQF~TVBX7{Qw1E35CF@1hiwI{L5s#>zqpZ&G_c{k2I#kW+F&F1wR{q_rcHBP#MlJ7i`bZ{&0@WCACzGK07u7K2(jgor~&f z*>fpda<0W8FT>9l5|m7odoIpDE&L^Fj+H;+f{5nrsB{T7wUazLa`dYhk?Uuz$EqD} z$9@vRh`vt@2Rf&TYn#SIa;1;yzMgR)(KxKhkd2=#%&j$k{tf25lF+bD?SwVd^3xc- z@hD`om`p$?VO4BX$nU0_B%gNTU2Xa|i&GB8W_lHdiOafL896Q8@qPGF`}Gg^alWBD z3^-TcBBN4u>Ord;9obbf-;vwN2uou$UA{7oKU2)Iqki#m6cIpmH>B^+jGncX`A{|F%AP5G82`kn)0T4^{hN z44twtr5F`^$)mqD=pkxHVp3{fn@6!AX`0%mk}e9Yx$Yi9WuzxzKsHFt_)e3E;HGQu zF9G<3asWI+z<2zlZZVX?e=&3VJ}t8U0bYA1`;=(^lYX#57`a{A5Gn30ORJswEz&lx zA`Lc=!lMp!aIm&j28t9ViZz}}b?9EC0rB{9T-@wm0&_W9B67)@CzgQNpk^s9kcwr# z>+6C~^P8r zCGR$8Axc`<8D94HS_OL07yV#I9_vLD%LXAa3l7wR5_bX0HjbF17FkTUHax%#_ zUjo05bPPNTTP)_o8z3|87VT>=6&F-Rv`DOP`tslP67F=>Y#(0+RJN-RSWn)|E39}~ zwmkxDP;N<&a{jVGfe^EBfZC{l$S1MHyUQT8e$o_4TUnK1RbKSB62H!!P-|KuEp&AH zIPgB@=i`mKw4>LqAPZCNi6D~dJ=x#F4L(8$`};oQ?`Qh^S*ZZ46Sl}YLG@q!4IF}m zpm56`u)zA)6Zof{$lMnUwrHneL*d`|`wu@1Cjg!UO1UlAU$^`}Ohmu_C|CmyIM{cN ze93=*(|=qpA^;prNmzVH{_SFB;8RZ+izWP<#9vts_yC9=N(cYj#RR~oUM-+k|Htk7 zkFUWF@tg=hFe&`Yjcf)g+ufbsNX~zAK~MtI0W*O$1?#__A+x(TZa0 z|F|Xp^N;_p9Y!akoVxzk;kd^QRIXwh8DSFO{=9h%o-kkD2$=Cu328Ao|BJaMaFyMh zxP9k~nk=XA#ueF(p?`zOaixO|9JSf~Ztc4C{8_}oA4j1jQB6gd!{6+&pRVqIyf;5b zMk+K`xnF*b{cTYA=Ud3~gNWr}ZSrqyL$`b2+ory_sr`Qg_4P}7^M@+|WE&}XQ{A#2 zORN3M>->vhw)sCt)qjjK7i5v{CicpYf3O#cmfkeabPl`fRe$%ae2;Oop}=&%Cev zX&gQYJA&H@2d(}A4%Q^>qhQ-K5B*i3oUQd_7W?t9t-(N#HStOK?Y#c4Io@5K1@Xa+ zAa&S{0;si0(2~I(-%hs&Jp`K|b5V#erulZZ{D)V`e&eP;A467F4pD0oWL4K0$TWy!uSCslH ztHO*nO97*0yy>IZs7ZAfpwiVvK-N8-gIJFh3UJw`xxAuu?_>~*gtfcxG$48*)2unF z4)g(xtLF5Iu1pQwOmTQ1(jlImBMjxJ{7~v_4WsdC;lTV-lH)sM=Sg|1LX_;~;-(@X zUyJg}>~%#y=Zliro6)CV;?~~&?Erz^*gA14p72Q(1{WpcKtu;S(JF$XY3)h3fBCn) zd2Zq~z+L&;B;3ZQdoXYI{k%T_i_t*-ig0M}*#vf(y)x9g;I1ojgW1l#@eTe+wBLeN z_!)h}Ot9zj!As;mcK{}!CPPX?$MPwosd2X=1&L~02?d$9g@Z5Yiu2Z|P??glrHc8G zk<2c42l(~D`Opmrr`5ZJMq5rDWm*ZqNpzyM_Vz&8pk@brh_fjsTk=PiAVZD|063h1 z1nv7DT1>I=0Un6LOIGDSmRa*cst?DOw5mVk8L}p=Z$p%h`_?*qMOp6OC6!+-0pWKO z{m4qlzJZpXcOyHKPCn?j%r2iPLS0_%eOL1qvYts?hy*_Tz%Hxjd+b}w_=J_ zC}MD~BUq2~1}Gfi z{R!Wrb+VzN7#02m>WXKkI|=^L5_bC#d9#UIW;M(bEQ}hLxqD{8^EWuy>#NmLkYK%P z22swm0k7Q6=d?@0Xs3w=mUAdR;W|jJPW<`D<@Kl)FpR zoVhs8S_4Frn_g^77yjeb2^8XjAxBRaQ^y?SYWv-mOc_0x7K^mKZ;RI02!oHRjYQIQ zyp9Y;lf@hWY{Uf_@Km(vGo!_88px$iLm>P2swNDlNVIdoc1lFW{9CeAiOQdO!*Ckl zfq4a<2d_*oKgelX#CeG8ED`_C0x0q^-J5WW9hcWibt8L^kk#+@ce61LHr@r6h8w;! zip-C>9uWtDtLB#hH(_^Qq}WJV-APlh9lRM8P+OO5C2Jd{F=Q4(1tHFw-qmatxQ|ev zKPW!-H@z09o(pY&`vrnkB? zzHR)V4e~5Q2EU6M56U_`l#@bG;^NywZsAZM>^Y=Z>=PNtTEVnaM@+*E{&U{T%sS@s z2_~fkr1vb&g2Kn}#Qpk*eBWM1&|xzmf}SR7`rD$@-MlNkMk+>I@4=y{q}Q6&Whq~F zDo*zSm~LXD0O*;+OzResxAg zwgO^=B@Q4ZLfqR1pM$$B=y*l{+=zS~3C$;l-e6S3}qHjY@_U4#6H6#}c* zJiseFoie>Uv@D^c(k?46HU5o=c7Xm)n_I$$G4`;u1+6u5=J(2VdR&vqrDc$HTtYme z)1y=Kr%OczXDBpldnY4LeWW_U%zxK?ud4eC-SX%%fGwU-a*f!QLg~{jKX6vgMuB(^ z6BFPTouy-%);ex{8K1hRS+&puBn{XB(LBC^2f-w*`dtzBbm7Oiam z)c$k^0Aqf4*R2Nn>J#224Y;A*s1HV*aqRt6KWr(gmbe(LNYVpw<~5vh20d;9%(A76$7_njo7UoIoA!|2IZ!%9CyOp^53WJ`b2qI@9XBuTS(X)ZE~at=S8;-s~Bl0a&~ z-fVy1VrJ@Az0-_?laKKOd)iq`wa{*PV#ezbP_g*@1~qdEmfm=v{DC2IGdiAGvk? zOYcnoguez}*|5k1ff1#%asCZ^tve*Dmk7L+)=Z(|4kah)a-}^Od3w^8E$ZuR@shyo z!hWMKq93wuOj>_(4-sbx4pK3d92x#Zv*y*L+Ez)*YK-#g0c+tmu=n4_c2-w%tfb-* z+@wvI7!B+9>X+&p2~YApY}EJG+MPHaKuH~0vXNvtbeddx;}gd{_qoj^&jlQFdY?vr z=)>q4jQInD+1T`rr`JcIB%E0hwgJd)nIeL`kD*ItSKy1aVLLGuLMuUi443=hzU?J- zCP8tts-buc0NP*GdmMX0P!+l+4Z8Vitqef9D+sotG4#;l zp4`2w+BF-Gy3utUoQq^m9ad2i&pFGkjl7r`+mG_9%9eY@nP9AEtHh8^T@zCpI|Wu6 zJ(lM}-ULP8`)an{t}j5$R;jSbI3KXhC|J0jWSc{7@JSTzNo*^59kzZe2(>r8^|jN# z^wRCXP0T#BEsE{$SSDhP@>B`MLdeVBLKOW&a>a`|rou6nOMxa+_vLbqJIIh(K$bxJ zx7tx^DV7TTa3Lx?;iT zVj6T8UfIuGQ(|RI<3X8Nnb@x&pMP9k`RHBfa6mo+OlZv(&oB5mP&2R(u=JggL>h)- zJq^1tX{y~Hh5h>$6jgW`ou}qcAn!x_G#AR z(R?${9T=WSN^+`xhM@{{n`7TeBj-%sguQKf`zXf3P~hh%bN_3Uxf7OO9m$>fxbM{1 zacRIn8Haqc zrz3i>vjK}$IXKl!uuG^08?WWGG#^j3BSH*L7FD24t8fd)xeM`f>$~BB6w9+-HS1{w zbQxcpBLMEJ$i3&-;cDY zKZRJFi*Ue9kPNloG3B(3<#K$2>B3xkf8ST47h-a`A60rGFV~@%FStv*FJi(tPPp81 zp1ta*GF=5_>*~%3@bww7Ng%gSSbJ=Igx`*C( zSRF2vQg>BYnJ2vT03<=YfgrECJWNrcw2jc{e3|c!6w~cHHJ2pozr9+iVE&xi0p6>2 z*dVUt+S)u5l2=qyp~o;s8+}J|3rH0DY2#@bl_RcZy?JrReZ!^0qzyIRZH zSt2y4!|A~&UD_oO40gEDtxNI>k>OYvtJznA==7JNz@@_1=-DLtC<`Oe_=}9KCOI=# zGp+awsjGWjY8_rP>o5Jx@ux~!BvRlha(%*+;RBg5q|Wqx4Ydk3K&N9(4%*jQJ7WwH zimU#+U`3O>&pb5@pQzpgDAOxibmI?WMdQA$Mzrn=caVxZQ4)%f;i*f4ojD-)rkJKK z#+@HzcXg1okKPd0ReN%maEK39N15a%loT$lI8waw(qmihzjYm0^V(vM5?8(f`>4oa%SdNBUdTxfo>wk!ul98^_>QSTm{>tj;<>Bmz9-Oqm8Y+tS&;Al z7-`jb5;p$0Omxa2l3bxssE3ygJyQC?)?`4*UrRHiJVVejv?B;;y8Ozg>+W}*365no zQO8W)!HeBD*C3`;w$(wcJw2oQy`o&}^Q!P2`bLQYAVSYgIw&otX+DXJON&yFw1Ts) zfTjVDP=v!U-@ZfaFhg6(&y7}KxdzMjPT@qr3$2LN1A*YJ3O^Gw&48E*1l}B>vIwBT zEiOk-0nb_KDjlAN0k(3YNVp^#KK#+m0XRj{DU(^#*U6&UgFxMHLkpOFy*!t`9wi7N zn5x&Xj`6v$R{m1#Odvnk1$(L&?48I&R17mMOENaJWezK$3iDo3twbm(2 zi<*|q?qsuJEvln|PTP`8uHcQQ@BEg%kv1a>dAIUv1t-6la_I3@f1%)+=MSaM=}>$J zs3?)F7MLn&uAC`8jciwj#Jek59utp@Oj=xhcsV6U>tzIaLDFn``LPg-Sm)uH?W(~q zcTEJ~I`v0g)5hhg(xj*M#sZ}!OJ-pC!V0_2eE!QW)sgNKJzQBDn(F?Y5g&jyiSwqb zWkiPn*Nc4NdS=!WRtvgIb@Y=Sujh7#^XYpenxqr<85WyVuw$E!6<7$+&!1aKf}` zuADc$?L(fH1~xOyODR?I@kNrgPhjJ5lMrxsKF$uG53C`9C04b<#$oKJbKF`6X+mnp zlo4iCd#;_e6@C?_(FQkTS+gcgqin}cKDQ*G7O8H?~VKfpUyO`@%;(PAOy`uLa@PbI1lyjypXu;nY(F>}@h zgJ)KZneZ7-?tJ0~>|B1B00rqHWOKmzB}OI0pt+J$ICpx;{<=s`fy^!kGk-K<3{9x?9$}WH}(B^%na-uHGG~tDb+qI!l zxprZ^H4zn>8|zZfcMOnd8wX;F!Ju;)16zZXhq)jB*v{EG1V(aU?~4_}61Dvg$jTr(Mf^`eP1n{HWivR#U1;AoU# zl^4|3w;fD9@vw6{+%^vx*t4zqTNIGl!Tzbp$b-B0g*H!6pAeo3%cj8?f z=kgxA&;y}*7Z_mR(3rY+_1=^eJAUCszGzBu<-^zsc7{LGDOGuc6w+3qInl1=<+9-q(PjLUYcCm|KYvMX zar9l)QOk#YEibJ+D;;6@Mj>s)YU9Se_DhlWtc-7MQ(ctjURd-PbZ^!S3U7CFRZ1t! z{;IgUMgHX2JU70X)~L8F+Nq}AIWseqM9)LpvhL6xq6 zr}7LRw>c@41cUi4pHHlYZ8PJzl*=CSu&3jP4HvL7Kn1-#xHQmsK|o|`p~u_r2|EKx z@p1|uS9$)Gqq^e$Q@mz&q;aV0z4xV9DG*(BT7AEeG^Jf#0b+#d*X-%?)7OMBa9Fr;OUWSR6{q&_tqE8LWtF~J%Wy?>H6!w&CJc1E?sTuOVWu` zoriYW-@zPK2%5KEk?o8(r3etm*XfPF?1o_7mU3^z7dj3DU;FUD#uH-c7n|ivN7*iu zWbD?&q|=`xyf|e9J-!FuKQt_Ic%5NY@v9MU z&QE#kY_1d!xl05_Pv(U$w@}7V%-Fq?{7G9HrXDi8sSbMCy)Kq@G4r5^>bu5gJIT46 zho}{)k5QW?E!Y7WF}3Cj+MrQClLjg$%L_wp2~=avJ^b!U!J=|3^G$0lSK6o>^rlpn zeMzSDY$`#ZT5R7oC{vl`*M8em<5AJLOws^ra($mS2cPl?K1<3z>Z$Lc811(ve0N$T zW7hb3m~ZG?jv+aXfd{mLnU^8B(PX(9t)!KqO_=X06A(EO3vkgs4dS4kV_RfNDA8f6 zSYfb9@N}3ane3yRrW0Ry)v(7;mKNBkG)XZY`%nTZzpPzjKY`U!7RDS5%bPSYYak6lB$Q~6q zkM*{TJ!I(g`#pUG9?`nOYQC-o0AVQ)SuGkQl_zCM)p5}iukJl$?BLRzit_UNd{H`9 zIAahmvt7P{-IwXK!c2-UI`h?+Jvr6rE24>YPa)#Cs%idh?n_~O-4N#|M=DjN2N<2f zdBr0_E{cdHB_0ZkbR*UU^?0E!j%LLZv1zy)OfHdD#$WtB=6@Ihv|4xKauCn>^EgVI zn-5GZhs!R6jYkM-Mobvg?tPgJeB?a}0=$ezhO+tBfrI+9q^N~COgs-9Rno9z%WqSK z?i9sVEN{Y>`sfIV8e!I66zD&!UI2=YrSdQ+0PyyIxDCWT&Pp;3HRpx5F}P(F6W6UISeF5+VD) zbb`}_fd_t@q>*x>J3nx*O^VF-_gbU-;}4QF_uHo(s&bkzl~AVDV(zq^Bx7cq z(25dwFb>GW9z_L?MNOps$L3N(owRB-(&^5|wAJTk%&Cu)H>=ND9&Of)NgtEU&JkzQ zhn<^KcrkWK%juF0yT8W}%dAczb>Dmx-Fzp=s@hn+(dn%(KNBO#?kPlF^Mwwl85dY~ zd$EdBtlvB+3t?wxIueAe9S@1$->rJ5vEkq z?DwzQCE0t(V~<))jgAQp5V5ATPUZ6sKIs!7xQ1GAHgytKHWa1vL_M{c`-gSGlyMv<{^}Dqpt>*Z2KNZrNnS7b{~vN=eLp8kaX# zerPl(Ek4sD@ff`GMicEeFGU?+7nJlq8?!$wxza<7C=NRe5t5kbQejS$Vb@Ucb;Yfc zH{^*iz8E*dBy!}^Zw4g210h|PCf8~;$EN~G^trDIYlv7{fVf0vN6iz)9Sp3DT8;4M zhnKmpNen;rw_QtH*>E^}NoZkRYCM?wnhLxRBDWlZ7=B1Ie=pR=5Os`aq}->Q%@CV3iradL-*aPzw1_cy$(T?e5V|fMp(vE^V(In4N)?7KDnil*O0gQ`ir^}nm_|hAJSLLNESjF7*o;z|l1dOE zO_`U%cN43;l8L7qKlVP~pb`vKW6NEr&k5C!e;`=kw|+_dcZoRku!^cSS!!fLsfVzS z7V1c-*D)7NLdm$i>#y|frErOHsA>(wcoFE9-X0QJ4flh)V$561*)M+PSGGe8Awjor z_F-ds!_h4WcCr_hNa!JlKba=c;}hG|cpnwXgc=KMQOsO;cBRU~pBQc_p3R-qFJRd< zkt?TR|3l(Gu}z;NZgWxj`^p@KoChI|G1WGHwixx9zk1`d{vB7OGNc1MjkJroOrw^@ z=A!lprL}VtMjozn zZo?xagTI69exoY$m+L`Mk&w|LY}0+{vSGgO=nm8)>Fg3xjD0yVEn-nTs&nAbx z70+IgiRfmeekx*=v&>9pWu(uSr2O@I}8N6pQbNNbEyT5vy0X^P2EPl`4l1p(+k&B_Y2{r6v9vE^C7F~`+M{QG zsl5Y;UXrrhW4E4greuGyk4}8gTg~9gNjs6JF0|c?8@hLtzH_>&0mSP&so%AsLQGH+ zCV1_>9^Yy95nn!n$Pk)~#vcL(!!uNW15@vn4pl>CCY*WBk1{j7K;3$I%3Yf@BBBmN z&m{&wfWSESl{4|8ja0}3NWMQc-2(=OTlZuLWNLZ3Ag+&@#5_SE*~dgrd4;=sgnHX|cC$57VjB zxSdc9^w+Qq0;X_6G-lGImae#NIm-^^#0_;0zD9)({$B2P%najJ_LTXsbk940K!%o0 zVmXvjCK_?3Wj<_m;Q+!Sd0dZU97Sz4cBkSCE4tfQZzxt*+4%>!kg-}w?Ad!MgJiS$Hx}o-FBsB4QXd-llCHtt^N31<~dKj z8M#mf?qs^y+n#335rMB$NyUo~u&0h|Hw#w1EnhX)$f6MnqkKYQ7w}N2a{5E z8kOfyG`gL8k?drrUNA{i=)0LRIIgS5>B*B+6d{$&^VZl`v1_hwDz~@4^ZB)MDz zy|}azyjh6Iv#-J2?2h)R=Y$xXY>>Hu6uR^%VBYeb!>y|h_dGA->9cr8hD2;k!e5vv zkm->7Q=6nz%o5$M&?*GBrh=Fw?s8p1UDiE9wJxon=HC{Ny5W3ZFelXoY7LjEh{jh} z75sSl_?Y{Cag+fMgF8Sx9eV~(Cs<#op)GMpzZ|$gPbgu`t;vD^dM1zhS$8w*w+Tjg zR7C&H;Pai2hpU#~$S!vx5?7I~kcZrD(eDoCi-`T3^gTb>yK|G6&Ij0!(zztPtrwi4z>+i)Cse~)4l2K;|W{cjraFw>GJSUCcLSW*76QRcy2f)L(q zmuEy=K}U*erN1GvafMUBpyLvu*s#0q zUut?dO(57H7wmx4Y2o~ZkN=SJ?yh8rFyQ?EFn*)tgq0yq` zpT@2rp7cdZ`!1K@)gxj+At$#k@+O({iOO#K+V;%9ynccWL>oLA<0s%7r9N|2F%>@7aat#pg=>NG}bc;J~REhv^PWdkrD75!F`*z7w0j=BY z_=jiwd#V9@3Pj*BV&~(Tjh$ivRqg*ay>Wax7x*bhO4BS^3z)C?3V$CJ|NdQnFMZ_- zU_=!5l;D5sQvRNj|DO2b!9(ILF8eo6#qTL%&%=z+cl-0RdBGVV{tZW4TjuFQGF{9ue#16R z&lgBzi-xhMpkKk24BiNizI2;EE?}m=2qWA7gD?V~UlO#9vek9)2HO0f*6 zy8xwmfXnS%rjRHLC-P<&BzJJC*`4MEkdp$-%=@EV8Bm>tgG39fkx0}0WFUSFhlHtR02S^RI(sig zjq_7foe5=YWY9<lP+WIN`L2*0Wpn6U(*958> zx-`?-Zd`8$EtR+Z04>s81A#GqKu5ri=_5a(W?zW>+iHId&?Vk%0x`f{aZR9{IDEU| zaB{ZBExHa;tl=^#{BF4b5Aqq1B-=l9bd2uzgeOQz)XKICmB#d0qNz{HBRUm}5i=JB z%=SJf(meq!eQ#9)t&KS+VXwr=?7Owr0KSkLhGPd zbW2d{Q>DeNH9vQlD)DC)AOyNqlRP=!B|RRf+?hT=ob0mR*PK`fBw-62bE|Tq3*?c{ z?His9AZ4EdUjTQhXJ;EVFY`(O@##)UsDh|o88WR_5ghByq$51 zyZtI!Unh|`YVICTg-8ZGw_Du7(4T>YDF$o5&YpFuXi8Omwsbe$?0f-Ct{B;40fynill&rM?gBv~8rrpZR^&Yq6;%hv>c&F$dt@8Mr{P zHnKpTdLq>YpfPAqgI~6OazqS*GFXC|;<*|C7tv2=W2dEi3nxe9qW#b{oSk;Pa z52gf~ATclwI$+F`kq#4ALeLr%Wa$8$awauZl>>fy)@zC!kPhF>7ie>DbM8(F+`W+s z`r^m5u0m1r#hEEg zM{ZYaVDVfHL8;<+c$SlY{`14>`cj~3l}wvHB{uXBQRNnc=*WPHLt5f+5Drp}`V_=; zy=qp|>49c<0uagD?R|^QIIw=Kn$9f=`pQP*#-anavVCJgMd2gEVJH7(gx!Fjs%2Rq0hK3&|p6V)v8nG50e9b8|J)4qWH!oK;C zk`v%aY)EvZ32Z2hf>yAd-xpcjH&_0~v}sh_GKkl=8_GG)-0L!AD?NW>7FrCx(=^`Y zs)ZOP9+gim?5fuaKDysND)a^NRpfB=(0LfxmV~W5eln~tz0$L2{5C!8dUD*cs1la} zX{;QE20CjF=zmlBqe}q|@FfO*PPcuE9s2ts55!WwN0r^w2{FICo0~Y+)WE06wEq@4 zpFy`Y;r;%TCfCudQ-`3)zfh)HLRxiyCrD(3xAEk1iKf>e7Z08nP_?cDE$^xV?3XVA z&srJuA;X?(o6(!O38Ai!4kMKB9Mj5au@J>xAZBm(V> z`hG`vZ9Ms*7eo5F^%dS-AMIObK#BD3=i4j#cNLzXL0GosxTwg9NYo>J! z_bf}Pl=Fdv+TZzn-0z(_rMER=OIhQb%BK>VAju^M`Ee#kBC_s1h^M4q{a zlQh^&c&XjlgH++OGWsJUEnZ0HD2eH$9*Oo#38?tplQ8s#@jJ9J?(zG#4?0R-3t5QN zetj-VC19uL5J4`z)Zl*m*6+P!gjk@n){{8MfiN=y8adrz9_PLBtcKz@bFgQfysHz* zGZADuxkBUE=Kb5^^<4oO9vfOUe^gdZ$YIuqB_X1WvUsq+fVeyoSW9|Zn(X- zXYBMG!F~IoH#{G9r<+z|Sdog>XH3&74V{aN|1LY4(TE)>GAzgVw)> zz0?tGp<22bT#~~P)p;Z?79^NZChuEoC6OLMNE|wnr(W|!S4;{G8;jUc1kVo$(;{j^ zly0OiW4GP*QBqUQ&_ZiH5pog&SbCEIzo2hVU zNk7KO+9M(6JSVnu4G-~PPPCHDa{>K$_qQnrjUJ4uMtN%*UCb5~q(RkkZX4>rea(g0 zRhl*vT_|z!r#3Kgxjgp~+bt$`Q&QzA?)_24#s>93$xNAB@7$GGt*1Q?H~YA?k5!SW zhK^g*ZSRC{?)tmmhWvB4!zNK@Cl_NWX?tiP-UU}7JxQc<*eD!NGjBMmj+LX%(3;Dr zxZLd?&{Ia0)asM!Tkv|1R=0ihv}B;dulSH_+%r7(;K7Lf$eq za=G|3h6Bk~me9Sbm_yy}Mz1ui><cyyW$UXzBs144(~JDlTF=4XGjo{z1Z1Raz| zk0vCv4h3pNg=SXf*==@T$#t6lo9?76Y-0fj)hL==a2$G?Sr}BSGfOuoe;-h z*WftdFDRNCk4ThGf5G~KO4u48$gX6$q?Iw)jNfdXWJ7VX)5HL`1M3RwC)Ojn*b-7A zx5mf6%?`7Tatw{JGzlcNl;<%UkjY4wn>2$m?D(Q)Sg80&73OpMbtVc{1_3ex5hes> z!Az6w8V$-4v9dznpvzDkzBUR-(suL|vO;$DB)NzMmLt7%)E%FMg2zb-QXs?gmjvj= zd`>s8nly$YaRpPg?ELF_yF(3Au@_d~c7_|yHmGN4l~@>ZZ&Fw_>PNh0t(jJ2&ox85 zru3vEP{3O4Ry4T4D_S-aN8yZtUA;3eaG6PzXE4}^n;5rfx1}7-l@&$@XF!kSF9kjR z9vLfBhU`8Yf!%I&D7WZh#6uDHkSV0ByXyP2jt`KdtqDkeu3g>IikxR!46#3vB@F;F z!#2>UP45w6yaRloJ`ZBGB+uz{7iAu$KnMSU9&6f9&mpncuSsWE(Bsv|j!azb@?zcZU>KJB1xi2=_VcT)8)Fsu#kT>VTU0YepEz7uLT?i%D zUe-@cp8Xc*;!ZHdXewhHgmtUCUYTz>5j_UL1&WxYC@O@MP`&wBxuNaiPVZu0NxCpA zj}v_toYUFGVL8X2iRJ8}%M zV~)}j!5xi$RCR&SZ+Y^Vz9t5Xfo8J?TkNpUL!vLM#-TIiMCcN+an z1F+_!60w3)%|f_iB+-_InrP$fpojKkn-5ge-(6UvOB9XC(5=lVpLQP2G-baB;UK6j zL+8&3jR_U*kE7G|FCHjz6?Vo-9GX15&NN!J(R2BW-tJ0A@H%nl+laHx5?L*%C6#rOnADPm&=Jbz zeZ>B{BOSOspLkKWGSf9)o+Ha@iO?WHBmEQVG+-~jzwJ=6Yo17QEA6vBy+b-4b5g{m zeaJqha|*`!RPh4~QmmKbY7UX#K|?r#;@v~r`{MKpr^PChjtFa0XOssj?Zg=-3IKqj zK~7Py%#(E%T9$QpRAdG)(c7H?qcRec;V@R z8H$G2Pp)Zcv>Qjfq98f&P!>H z7A$>!kW$(ecKr)|#&VEc1yIO6_oSsj?t02htTIxFO=v&Tur6q zy&JB~H$1-R>C(7g6+-L8ud=;YKWZr^z!hAp&3oarkz@6 z$swjiBYt*;;fCnQJ9zQq+6l3+xv9emsi@HLvE~;O6~M_EJ*~6(Ey_@fhbCgXRZKMsCnmh60+GgE=T_F(NE z;7Ks&0iFb)no@^RR)%exex-8Z{GA0Ls>EeENxc{9Ec=SQ^EQ#OC9MHnL)%>NGu2KG z6k0sdu#+mVQrDg;2PHW5E^u>#fGYhN0)k6atc{yK5Sj~I5zwPB_1O1(?LSmRkLbsZ zk9N$Eyf?0F9rz;4cB#4HwrP&h-*(}eyTF!;eQ}qdN**_TX@OZpW$mloJH*;$bC=7KwUT;dbc4-fn z3a9MJNu)$1#lvGo;xBAXoxNuuIi7I*(1)M7`>&pR1aV+eQj8aga_sFxN|xaB7T3^4 z!vMnCT}idP8C+{kGzLz6X;l@;+7a7IBNNWRG4wT@GWx(GxZH-k#FJ*(NpF_TojcoL zene``Ic4gq&|!_+_4pPKDmW*oZR>pAGPD!dW(=9JxNMlD))vbAB|eCr?x!~+@1Nd` zvU@+g8CP2V;LT8;Il?FFKkUz1bKRlSCppYzu%j)2m^lBODYcm-H41Jg?EN(_*mX;u zCzMlRf|NV;q4y3N&cg4VCgfgl1=ZTEOY8wI9EhJykb5Y71$WG)c&i?VtkF>yYWC8m z<#^hFl~lWd(0TKbS%Y|*LK9gYDdi+C<*uHuEc2pKI+xgZON%4yxdki_{MTroKzzAs zcteOi3u8%R&!kMlt)l1izW!vge-&Pth}axavg8+s)77|FKUIEoqv{s9Moo}X{=pfC zt4@_wJ}2xgNybfEKXI9)&T-_KGhdisDa1Rz#`FH*D}R&@l6?R0WrkOj>y1PHKpiNH z_`e#bxVFvq<4?AqCxG;s&(!h|L?IKQ^kOp_FMgYwMh+r<%%@foDa*di1 z&a5PR@L#@-G>od|(3-GrhgLx!F+f1T_G?5HDSXrN9Ll>E(q zpl3X#F&9RA+ueH7j?PA&SWOR}jxeq4)Um~^rZro0X)F0~ZKkR#``Go3jCF0u4K#++ z;Tzy#RM|R~ro)f^_G?W4^lP|V3UL$NH4%fa%2!6OFcdSl{d8=gzvjtccULMGI8v{f zx%u;!P|0MZ!O390tBb#0}`^1Tj|UppV|lNB$b%Pp=G_E941b{bHt&)!6{*6ZO% zJBx>ZYXbsXxIWES4q9h;xXq%E`!>*zJe)ReAPBV0gSF7jqCvU#3~*K$#K=;6o{!5` zS!oZMINCv0#qPPLF=&E&e`gV&W+lieNc)!|5NNi?DEm3GTnENiQ+;lG1_dD}UlB{s z$gdWAww$^Y-03^oAING#0=A~SToK`~cuBPM>J1jm-nMsMu+%en7VZHs55@cmw^i20N54V z6ADaBq`_zwqARjA9?CTuxUadFJd)fWW1l{=LRtKzJd7e@&r#)6>E`VfP+C1MZEujI zmZZv|A5XN`!DOLo1SRa3Vua^Jg)m^e=OpEq#M%q)PBQAE)8eA#4`DY$6WJuA;GXJ+ zFH>z6V(}uFib&u68*B8+W412@w_$)d@vKfAo~=*9`3A_pp6Q%!?CtyAVuIf#)MWP{ z*4qpxA}d;=b6-H6IEh5o>#k~0G43gLG^ z;i&~;IVelX96G8zJsFc4dVwoAGl*mP)66z}_gjfh{qmT6>s7gbI5XO@bG+aeP>4ctQ?SvgLc;^4qmwq|(^&&q`F5H9?JniGsGZN=cV;arV3CG%!5sMT<@ zRAdxXO-Fl%3BR(q>LJxB?``tlmqDobQrffj8}e1xaCe{>mt%ddOG29C-rk6k%HL8M zK^8*sbN2&2N=>~|TA;0-xm-X+F%K^|sxd=6QUn!lTR&r!QZd83KK~bS?-|upx3=Lb zqJoGZpn!Bix>Th@C<;oI4oVS_UPB4JN$(=Pi}X(DJ@iQLy@w`EIwU~gEcX81@7rg; z`i}GOjFBG<2w5|8%{AvUpZVO^?WC$N=}QSXTyG?#oQlsY!(R&Nu_F00C0;{dBDJJx9a*gHR$u%ApXZ){RgOX#}+nEGLB{1~UB;(Mq z@pxQ@@SRfG{|YvKNJu-hzVyIxnE3edVUM7s>Nenqo|h4H?DX(DN4T>5f($YzFQ`GK zTVtF8Y;`wWkbs`(`RXeTs@C%V%Q>EDU$e9*h9yLv=%oH^R$8`m-aZsOIldtX=ZhZW>a_%F0PN~7h0pK( z>*ysaz_25`;-0mbl^aP^^08UqgUM@rm(bDXK?}DKAmm{Aw~)gGvs?1QOsEG+GOilu zIpWfK_v4Wbp-W~z5Sg}HJNP$0MiAiL6uLi3I>Ymqh~rGGWC1`Bg#iA)&P*SKYWDG; zVczzi*a67b@gfmty0St)z)AXEBdH{xCxV93m5iOL^+SnFPT$F`4+H0%WBcH(aQN&Q;2ZV2j!K#-joXmbUPk_@HbF&bc+!Wxb&Dhix#mg7UBlaWPmfsv_ zqRl`1g4CIC=@XDGOX{3mqMX;bKJ+n|o(!DeM{zmcTz|uu;HJO)h8LJ58qT!2h~oPR z{BKWaiY&aY4IABAcKPc}+$j78xRf*1&{oX#-7|zXO==WE!>8L-=}@nof@T}P$2sTI zk$wv275;#VP#nQ!X}`^9!L@jUVc#R)MY;;A^z+WXGxv3p>O-iI-F4l>Y=PZC4em{= ze;YG!V^jJwEMDtGd1}Y3MJ|!cBcT(rH341(;DH`j8$0d)7Il#Pia18dr9zU=91%eb z*%7jQ;>mN&>0iibxGkk-wC5*2 z_`-8vPXG~R4Va5E=~z#5w0hF$*#4+f#B+MQ4dg5>Wgh8J2I@IHeb)AZ_VLLz>uo%< zbhY=|p5ea0w66Wrm=U>6fXH*cP{fauC@FXU{6d~@$@gfl*WvR+JO>zn`Mw5+U7s;3 z&$c#bRcN;!UmIEo?iwsq*`o=LM%EDW|Wne50^|Lfs=( ze~Ibq>VBRUHc9%}5w4v6r2s0O!qV1z8!28x!E*h|t7s`yBe2U*j2$z0`z;vIZajK4 z(PxSw8(ssabfsh0w|a#&wUp)F{!psISG3gzKQ&X`0MT)6H*f0bUrNiv6)KF$q2K(} z#Wtg5@90&b71t|{P%<|C`%z;w@yBM@{+2SVx<+P7bs2!)sBYi#CH6)DZC-_s+G#lC+3;*f%rrFsk>4u)~vF9lCKUr9)DoPb({et5rnWe z-CraGr1%_PaZh(Jh6T(3M@jS)2f*>QMz9b0e4Z1D@?HC;!34r6qVX7Z{pBTL>Df<0 zKZ8KYWIFg44>;OwBo#PXblbnfx4h%W1Iw_ENZ@If`@#F$fOLa?n+_su8B--FT@Nr- zK0v#iNde`CvXWC4V5d&!02$m6gA*K+rceQ0vt99&fQwcrcVkxHUaEp7*1q&%l+Y%3 zgo3;zaO*!;7|E1mt3>ECVx)vq02w{@!q&NW@e2!wd`!Rk7(IE>SzJ1a` z0?2@M7p*Pqg@YgkH~ncHpWGQ!(i5cLS=a+H0d^YpFKgrnt;CTz+z3Em`E%do z*Iz8uW=8B449J{`V!O@o7+N)k*O6l+$xR>~Fjub_6KlzJNWDp#)mKg*m362iM)HVU z&*t6J`}#5a^Fg!{z75O#ai_*Y<4KnS*v~Z_8ayVB&-;gV_vdsm&;&MI10L9QSB13A zxFIxhz2tohJJk!R;?lmzc5K>s+IuJFSg)`+uv7q)vzfOs4YZz*zV#K!XG}brb{cao zp;^*jRP!6gY3>jiOta|{Q!?>k0vXoGS6{D*ZXYMnfVeiLJIEw-DCycmfx&vLu7R%v z;9&7Mt2`~`ZVv^G({4-Ny?H)zQ>>xVL*VIVw_-J=m`O$$k91OAz(Hn8d*C(iB^W4n z5DdzU#*1`{_W$Cz*K(JEhLGL*{=SI*&|%84Jasr;K)=MO%q#OyOu~RtS!);3xK8R zC&M1%0eA*pzi*l+TuV2TYo@TijWVQekfbQdj+1omCWSZubP73Uy*oya)$z)Nq4*|C zagBaU2&N<%ReNRS$VEbc_W}lmqffu26=pmd-GMgYYT;Q$~yj7*-=T)r@x$^1cKg=K%8r?3c~>!mkkV@4mca z4O}6Q;4^b7ds;qdjhWESJpxN2f!? z14hJI0Pc4y+7gdg`^3HPk^$uDrW}|D_+3mO%99Jq`}MR3<&y1 zxccQW4&_qR1fj_1Vb8l9K>jSZoPrtb%MF5i%C5ZACTYrq|c z7sq=9j2b`KOc}-rTbt#DJ-o@*>I8Ezn2p~1bieIG(mOJewsiblPE1DeC)9Kg&sKmU z1PknFrWUWO=ar*MrnV1+)F0CEQO5<7ZpTU=;2yW#%oG7a%B8pPO61TmL`zo;9qOUO9_IFgpy2b#61 zUnG5*(hZ&xI~gNB<=quu+%uhpM2UmC}o(|GM0OXyVrthb-J1Wn~FrM$5s{1RyZ%XF@S6-7XM(QH_X)lo)G`; zuT(6QcJG>Z`!em3-}1G3;>Dv_W4aSu@8pUo`mcxE!C|&5wgQWzpm|T{&K1<=wYOX- zOy*Use-Ndtc!_D2)JW#{x-bMRNG zzzG%m!m&_KH+FUjq}BC(6enU(@Uvx<{f4W|m?ld8So3eS>F)q2X{=N$e&4bnf-g2? z*^PlhXC)=ng&tiqDGTu6Tbpfrg@G{naMU^D-6iZs+u`=|J#5(rH}zp$>Ds)K4IF39Mppx}C+4Ak%C*#l)H zBk{4r`hRWP{#Jb1b`1r;pTEWr#>9s|`oT~<&06}065u=H&4VnrT1@EwIzs=eRP;YS zlp_zQePpOyW3B$fhg|Q?@B0EVvnWSm&_BFKmKn$gzAfVK`A_2O|J|E^U-A|W(0Ke% zqo@D2Hp)OGVxLGk!cT5OYLO-@G##u!l*m?f7J^3qt@vnoE ze}A=j8;mIG=!xS`qeW?Z%uwWiU-++90%faTMN^DERC`#++iLk9K<~cfd1)l}n@{|o z5AUVKFi6LZ2ze*snm@+C4}-Or0K@X-FL~bs&DQTG zI0jY8x$A%SaRB5zYoXZ!WPS(y%Fee;ej5Gxt=~Ril*{{Qmwv!7h)unh7tF3)?a>=- z4gUS@^p{%%%*#nmXnJ)q?1Ds7-sn_&Je`!SdnfYniQ7s%x$j( z?J*HDiIU#pCkGb4zsSFS`!N>~PEaw=4;fo2jQ>n+nFR9+N=(wFkH&)3F6-Z zbMNU$+D{UJ(=`9IKYw(^vu~7%G=Mw8owkTSH1vAla-3rI8l@*wM%tFVL7mSQfCn`G zcq2t!F$|wqN?Ti|_4sDS_lcu?RlGa?v%d4cs`0%O=st!XBxLSFe?QRu<(uCM1%yxD zdh-#14ah;Lap6a``4VTx8T3r(~wOrcLbePMYGD*0=a{r9ywX$He$r@|x! z=yT%Ma>zAm?=nS4;86Gcd5i#$>d*M*Osv&gTckhZd(qe%&Rwp$zZyKF{-K*gDZI7X zShtcg*tTWO>L~ybAl52H+F&W+JW%Of{otHr?au}f-_$Y8;^(5aR2ZrJHbKD#_6oTb z0bzxh?InM@Va*y3sDY-%BS)F6-P>T}|3;N=#L|1k(2B)0$%#LZpA{{f}x78BODBn@AIlJ(^UfO`6=Z zLF=nj?=>Z9qVbCMP7Q?$gL@Cd22N$C#@a+^4*#zL`@h>X$l`teLY98NU3`YsQ=aVZ z5j&{UtBl-|CyFk*+NbXXVw(RGQl@)U(d!B97?`5f741|0bdNb6WBwZ9kT}P+EHAc5 z`I(^#9QEn4yI77v{hQc6;-W<4|Jlf<`3LrFj-HxxbVLIy z?%n%+l{8&HdBT$3&5zdKtg%{*8G)Z#9ektLhmDq6HSBkSWO21?ob6RtBL6i0ZerU@ z-q<0)J!~2^Nldqf#I{^1b%@l)KG_oJ%h5ige_k_UaG!Oq278qs>7BCT#a(C2Omb;` zCpl;8&$IK-q~>Ud^S%oGlH9zbheYqk!iq&OVYln*JkDqxWtfJ7K`;F6Hngp5_8op}Mj`8l zUu_8NgLUI^nzArAv7Gx~mRJ8hOn`?v^!CjcG8H{}t#lN2W;|efdSle#KF9&36Ue2mixus>> zmz&LKJ?gH{d-ddw=aOP-EPzt=>`zM+I|0po`t*0n@Bhv+ZPZ?BK}P&Byz?uyv7~2f z91jG|54I%%j|pLHRP9QdV9}&&woydmA>bHrWt1Zr|5s_+-vg}XQmk*P-!F= z7fEk2|2%5qu`x`dO=6Vc;gc&g@$1t@L`4k%?p{N8+}pr?8}zp4pmsF2kt|)vYs`p8 zjO@U!3*=~b?^OPHaF**4`IY(H_CIxm51p4}kQ#rCG2p1Fma233ql_def+9_ZSKK+r z7~UetCYqBlA6FdZ<|-^Qk9hb~1SDU`P>knkVKBF&?CE%U&9Wa?xkzKl7BTQT`+A>5^{#o#wE06JhM4gV7*Xg*mLzLq0myc8S{^? zZ`c^4ba`@5A0Ky*FQ~OUt_vP(_>zShC z65MiVcsYsZ7-PF6YV_Lt4fzOv8ssX=I&$44#ax!GC#%PB52(9V1n8bgXz6`}&ddCQ%;!|EI#+#S z+ZQTKM}ma`vQbwwlX8EOmoJ3j&sV)km@9Wfuq$A^=)-A{6s7aZ6U>o*aH>;FSUqcP zdR&6bvr3uR{6m3$6X|IU<{^4n=i@=O324N0y=H8p1nk}_X6!Sra^WHcOE44iwV!QA zgbb?u6V()_oFu({tVLR22|u1Vqb{#AfHPx_^$Cpnhw=CvgQzQA(8I7J_z_H$I(UHy z&g8F5ZT#J)znv$$hTLk7DOBc-8tExdnsRbo zm&dY6JUgu0tn1-+|9o9pFVC7oKxT*LJE*il$lC@l6SZk7*#kL4=;t*>sp3*LZvj?^ zl<@(c>_PgXRFW7ZL%4ctrCm0GLr*C6P5uFf9yY6pt>C1w)G_w+n$Vp#5A6}j3Fc?< zJ-VOywi*)BQ#P4b%iRyx`tWFVh277NqUhz5`94sv$^FxlOX0mUQ_b%7Gd(~gbZ+HP zulk{;V3(#9_U7ob%8caCwU8L)w$YbtQSAaz(r5m4^d$)h+ZqSs`b}`26S#gexS+1P z5OVBTL0%xvZ(ZtZc2h~1?eMHkqajwbSP8Ay6~7(VC|y}jH1l?*%5tXE<#>~YCU*Cw z>2PL-yA}UFU&5beEHCWLhs|sU6q9b}<$$niZ)EVXdYeCj*Uu(+hUiCIov#h{0`oHs zI7Y?hrc@1`-;S9<9;C9vp_2Id))_HHPIOB9ESEfyUr*+wNJ;S|G-X<|jQtnh)=qJo zNQVlUjCO!Kn9SN}GCQf~ZnUhX@wh~yTSrS=T2g?&|F6#4RVf2Ul-+jFN?JSMxShI# zzDV2hbx#F27^!y5LQr41Rr>|}ktn!b%x=@rB1v8S%cT4c>lE>6z;UVW&g`^X+f8lDvU|+LTM;(XO*y z?UsQiCD*stx+d#aLx3lZe~X}hY9LeTGruJxzVaL6uiw~)_$;WZNUDrsTr&pegKFsX z#xqsvTyg~FC{Qn!

c@H%R0kW-M<`*&BjzUQu$G@z!{eqc}P20lYvcboJffSR`{gwY*{y2H$Jr9ga9^ z%GjfvB55sp5#}kjrT1Bwb|)PP9u@{)(KeRG+Gysn0j2#`9_{L_s1l2iQHhy70ZSb{ z&rJu$i|i*ArkU1}M8FqXjwi@ohI9rQ3`trt8+eY5m3-Z)GfJS3=5E?HA)Hpcp-yJW z=*G?T;5PUqK%D6qAJ(_4KV|pfXuAI$iE!8&fBs7FwHEdaK?9So42f|bt@@;b>&}a~ z=I44T?!CboHD+;x6mexf=7(6gr_;ZM+UmHYzwazC1Jj{D8<3GN=PjC zoh_olgwbNY0|Pp#W}Kg2)o342vX`*IYC_S7%h5VVT08?9 zA?(a02j+^%&KLE2^n0Bw^xBi~02MsnGh|1=gc;nhCt}A1N(Zdw>K8V>lX1rlna(Ry zk9O3BVqT*^sgEgKI`rs7=sI4W@cuSh^8Svgubr2xs|-*cdGkL{iPr}Jg zhy3vvEfFG>BlNOvUXQqsaL`N;^L75^(tdU^RtE)pBAl7{jS_RMs?LvOQJv()I~VF` z!4K2Ah8^kZ^-U*=1*a$ay_@BgIe&<4L?3&J+TdQw|WILLhxsNtO@|VjhBh{+XOZz>lxbxEXqo<0^V`cO4CLot&C%YCDRd`#P z6Pn=i?XhC8>h4=6#j5T_Em2TNyy_b9nePN=S}A)&EX7Q`M)vVx`ZK}cNtZ87$s@7s zm@}0l$(C+N=aNq$J&(^TklX6|p6|Qs95r`BTbdg7hdc8}#;rsg5YrQX2)Do0W0u+H z%lC>uQLE=HKP#J-uw+Z+c3;U+I+>;JX%{%x*nAR~vAJpB8b6GHqw~$(#@e{HaY`hT zAQP%?(;en@B?lLq0RH&4tXx&TCsqU$!Kde@Jt74$3)xDSbCPpKm1;|U^R;YH6AGTz zdunjfV0>NbSpK6_bGfwMau)uJ-S)rUscuXr8hhZm2omtJCt@tui{nwpYpq#z?C#12gZVg z==e{(S~F9{9#wV^4h)1(6v0O_H|!UOoy&+b(`nYMmyq=P+XlTU~Mx z{szeDKMjz0jt_pysUj0>Mx+ZuGc_<1?X%SFNk42BEZNfF}#%_suf0%HospN@u zYi6n{k8n6Pz26bxu9rwR1?&UlfvNMP;%il+!h6s9Dz4><5W-UjW-Foat>4e|rS*hg zbjDr;FoUxI%|u^$*m=k}5VL$$oTLn%9YuYjntpSYVVyj@ZII-VXI_lvN*i^ymtZ6g zl*f*Ly}Z>jEWce-%!iSM%1=2l%Ltn`1Qw-ki>QbuB2#$n+2lZPoP=El?BWx<=Txrm zMs6kq6|wp1SwUmI_I8fT5cpF`S$^8ZVYfVdfNTN%`zCUzVSU=rmFt6xu@p~h zGW#8>CdvFm{IZ7i&L<>lI`bR~YKpCbb5-O#pD9zaAnWWq)C2A0F>x`C+w@)0V=_hu zeCgKDTPBN8Wl&3GpNT)Zaet!h3U_D3*F-osIHwL`W6oMNLLb^9;vPL1M};!i>QuC- zVq&4(+JqZ!S$wx~t77R=%bHfr2i6}|qsjubFy7nAhu}T^^lgFHyyrH~N;a42U;A5ucPRIOr%FMCm&9HGO}(+BV7>gUG2f0;4ktrZzgx=YcyFbzC_l}~Txf@NqOBLq6j-2p0#4u= zctJJI$s@B?K~xPtev!668`}vMDw#0n{{4!TjeR7o8w3yAde1PEkmqAKHOSK=|DMva zwH?x7z$94dSEV^oW=N`IJXMfV^AXB1Mo$5cERQymY=Bl+V#Fp=R)*7_*v+b7CXlZHE4##wzY!#i9iv(00M z4_gIFUP`_P!Bt(ghSi}6^RqnmhH;JU<(CG`Y&*kBK-3BrKFXUW?i_Te*C^Z2U`kJp zqGrq)!bA=R%8*KyjrEP)%G1b@nyRD+#exfMih5x#`?WqDCyuip;Q%Xn{4VP&Qzymen3;*19WG1<#-#)f5O1< zGWCz)hcUKqqN3B>lwwj?zcW06wtXuISS22MP8(-&K%7zx)5hjm779`J--R_ZAkh<~ zUfu^y(i~0S9z%-+(i&=g(fe+Qx~9xM1B~)BeqJ3Q!{&EVXr5wbtZliCH=_weZD)Zk zBJ|^{14@u>10^9;7o^5<{o(U`xS<8rgn{c=$aI$o>qd{mXwbt>b3|R*+HkuIdE4*T zkut3IGo@H<-s!A|V+qQN3zKZc>qY7(!^^|lhHYF`%G!n|JsO4ZgdOQJ>vi@Iv7_}S zEUs0YO#L>`VQhjiP}p=5b5A?FL6QKB-uGa%dATqWFA`Hle7&-}FslxcDe2Y$&VDp&JAU6p*>5A5xp&{<3~LMr zs<6KYHtbJ8eewqRvgJX38kODq$zzF1$+Y5iUu>hMQ zC*aXQbV<3VmrMhhOg-JmE#}K3>;xiGpgC+{}Tw5rw!{;8>0lpZ7>+of(!Wq4Nt=>ZL1* znPyNGTL#-TfHIL322S$8?H*7-nwQPrC!b`)>^;`W)27b%kx`y?9>AJDn zunKcw=GqhJiz|BmV6iApik{zhW!-DhMEP8>@5Ef9)EKzC7o?@`RwN*!Z7!b=#9KYl zp$|6$4c#2gXR>_{_U_m}F`3tJ`F12~BI3LQglZvhRkvffLq~|8Ri*iQ4aBKDg){n5 z+)hJm%JfUh^MuI(o|A(1)m-xvQ>_!u4zT|R$IAkb7(v%yZPjC$kbaH#y6<5#e@bDW z`FOn`kCn^HhpyysnTu$m&D7s|>c|uXJK6?7Dsa!o;7SQRvgva+J$tY+9QS39F@xvw zO`FD~a7Ez??49B1a<@-O6YTZ$-IASV(u~&nVut!Q`}=b2n+v`ri*^s(yq@VB_oe@g zFH?3KBUQIFH87#nCzxLAdmbuvWz$<)vRS$Iy;e0$0@R_Dr;!q0tzZA*w~Nb7!XNb1 z&zZ)}e@2eSQZUu$z%SS zjR_EoRU{<*IsNNxOM~A})%7?xP3c)(V>ZLJM`8Kev1vfWfjuT?JEp-*D%fCUTe$z& zoQUC`KlCH%Lal>9=NQS!A3_GGsOXbHMrf%N zVToWs*=2=s>|-MK5Bn(=Q3UzdLc6q>%R!!*fXW%%%|DTxtRQH_i0!#A*PrbjwQ{wbh zj(4kc4Gy!u5aUFF8-F6Y&kU#FJ7D2tyz_%I!!*Mjiup2SpqH;xwp@qVrJ*~6(V3Gv z4#x$o2!$EetM9;ckQmS{R9|>m2^waLIUB z0mc^k@(Jypb*`o4y#H)`sh>tzrbqnlHKz_Pvc?&~ZwL>kAY2<=2N~QyJ57rlSI8y&Np^YW4ozTLWc12R-KMFzF8U zJc@bXaHo-LW*Su<39{SR;==Kt8G@3m?CLJ<*LkTw@W8``wxm+zgP-1X0(h z39@TNS&kota*q6*V`nI-6r~4pN#T6p>SAEQ=LQPpiHh^ACFV)`a;`Lq3&NGaCKmt@QdO-i|kW^JoLTy&iGT{$BG~v8>_G++?nSHg| zx@rxD5VY4&_-;j|?Sl$~F7jy;HWT@;h`iz&vxo`8bA_)4K(=~+vD|Mli@*1a>^A~$-i9UI^S5*hz?CBL{e)#*NB?a`*D|A zL8^cF`=BbjsjQ@Cl?ACIlqy2#F27|4W1=HH3S!qJD7D~x4AAS2&52-Bwxa+C%PYgn z11}E#bk|2{;~BeiS1*Im5_SpP;p?8_dTbH^?xa&e7TOjme?b>who1|7gWb*KB1PZH=XHS#KMI4IXp zL6<%(ot-m3X9}tSP!)A#QRV`4Z-q#5w#zS9w3|%XiTsq!>EC~d$ z5=Q{Y;RL1M*$vD=XE?cC?+P$Wx-*E~5H=gjS7k}NJj&7S5TTpK10+*vf{&#F$T@-2 z#753EKhN`OPglH4lI%`7efses&}%dTwfG~!8+Kj~6FjRJkr$+_kFCCTc*jFTyzl=| zYy0e_-?+dF@@r7Dw6^~$$B4EVbvkPV%@(!;LfOtb47T7f={Z*mr)4b``IwSC`)#&-u?XKWerux z{82V<9~G&f-XQ?lyLgifYhviHUbp}Zy$x(WZ7a^Q)&2ZVdji#EBG;RR)@H5|E zBt`N(7PRrW95xnOCIz(~p6T;kpwg~TL)Y$i{SCkBwvb7e$LblTpcM~*<+>#HCh2OSt-tefYN?%zv?xAe6r?zl=(Ia(y(pFUQb4~ zPwxRxX@qaXn82*5SU906c&qm$?EZRO&$ z!^Pxi`y!%_ed8A+{)HEFu z>wpdkR~{{1SW~v2|4krbB5^~1P9{R+>?i0uwr`GDpq!9kl9b)7UjtTxl)bEXyz>JH zF_rUz`)S_=Fdc&KDNRJIWZ_d7i>{oRW39|K3NjacsJ0@-b?ZF~UZwPmL=Z91PY~?|T zpt@?#?Vpv$r0g|*-05nq2(uBXx$In~Jgp%X|1wkXlA4YB30o`MS_n-O-uG^blc2?u zNWj;fBQSl}o7$OgTJjk=vF-Eru@Y+fj@eh4KGaSRui_cjV3+j^=wC*Gz2!p_XP#CZ zlyw`~Ngnt2FJXiI;F_veg&BLeH<2m$%L;1`=}Z&|+&N(9FhGCpcKg(t@<|l5c`Xc| zAdO$2C*P~4=8ibRb?6OLYkB+W7riE`5jU_1S~9SrJR3L;HuKpCe#G+29l+wA4gg*= zI0ly9-_BRNq8N4sU%aSz)OwKwa2|Q?U6K1B)(7&vC;1*G$~)8hzzZR|C3aQ|7{{6TE>p0F9T zW%lxs)`|N{q?8q0>WeE41ax<|lW1kUu?6~umwL_EW&j1IbIy53k4ApszyXT6<4(4h zeQqoy3@lMtHF6^d_hk2^*Y1g-&lysk5O&c1mCFO|c6cc4wME%Cg;q>pM&VU5x@pTz^eYku4X zN+LA~PsBzmFiqJygJKPZI~zgVR_y??x#C%h!1a8zZSAu=voCT`P5&NyV2T#Wf8_D0 z&kx$gy7V~ud0J%1Q}J5;_NGE(R1eQd*p?>5Oo63bKNHAuQ}nHD9dLmIrDAyygKy+{i+3o!RLG_0aPANEd351$g`~y z*rTz9r+8280vKLP->V}^V+@p&Xu?C$2X;q%JYBm>?eYxqY#Qbm5aIo+3+Ah&lWX2XgvfeIZ;9fSZ(f{W|aV|20Q8g3>GJ(I|7(6VesO!jPhil z0HKt!^}@KcKK9fI7a|NJ_`GK~yvEia#i>8S;u1552E2}J%)2qd<#%B2Y$V^Z~UdQ)`3qi z!KdJv`rccXqqnj}Nk9)Scj0Kc_^TNhdd7zjVyKHX{g#4?vbeK|f&VdreLF;w6NT@Td#~}h_(`C&0uxbCkLQDA-o|B|xP-l|CT=VmlRVj1s?S?V6FO zM?Tq|$M5_QVXP-P3m2FyZk4LR`?%Bd&QL4pbUQo>(;&tat?6E&n$|t3dko~-jHmL1 z3wnd23`f%G+(~UelRLJDpz6)SI@%OdSW0ewTA3_5=YWvwhS17bxuLgo zL4kAmL?GuX)F|jI1LX3MO+h^MR=V?O_$MrcIkmvTgNH|Vuv&>^6t1z&a|B&9<81K$ znF8w13xNQYvy?0mW8%iEttgxAA1q*JBCZZ7_TY&K{^;(pF@nPkas=z?h?1|k*v7u0NiGf;!}RiF<0tb9SprI0XL@_V!_W*{7*ZV zfM~Hi?dEyyj|@zqz-~e6r^W5XOJvk)IykW`dQq7G?%>=_D>>W^>p&&Ye>eI~IPFI@ z1m63v@R#`SciZHD`f~qj)8VUImD9ND+MB{#@9`uA&->$1IcB8Nkvl>k8V{EEf2k+O zjd-2?k)JR@+F^yCSxwP`-R5qHn2jZJNnXVhks#DtO-+JGI3-r~EU6#%5I zur5KjKuF=%guMBYlI!-_3Q9784huSSwOMkxX|biEx&(>m{rq!Ug_eGk`2e+6MY@n{T=CW8-Ogja;x#dkKs|;{ zV|`rq*44-#ZpHNI-K)#p56`{^42aBkymq2QQ8Mh_|20xz)3pvxb1HTUR|kfhH7sm7 z1u!>oBOV}!&XF1NRZ{@}b#=)G46?a?$Ezno>b#4A(PBbTT~DA@c}~VU7z@f@@NTr7 zzM+FBt1V(Bvg}{zhAm*l=d8zz^8)cCf`9GM`NqMgLP2uZ#cL!6f!jya%W(hNC&ojE zv;#Ll?}mtB!lnD#yDk%1HJ+f(hYF5+9_3V^9;223s&J&8$>W3Eb6=Naf^qONNiMI* zt|F+E46`{XybfxqZ>|V+pG_rJMniRl?CU*_hZs^LJp8CH#ptO{;3B~iWG9?GB2}tx zl?Bnl2Aqar?1CuV54!?6PUKUDG2mG~4`2lntTZPd>xq56uF6xq^Y;SG&YL%rEgK)S zPQAweens$oLWP{4>xP&$Xk}f`dEnghYWJ#$l$^ZbT$W8Z(xD zCy^vLtEk2NXFFf(wqM&mgS@zcqP*fN6o5{IOre>rzYCuZ&kNu4F{H>Ez;-Hc+^WAB zs6_^z?Jx?|oZXQT^SIUCWQp$_@eTLYX60m&D+_|czbTK8M;|Rt7x)=2?wb@?L!`m< z!m0%|HN)A7I%*>^4?a_@!CSwT?kYH46OPNbud8lM2n8L^Jzh zdXxwB^JHN`+LaRc1We>s*j!tquQS6IZTZApXK8e2XP)DCRf{d={NN+~zR4VzB7wgi zk@@l$>BJQYxiuK4KUFuJpX!F5vu-q8^v!BO?6VrFq`eu8buN05!orTbUX6qO3-7+! zl<;qJ;T>^8-P^8%+ceQ1!W5qcZx7JvogO~kTTbNTou~2`C4Hq6))0nKF@6*3@<>G5 zX5xhC(|m#7Xdsj`vv;=6n!=r5PBZ$gt?s@yYE?f_DUV zgoHcy8Sjq>P7Zh&lBA9QKyx;bQ(d2e5Gs*&>T!cAhEarFd_I)oVg|Wgo-!Z;NvfVt z05oE}9>z#k+prG^V=CimYf@)o6xe&0Ho?|v_HDl-Pc5&IE_v%=Z7$v(3c>9>rm7B3 z9f5X+Ie~}~)wkuF8&f|!9*Oqufbr`Y5b-JH8$Q}|WP6;Oj|+p$6a4uP_){Yy%o#TM zI}eZOY}RW&a7fCWw(*%??crV75NMExHyu0a5-;-7QTC99o;BkD?hrA^Z>?#>68LMp z5`ntGcshlL5-oW@mtEr<+89wcX-4PmiwNd2l+@}&TQB$CqxDEn0wy$h^jyas)RI}x z#dn?~{R&;Jqwms`z(!bpkm;gX+j^bzN=EEdF3Ofon-19DcPkhwf@XeG$SWF!SyOE~*pVTLss|~?XVrHX zW5FdbfII|a4*g?7>_+1~!>`mAL$UQ4c)olx(C2R_Z@S)jwK6q1)<}-adkN%HFZo4S zwLTxlTd#!cR}5LZ)EJNm+SVFvVi2kCdf#xz1z{|@cv0Q78O40v-bQ^d)z{&fkOz=d zxgQ?5veK*F$1CC6L?HD=D`bmKtU~x6F$1+-+`;2*Tip<@w}S+z^kuyj!N?McCNh-b zisC87u{Htb^l>~bN{KH^)K&O0a5;+)C*|^%lRPrrPambor?akB1*WJ;ySw=H>88k} znEM?Iqf$I7r5cIE`X5YcTzAq68l2yS>V&*p@F|OlWz!bGK8+lpGQfR0AhjmoOx48) zd`B2XeI!%vL9II`N*?P3=FQk|47u_B^p%in^6paU++9{!i1v>-gr+eF&4^qGKV5tu zSbn-B@D@J|WuaV?f~&~u%rNP_5UO!2GI@Y-mntpp1Lp(__~Hj{jXZ(xEG@VNkK;UG z;I0VI7w$B&Se!2<9RUSYdMQaZ!d?{nv%&+$S8tN#Jh-Hrq}+&!vl_VLHd0#L6GaaC z$#KI&(=r#e=_EwXd7tX~$d1oCD4e`)eSk99V|}*O?sWLhG$dmGZ zKudD5;qrd<{EO=X5zCOfT)>wKN&bHtd+V?$qpw{U6(tP1V+f_Y zdr&|`2~jEO5|A8$p+OX-2as+B1*N+YC1%J0siA9t0frj7IS;?{p6@&FD_+;(FD@>g zXFt2vUVE*3-S>6vRY$4y8Qx6;ST>kz{T@L4bl&76Xhl3y&GP)M@lD}~2O}}IoQ(-`&EByjQQ!x}(j7#!*oK z$(WI3e<7IfsKNsQ>GCjmWtxRz+xqh)T&$up;v>aZIQ9 z+UdphP4`5&H`i5Q#29>A6{i0FQ5`6#kDY~iI%K=c`Q!H{#wyA+uBXNC97Q-+Tc-#i zQRrtgslnZqf-RsGxvTfu?yo<7RZ0S7d@iI;{vT9*XH6PNrCU5i<w#zlY z-_Nw*{V>RS5;Isk66#Bg^Z}D!J$w;TX!AQ~_g2fp(B2M{km-JS2{xnQhv*x>YPRXn zh8tJKF;M0$Jbd#h!LV)0@z@}YrKpPOi-U=YaP=w5FV#w10rb!&P`L5D&kTF}Ptqik zR#}&~!|gVp`n=fPx%64*S7`_Zh1|{Qz(cLKr)R!4n+yzvHegR00&@fgq97s-xlM_c z&b%vd!rjNo?Y-~(5N+IUUn-l1wTPgIv6;!}8DuPdpFtuSU=n@P&r?trEy?0}o)OlK zN$NfP-A*u3(b;0H9^S)Z?P4Ax$pKgJrI0YZ?PNfBE!|T17SN9px`tOZiqf#Zw;b&);Zk zXuU|i=(^L ztI;`qaB+>6^`(2X+omP4C)TPUKd}!;D+2Mw%)5--+kih(gJ1u;Sdm7>Oym(ce724* z(J?%__G~Dmni)8Jc-{y+cWLNZjmZflL#DNGA@`>Pz>cv&XgU37L1`c4sIrPGh^q!* z$<3T9&PC#@s_v+u>8m3jjT_Sh4RB{7#fr>VDUb_{^H&%P%LszDl1GNRvY#ww`!nMA zFYBY!p#BJ;3P{E)^MSBGz}BiUS*8GAcs;%5M-=L z@qDnKGx7)(Z%#j9ez>osX1qO?$}mgC5f^E6Q-y4LgdUhkDVFy*FxyP(Eoe}dA6Z|X_QpR zk|VPIwAt{+$rl} zf?cck+e+^cm<8_)u()hox#Glz{G~vkYVYt{n=c<1d@J_c`GWkg<~X)qiu-BlSA7-M zh;y)dX_ST3hw`;Pzgkf_90S>V?ym#E7F!&v#KsYps-OH%gj;l^bn%Z+%nT*te&U@P zn}O8QVnhoWBV@AI>Z@iYN(f-7p7S357>~!t_bVLIwUHwsZMCinY1`Bx-wpp{w0=;0 zqRn6XP&#O`N0Pl5QM8e30vH6vJK&H<;HO$beAZbl+m# z5O23Cp~60KaOq)MAObTc7d#77g--CUB8{Y>Ho}3BWUe(f4WIrV*QGWIWau8bl`x8k-?SEV8-&d z|A?I*+b}|U?Uyi^Qp&4GoR5jS)?`_zWyVF4`-Z z@V*-xv<&nfeBe!S)Dh~cF=xkA6HxG+Z=w9*dIQRkrNW5@Jjh+8`Kwgg71xHoYP2yU z;5HF_PfGNdOm2U?^nC!H>M3Iw$s>X<0eHRaA5ild&$;yY8fXS-rAZoT=F6#U0(%pB zW@bLGqcTVXDNKoHN(N-_o5EQ#nQBNvA4N*vHV>A=$S}G3&+@a+HY zaijF`EYjP?Mc4DmwYFbYEw!6dpKfDPyumINofh3>lTxCKHE1`Gw3a+g8cn94Lar~I zlkLnQuH!DGX+9e;(*01zE0y5RyR*_cpjzZF)r7B+-47Sle2_6lRi)`S*zaL_jyo$I znzx&4@7m3uE{Gd!`u`EcsdP}9L9Ibu+%|A61CSGLSi>Y zm?(aG*ss-f@N^iWW*OZ?TdHz4G{$PI;oq^vrIf0>nnebdHGku*XmjyPj`uAH`+^nS z3?nD!oM%A6twWG=PcX^$QI;_aru@ z?EG$9*=ogz*`f`pSN(lXM%a$ZawZ6Rz84&7!Km`9l>|{a5e%kI+Oe!&g=L=1b5hnN zal?cMGS0!>_QV-GK7`GypCo_cB5Q)0^_xksBXOmNq(CER&B-4Jet#$jz4!O}lFy0Am!%f%2 zbu>xT8Fj$-Jxw{KP?DNWr5k=ysz*X3vmvPm(j1BM2ikbnNw(lHyFN9;wp%Gzyp*4U zhHA?D_YTMHBm|1bisYRo-$;hKWI1xmiX_X`11`0wdU92axd$%P(rO3|Z9}(oGwuVW zS&0UY5XQFW$@mPRxk<0>^*Wng|M2?pA`NO;g^dj`s@jHR{)HHLOLP~X#bm1I=Bh&a z;YI3Pd<5r1n@3q%!yEfxcuvqC7k-Vjp5)gq9#Uh97zaDos+T<5T(}XEQNi%bYf!Yb z1!`*ajCS5>k%&@)l{$cQ4#mKl(|A)>i$wW&Y8JGplpcDnHU9Vjowct-o2`#9t}k~9 zB0`{x(6Z4$(5AbAd5-NoX%R9K>4T45ea{u(Lj_*#`xe5)aSDcxNUYGm+rxwVvqSYw zgV-OM1cQ0%Xm6lcOhb=`0BNbV#y7vbYh(AU=39iDp1O821q-vBDBMC)$`QDj96QYW zxpXigD1TER&5jYk-RP`@&5@h()2wdDwwmmDpqUytw2sH{bWPwQ^NhR@xAY7!v_xma zd0DHN)xoPyIb1zSp&_A@3-+4?f?m!yHiH>$X6V#1SXYTnVeD#q4@EI*_futaxxLrZ zxOmL7Sk3_!wxjeJBoq8vqA_$-ZZ%~Qku1+b-rLd$vXad}2LfOLbgpHhtQROqU0Bu% z_FCjl=8tZiZ4f35t47nx6P(p zA!%LV3*oLq%Uy@j_jzZH_NCcfXXroqM1t{-B_i7PU1xyg%Pav|4*vnV^2zQh+VZ!0 zgJ=W6lW1qvpLZZdEvlv0AQc@DsXgV5?lq}*ANJKlw@J3~{MjvJcGfUv1vHK8?!{;V zk+4Gw98BRY?x56i=3;H|MRa&%+N;AJ{RVl-<7XYHZknSt4rB>70I985l3yS^<8K&R z%y4frg2LZzX6#dv)HbN@=u7bfd9{N=DtSaIE}{||nGag{UVCG$&S8>7=*cnR&k&ttFwS9JmoLyRM`o?w#4zh5|0yqw#=DI}((mM@WEL}yQq1q>7I+x9 zAC!uVnyaotLyEv{MdO$^2<$~_WOPKh;mXyWXBD^2Gqm6Wi%N8Os>OWoMtY?S16dmH z%&qnU99{Zw5)6M7IhQCWwcCv6?RCxcTUS11p~*&e6&Ma*Gxr8gmOW>QX6D?!NTE`m zcpAYO58AQ{yG5Qb-Y#dtc$o&)^nNCdAfX9voCgp}RgnMYBXWG?=_cZKFpDqp6fXG_!b#8lY|9@m^S}ahc8$!HTmdgF$i|HD z!3J)oK@fkLbKs+e+}YBCQq5OSjVtl>UnS86Tc&3I7}u#sp%PbQw{qImSv{mh+Fl}A zMa{cu%sF?}#C~m;E&?z}X^)}jHH%I=9!nGoAe2#jXFD9J>-JGV>jnvMb8W*XD+5MyR+~uY$km zoYqBs`GccI06GP});ne6QhUYbqjPIRK!|i>ke}y1$!`#;AJ5dOJZ{^JEmN)Fp4UEZ zd!9OEE%%p$xPpuEmDv-zrBpoOQ=ya6soWXkbR_}Loc563UOeT~Huhd&aBE?p22ezm zyG}v-Eg$xt+9Ay9L;n1iEIJ>{{Kt;*Ms*wO(Y%`Z$xs7(;1 zgNZ{$XL;5qpdii2=TQGsk*Koms>Z*}x#E?pBShZ3utznNb3>x$3Rkkm&f0rh#J9iW zQ8S)l&PsgXrJVsSFvB|>KH-Au=k(h6lAEV77ZT2tS9>31kVNv6 z5gFcSIgir(qU}YnxGml)Pkwdm$3;qd6Of&ZU^8Xg>4<8Uf;H7p?V@8q%KrGOvA)-` zPG+r7NI5Cx=rU3Q7e(-6s=u)+wxiuYm0sJkkwefs#^O|YqTmMWssqyMq(pj(tMg;Y z&(IMSoM;Hf-a`5$3OfH{5`2Qi6VR3Y7KpNYfZmB#2?jm&zc%(=b<^Y-5QcgvKFSTKQ~NQzYnP=uRCg1u z?gs#Wh4n1H&u&kD?6wIKmYi)*!j}3RXv;B7=*Yi?1iL6mt7z>NE9TT_DkNLTpFQII zIm)<8JR-FA*4B>zy0u+RvE%{TGCo*)w{Vf+Y6>K|5&)CwRh|YRqSAuJ6(Z`OCTI~g z{vv?O22r!U>3h!=K@ixn;WK?)lPyDr3~pIHq3Ql{@0k+aob(^G6oFI1K76aB0OSz$pxgYE{4R|WB~pLZAE5SHwj}nM6N%; zqM=sm-2&s!;}JjqyJN|QM9qin2O4-&oWv(LH-n7>KJITTGE%wG2OX=;G$$c!Zy9a; za%Sag>5XqY_})5AIOBREgrsw}t7NaL3+6HZ$WVg)!?POo{AH>`|Ir9F*3tkP7OR(Y zh&kLTXxc$F5{s_gi*6|0KB}Eylb|Yzc*8C8aaDh~gVCt;p~hDNs={mnr6~Gm+wDB6 zboCa00q}*Y2%ny`-sy9;s9xGHwe(?s)zo!PW4XV~!MIj}XK_G2;=;KJz=*9UIvTHF zI0CNQ=uJnIcX59bxhGuW>dp>p)5q6dRao6oxV1((;$gPWi%=tX;EIOZ5g<*b-Wk`M z{*qpjoCGkESQo4JXH9N+sBu9+_vyM6azQtwk20K_@R?ZJ+Y!UAqge+>X1|!P6^CZV zt6Gah)cmXzs%qWuNqxznF@h~MuYtb>y4;4Mr^1Yp;a@yNB$DNcae@Z2!--YvS5}#| zU}b@8q{=<5WGLCbpHuo;>%K0Vf>N44=UU%5O%mO3?$@MXly&2T7ET9M-JPwSJ+Tmux5*Ja97FxAx~`Bt z9LzM(g-a8F<0fZM+3N4kN?v1bJ3Pm_EY@e_e@)+DgNfB)u7M8lJnXlbx0qvF=qJK8 z4m3W7<5=h56&{O;fa3NEK+t`jFq57fv%mWO_dPj_5BrBe=8znt`uhyuNUGt5^(^~FxHbz*??cvl#*_EkwL7r`gt)pz0NL^V43DBxB^LpGp z%2G85!_W(fhY5ZY%qZ2{VUKkKMwm@(4y((*L>=iQL@G7$O-oT@XI1YU;mp;xwd}J4 zbXO|;hHIQIVJ`^OB%e9oc;=R!pC~}>T_^axyXHa>`_<oeAs`7tI1#*Q8^=8Wmcd=UAw^M>CHyGQP5)Cr)h7nF=EF6a%Vmq zTioqYstQ+=dx@@l!rI)0I6Rm@A-*zO=xFXLg%DqH5rY!V^H&E9OYD$(a zWK3@{j+c>T;h;hc?>2v4ukWzaayl{lwg1k(mf*n#W|QI8@iQ(&@Mq=0wirnU7bE1I zDH}q}QQxoq1ni=$(v2t4(z%V%Q4ckJY}#WixBM8CbEf62ie~w&6pkYxR3e!mktsA# zkRr$U;J}MCxg&mivHXBx{y1h?RuVHwf@;Ur6lF*Ewj%dqNdqLN5*-h|;lX+i(l|3M zAzw%WklptcNRsyx>}$xAHz+e@6><^mrf=nFlK10#QS$8(^&k<{!CkAdGw5AGl@6QS z$!(gw4=5M=Zv{}gR-lF#D8MqXzDTH6>`l=6mWJKEG55Y!HrGjrOZ1^_mpw;(OXd1W zzA#n}v~wFZke=*A`FE^vG8FH;DK=|BWZsaa!ReP~A3H?DJoFv;2i~y)Rg>@&77-Q1 zeXA<%oh;HlJZQRR10a>_JZo&r3E)NQq|~%7% zRgV`FtXsS+O3ihC`+k6kHb{U1lCW<}FOo_0Cb!3ea76Ms3u5o? zlDKHtVkM`qK;{{NG7w||Y>G~d({hDwHp$5HWOe&z2*Y&;WcY^snv`=Hy6#YO(_|vb zBLHthVH~EoD0f)kl92{1JfCJ9ZMZ$1GV?n{q#I$;As?=BvGFeF(WXrciwM>MlUAc! zwe9y=5h=lOh@+cN^8_s;q=o*#_R_AQeo4)D)vh9c^c@{udsT3cKZ3*}M)_E5Tu9TC z332PV%;FDcMYv_qX&70r+S!U;1VDUM5$*^STl$CPD^!VAZ^!dLX(JjBGfBkpTN=9g zUYf;!r1)<5n(fybu@Lpyuy+|j(qWG&YsrO4@3lQ7wmvzx4SPY^LCyBu>?_wp?Z_iy zN3MiFi4UlXm3)ZZ88iJ#T~thw!4U4Z$OnsQIcx*Lb1bLQoMUzhI;T>aW4<*HiCrnD zxtbxyMx3Hp%E|Wk7HrpT!*Wi}z4joD5gi4)5dLb`=M|O=AJjz5=jv*xMZW4IE2D2) z;!@HGV(6mwL(;f~%A#CZZ1x~VRgiV{om|Dozg@_-+^qH>wsH zI85Gzsmj8~N&xh}-d2(%xmQ}{RZp_zpA1)wr|-Hg0@v)3i5BKC zh#{k}><=z|#WtYYE4;P@1bh|dRbbmO(jG7df7@9jeaxvfAKWys8~SFp?(jrN0z6}< z5Ycj0L0J`yDs`;?DN|MI+`BSK*+~>DSx@sL(K5GC(pkal)LeZ`B;K4C6yHLYa$emn zlLGQ{YZdW*=u`LVZ_hJ>@F~~VBxOkG14mW%42uqua}6vy0F+7;O~ zs*WGdd$~>b2BOZjy1Vr$tE8wYjt4WvB2Fma3lp@74`?WkDJ7{JI*%55f^l4g&ur0k zVy3Vzbpgu#_Q>uBsV*=Fn_Fc$>2r_5&G^FNtR4wRt$n#|*Mps41Baf=v(PDRes-USo{v+_Owdj zwH`HD1ktogWJhUIrNq&4HeRM=PLjrfTp3VKF~XNj8?~X3DLBR6m>rQV-OphFddxBG zo>`;NC|dF7HEHa^4DsA_ct@l{%g8gcv70MkqVvqha=$)47U|GXG3uwgKrZEu-K z)+x<>s+d>TDk6-}YOlU)FJJ&&>E)qPJu`5POMl$x2{DrQ49hJmNQZ}>zUU3hKjRCi zv^hkqmFZ2?m;#3tSpfX2@#u(wW!>Z@<>q5#Gw1c9&tGIo!OuZM=c#jV;`-XW)8@6v z!qk4^&aahgRW%@=lr1#OovUhw7kXNsSn*ER%u5+14pRf1A#-dP+8j^+0#Vn75ncb3 zCoyNys&WZ)Kf1$t>r1**1Tk=%yXu9w;-<4B(9EOAd>jHy0#yI}6g@7gUmpW~q(s&l z`Y4FB*%D{+gN&-5ST9Z?6ZW}z7Y@}9Zl8K2pUY0+U|l!@C%xGyos2htvw;~|HGCKg2VNIJ!o^eZ|PF@V|E9H=gsyHXqO{du}g|B>X^&b=Mu&I$)r8!y| z)d6=EoR!E|=-rBJ`&5;tn)D#xFzB@5j-wkB{IQRqnC95=kEB%!iYn`Suy5|RCw$`# z(8K)hvR-V|Y3CT?tbhh~pZ4k}YOcb<0zby=f`ANwIuDNiVUh#xO%N^*+VX5@&|lHx z4v5#NpYk{ z^UW4a4v5EdutmQZt@{de$<6D9?MAY6j|STo1VWqIAe399?I$&xcv~9n;W6PM4MMze zTQxSjHkF9?uSRtod$$H^fSSd<8V7_)m3hPkAk3hm2_QwQpL92Afc{+NT`+y6>NWMN zaV3f-?OsD?(d&Q)1$h^PceU{F(>YxEYTgld23(sC5t}|{O;deQJ0J+&P2@*=ZWS}= zL5waZ(N)YPbg34KYSs_xp6nYG^*kZ;h{HPA**&tQ*9*nYjnPEJ6>L8uU+2KMCoH}% zQhtGP@*&3hux_{0Fw-?p_N|@tO-ktO@?R9dJ91LEb(&cB#T#9ai#(TmBN)3iN43^l z&MTUmF}k(D&-*B$Qp?aNnO4QoI#P~U&&~nxThf>Dx1Zf@_dJ% z*QnXL`|fLTBe2KNb`4!oI%3ep#;ajn|IxSN6SrZBS{_}HJH5ruOI>mRg1%|;SBxIeTye&I(8M5r{? z82t+F*SVFI;psF_PI+3AX5A-1D5Y`eb|_G^!6;dT*fRh)1q^UoWA*YTQ@#_I^4bag zBzhYDG(iaQ42AaEHOd}Fx+fTTEa?qa^9+m#7P+3VarX+YM`3ZaD|&SM`58l?YJ~e? z@IA+{u@C-sD3u_uBPu#Uet(^&$CkSC-XCJ;Ta4}v;R;1eQnX@SZAEm)jL;G=#EZB9 zgX7sH%l+v|$41M2O|QkvYNr(lpN2;>3oGcxHHEn}>Yz-A1OuQwhCx_<^2P$cm?SR= z4cMqvL4%jgE-G2+-tj z$qakn;$v%OTtnGPinDHjiBGW?rlZ;QxpNjXzV+ez6;MIwyn49HuVjtqR3@?b)41+&AVIr1k^& z5sQ{!TUj_-QFY+8*`ETmg$>?|8R(|C#8PmP<#!u(jZmjQh)Js8wxEeJ7}A2Q9Pz^b zO@j6P!RI0couQ`hptTfDYm0e`#cQUd43tDCSc4HhN*%+ ziH4j=0I7O}E?ZI!&w>EEmszmbWJnM;{vu7v$)r&az1Y)r`FL^w>@7QwyB%InX4@hDo9NNBnKdim#GE?`?ib7Z{ zQ!~+8v8zfTGtWLx_DsJ{&@Q6Fe|L99!PfV(%71_YTS%9vRi;bdm;hsJ!|(2!YNiW4 zDcjTNE`1%rvG|hrMq5kWRg2=2E*LLBc_CS7Y-+0|`%bGI2zof{${sgUj<>mw0(@Oy zfvz!{-HjI$o+p!qCW+W}co$auZ}O7BQsvJHpCuEhU%n}FueCa~sia9aF*xTtu8}XY z=>gb{3)3A3z}XLgF6S5%elg=r5JlbzSec8Z6}+`F<9&ZX}r;O_jA$z*GY zF|3T&z8Laicvmm2W))~XtyGsLu~fpj_aC%|o#ab#cb-W)8Jw5xSxZw@3d%%fK~)fi zmZ2%!$U=a6}e2#7P7}RffHv zjlr%{RH8UZb?QI(iz_($9C778aGmg&#?HMLc^i99G?{*bJ!7Lm+pleM72>~`)?P=d-QN2Y;)}WjItP$fe>MeJ^4L#Bz&+W|w^v^n_0$_$ z|8b|R#11zczh(@3r$PK)qcx8{yTCNT@1@U6!?DP(OrV;qy$DvrcnAB$urcn7dHYd- z1*&$fbI~@DSm{zRvG```*;^C3=t0~_HVB}{iYu34C?Z#jD-7B*c z?4WjzjI#OZes(_!8UZqsSa26Gz&MWC>6{Op9CJFM+QXb(KOKCoiW~(Cd&ZVos4RvW z(~XZbJ!-T)DbRO6EuVw0P118Rqul@X73<`->hs;n&`H2RVblee1!Qilb*|W`Skffh zJeSbu{t-%BxVw}86iJlOIQ#O2LDNOtra~eTKzKc;+p{*_hF8_s8k7Pa0$kfirzz)z zD)+U4$C(bI@-pneg=hdEwX-Hxw3pGdGtAVK7m#3D5Ae1??{+R(*x?4oq)c>uhK)MM zcr|svhaO*7F`}JB1>G7OgrB)=|44t3tpBbh_5zWgVyOdnKg9|>MI#`2MVv5n-CwD# z|Gis))(no+eTtF0!|7Va=g2~bon^97DAD#9|`q_)sppp^g?mFSlBg|;< z>8!hN`M`?x8eMjn){@?EQ-R-@Obf33=(w?w`_Ac4&EDS2*XogO1)ryVN=m?}RGFTE zOC5hXkH59wgX7IQ@z5EzVz7_~V7(;QdOh7Sq`h{ZDO8JXqdD*qGC;G7qli<8G1f^h zVVB#!w{#)9Y|^*?DDUW7n}y*3t_;46+MDz{J@qz~6TD<1X~Xg9X$TOsK{=>tw+1$; zE||e^hU@pa=T_?)BVxkU92To()M1|Kj^oiH}{=XQ^IQBlrn2H~`Jot|l)x4c=cXaQ5ba>I<^oYm4IB9ZkpvK(5{f>FWAXamO z3)7lPE|nS)Pn;bdIeVKaAKOo#QrF~lAa3G-D#W27#+P4909Q$a zg!C!xsiC%r+275Vc|6Ds~5HP*anun?#^YguD3)H;H%c|Fn>;=sbQrl zqu8BqxSF5-0Xfj*>8~>%SNlg@-aGkDmRxvA0PS-tXT_h&PMLy~kl(n}u-NJZr^r}I zlKp?3S#F7n)j@MPa62B&f?%pp!&Z);K03a%*?FdN{*&H%i^9|%aTMw&q(dJjxzEd7 zhcxu0b%#gs!G#8;(A-MMnY!9WB z2XM_J8bH~pp^75oCiLg?nWC!O?^n@Y)hSNq3ll#`UmsW0Iouy_#d2w zP}QqnURBF3D_Hs$QYQJZig;K6cGPFV)W+!Oqo8Rl*mtEnA1UW8?vyS){tstK{j*%4 zNoamLtscE``iPES{?RgQcg%E~u&d6M*pjL)L%%)GD_fJVfRmDJ4JSo zaB2vK_VR53cqx5Pu4=3pf1#!^*?;5LNNZ^C{AN|NwL2Zu%cZx{ra7eKpYkNun$uEJTC0uTetW3F_#b!kCbFf_w+7c9wuFJv2ET+ zAv^#l8vi8h!yo7Rz#a76MZ2zuq8wHXw=odc)M;{YFczhDc8e9JG6Y((kC3r$>Mcf& zt^SLkxg=JRQ$^R#A)V0@^Z4jIkAhK#tX!xAph=V-)eK42Xe(gIMV-y%C)RGZXb+UE z*V*53_PAv7|6{lzk3XlrdmWqda?)#m%rxrxq!*W_K9ubhEqK4qH0X_H1x52~3SlgK zsWM

6bKaHea^7V#L64l9AW`&Sm+=|M)PpK7b%j)?-mDAbY4whee4TgYji`V1_-TeBnDASAU;9c?eo@o9a#P!% z25)27T$wE40{&DG;kYtzSLh#<>t|MmFJ9~K5blRNqK36x?SMUdNy*)R&i@Kdp~p!M z&SN+eJF`=-(N5oLxB>dN{IPld0bH z!yAE|fd|gaC0VB$z2!eVHQKm4Su4;ZzBAs^0FvdOZw2VSze;}#OGjIV`jv)hZmn6Z zc5WUSaAC4!xGSJM&;iJ?yh(jiv271{Z=7J{5|m7K6T$|7kO3M^-<8 zv!6_7%W@)1eHqI&GZ85*CjW2%7i3q^Hq?+GXO!=6tBb3S`lsVT8Vw(avarit6w{r9 zzRsA9MMVlXRc#9vj9$A}r{l?0I~6>wbS`u#7X_7ud570*GhpZw)?Gl3eWjAuL0s zXN&OmEzHh*EV2+j0bGnKUiDsTcFlarqv66F%3U9w@8_cAbkQE(Qw@=mwCo!YZn0oC z;0F7&DdfXsk}4;BNm+f8At4=BTv{g%gS*=Om9O|e@>Dv6l;n@{@tFw}Cd~#Sh3V_5 zEVTgw&wB1*=<5Q_wnz}nDKZ;UFP{N-|JO@Oe8`rnNXH9tGJuacmm}{ESP#dOen*NY zK0ComlJ`Aa3(h_pI#bVEzuKi88^7F%R%c0lyPNAEVIV&_h^+$-eeWmLsL@kHMD>oc zC}W*;ux8ZI@_}%(Qxr{9v3FCq(l+E@-_yr%^^~my>cgDQTs(kl$^DiJP|1Eo#V?%d}| z+Ijuk_*MIAICfZS?t7UY`t?cOf_`SeE)F-oz`P^1@^Y674x)n$-LIoTTqDX6 zVFSHx*u}IuX=<8@HJk`*l-tg@@75yL|8i3%6o^y%Jgq$E)SH`%c6Tkv(y}j<*5H%r zq%F_C{%!uHw5}wAS6V*xUp^e*!bDhJogC&we4`$lcbeg_6yKfLv(Rj$jcou9{81lq z-9-ba@jn9hS14=~0L@P;AMsw3i1BdY?ToyKi()1oaW_raqbUo_I)r_MoqjKmc3(b< z7bH7ow_kqkkpj952PO=~Y6n@!}NsN4z(RK~Q;7 zbGF?f#y$}pX>tzd)lGk$=%-88a98my-j%EPgrqG0^$R|sT;1EFIl9K)_FZp3iuGK!}9yCZC=nMoKkiIP-;-=Yh%Phj-OEIs`H zU;h8KvO{RaO$gvKSX=7L>&4LmOba%HhGf05k_tBMeY8VD|MwUDpNBXIt-jN}Y8MJT z!jQAqhpHk^U1v|lVEW;q;@oEI`JsMj-DCTaP1GjsU5 z{7bL8(}f3ydgxQ!_hDP2M^9I`Eh2mGJ%<}b;%>A5)e1{VrQ+~WMhd#CGHrBM*0nH^ z{|svnfNIN6Z=UX8jQ708s#O2yIQ&B6L%F+OP1!6u=+XE;e#Nw)cVgP4B^~4))Kz`Y zaM3sMcjMn3V%by-T-}waZ0-COzBC-EU;dzP=k>R4UTeqZfCCoH?&h^B`TyQhAM5An z(ZG1KrN@}cXTxaQ+*iZe{v{}IF7spg&hhGd*vDSePVIRA{h|MKF%(}nzv%~+gi%Jj z({OtwgKeIYRq>Xm+W%N)z_6Qn1;*d(F7q&HibCQ%?wHn&7%|Q_ZA&DD|MwMnF++-X k0^`HGRa|yYSgU6!W<}|KH@U`?-~m5sDw@hg56s^EFU$4}IRF3v literal 0 HcmV?d00001 From f64a3309d14c57ee9ee5c5f4e13d2addb21ed6c4 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 11:58:58 -0700 Subject: [PATCH 61/99] fix(utils.py): support raw response headers for streaming requests --- litellm/proxy/proxy_server.py | 8 +++-- litellm/tests/test_completion.py | 18 +++++++++++ litellm/tests/test_completion_cost.py | 4 +-- litellm/tests/test_streaming.py | 43 +++++++++++++++------------ litellm/utils.py | 17 +++++++---- 5 files changed, 60 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 04034827583..0ac1d82e073 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2909,6 +2909,7 @@ async def chat_completion( fastest_response_batch_completion = hidden_params.get( "fastest_response_batch_completion", None ) + additional_headers: dict = hidden_params.get("additional_headers", {}) or {} # Post Call Processing if llm_router is not None: @@ -2931,6 +2932,7 @@ async def chat_completion( response_cost=response_cost, model_region=getattr(user_api_key_dict, "allowed_model_region", ""), fastest_response_batch_completion=fastest_response_batch_completion, + **additional_headers, ) selected_data_generator = select_data_generator( response=response, @@ -2948,8 +2950,10 @@ async def chat_completion( user_api_key_dict=user_api_key_dict, response=response ) - hidden_params = getattr(response, "_hidden_params", {}) or {} - additional_headers: dict = hidden_params.get("additional_headers", {}) or {} + hidden_params = ( + getattr(response, "_hidden_params", {}) or {} + ) # get any updated response headers + additional_headers = hidden_params.get("additional_headers", {}) or {} fastapi_response.headers.update( get_custom_headers( diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 77049896266..c2ce836efed 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -1364,6 +1364,12 @@ def test_completion_openai_response_headers(): print("response_headers=", response._response_headers) assert response._response_headers is not None assert "x-ratelimit-remaining-tokens" in response._response_headers + assert isinstance( + response._hidden_params["additional_headers"][ + "llm_provider-x-ratelimit-remaining-requests" + ], + str, + ) # /chat/completion - with streaming @@ -1376,6 +1382,12 @@ def test_completion_openai_response_headers(): print("streaming response_headers=", response_headers) assert response_headers is not None assert "x-ratelimit-remaining-tokens" in response_headers + assert isinstance( + response._hidden_params["additional_headers"][ + "llm_provider-x-ratelimit-remaining-requests" + ], + str, + ) for chunk in streaming_response: print("chunk=", chunk) @@ -1390,6 +1402,12 @@ def test_completion_openai_response_headers(): print("embedding_response_headers=", embedding_response_headers) assert embedding_response_headers is not None assert "x-ratelimit-remaining-tokens" in embedding_response_headers + assert isinstance( + response._hidden_params["additional_headers"][ + "llm_provider-x-ratelimit-remaining-requests" + ], + str, + ) litellm.return_response_headers = False diff --git a/litellm/tests/test_completion_cost.py b/litellm/tests/test_completion_cost.py index 6e4425fb634..289e200d904 100644 --- a/litellm/tests/test_completion_cost.py +++ b/litellm/tests/test_completion_cost.py @@ -881,6 +881,7 @@ def test_completion_azure_ai(): @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_completion_cost_hidden_params(sync_mode): + litellm.return_response_headers = True if sync_mode: response = litellm.completion( model="gpt-3.5-turbo", @@ -896,9 +897,6 @@ async def test_completion_cost_hidden_params(sync_mode): assert "response_cost" in response._hidden_params assert isinstance(response._hidden_params["response_cost"], float) - assert isinstance( - response._hidden_params["llm_provider-x-ratelimit-remaining-requests"], float - ) def test_vertex_ai_gemini_predict_cost(): diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index 64c2eb4abb4..768c8752c0b 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -1988,25 +1988,30 @@ async def test_hf_completion_tgi_stream(): # test on openai completion call def test_openai_chat_completion_call(): - try: - litellm.set_verbose = False - print(f"making openai chat completion call") - response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) - complete_response = "" - start_time = time.time() - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - print(f"outside chunk: {chunk}") - if finished: - break - complete_response += chunk - # print(f'complete_chunk: {complete_response}') - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"complete response: {complete_response}") - except: - print(f"error occurred: {traceback.format_exc()}") - pass + litellm.set_verbose = False + litellm.return_response_headers = True + print(f"making openai chat completion call") + response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) + assert isinstance( + response._hidden_params["additional_headers"][ + "llm_provider-x-ratelimit-remaining-requests" + ], + str, + ) + + print(f"response._hidden_params: {response._hidden_params}") + complete_response = "" + start_time = time.time() + for idx, chunk in enumerate(response): + chunk, finished = streaming_format_tests(idx, chunk) + print(f"outside chunk: {chunk}") + if finished: + break + complete_response += chunk + # print(f'complete_chunk: {complete_response}') + if complete_response.strip() == "": + raise Exception("Empty response received") + print(f"complete response: {complete_response}") # test_openai_chat_completion_call() diff --git a/litellm/utils.py b/litellm/utils.py index 0beb041e938..7f615ab61c9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5679,13 +5679,13 @@ def convert_to_model_response_object( ): received_args = locals() if _response_headers is not None: + llm_response_headers = { + "{}-{}".format("llm_provider", k): v for k, v in _response_headers.items() + } if hidden_params is not None: - hidden_params["additional_headers"] = { - "{}-{}".format("llm_provider", k): v - for k, v in _response_headers.items() - } + hidden_params["additional_headers"] = llm_response_headers else: - hidden_params = {"additional_headers": _response_headers} + hidden_params = {"additional_headers": llm_response_headers} ### CHECK IF ERROR IN RESPONSE ### - openrouter returns these in the dictionary if ( response_object is not None @@ -8320,8 +8320,13 @@ class CustomStreamWrapper: or {} ) self._hidden_params = { - "model_id": (_model_info.get("id", None)) + "model_id": (_model_info.get("id", None)), } # returned as x-litellm-model-id response header in proxy + if _response_headers is not None: + self._hidden_params["additional_headers"] = { + "{}-{}".format("llm_provider", k): v + for k, v in _response_headers.items() + } self._response_headers = _response_headers self.response_id = None self.logging_loop = None From 2dcd9a556776bc0747f88260deede3114d3d8140 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 23 Jul 2024 19:12:24 +0000 Subject: [PATCH 62/99] (test - azure): Add test for Azure OIDC auth. --- litellm/tests/test_embedding.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/litellm/tests/test_embedding.py b/litellm/tests/test_embedding.py index 39a9e7f398e..940f10e8818 100644 --- a/litellm/tests/test_embedding.py +++ b/litellm/tests/test_embedding.py @@ -196,6 +196,28 @@ def test_openai_azure_embedding(): except Exception as e: pytest.fail(f"Error occurred: {e}") +@pytest.mark.skipif( + os.environ.get("CIRCLE_OIDC_TOKEN") is None, + reason="Cannot run without being in CircleCI Runner", +) +def test_openai_azure_embedding_with_oidc_and_cf(): + # TODO: Switch to our own Azure account, currently using ai.moda's account + os.environ["AZURE_TENANT_ID"] = "17c0a27a-1246-4aa1-a3b6-d294e80e783c" + os.environ["AZURE_CLIENT_ID"] = "4faf5422-b2bd-45e8-a6d7-46543a38acd0" + + try: + response = embedding( + model="azure/text-embedding-ada-002", + input=["Hello"], + azure_ad_token="oidc/circleci/", + api_base="https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/eastus2-litellm", + api_version="2024-06-01", + ) + print(response) + + except Exception as e: + pytest.fail(f"Error occurred: {e}") + def test_openai_azure_embedding_optional_arg(mocker): mocked_create_embeddings = mocker.patch.object( From c3d90f9aee9700fa69c3bd119dae465e8364b0ae Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 15:23:58 -0700 Subject: [PATCH 63/99] test_anthropic_completion_input_translation_with_metadata --- litellm/tests/test_anthropic_completion.py | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/litellm/tests/test_anthropic_completion.py b/litellm/tests/test_anthropic_completion.py index cac0945d8da..15d150a56df 100644 --- a/litellm/tests/test_anthropic_completion.py +++ b/litellm/tests/test_anthropic_completion.py @@ -48,6 +48,42 @@ def test_anthropic_completion_input_translation(): ] +def test_anthropic_completion_input_translation_with_metadata(): + """ + Tests that cost tracking works as expected with LiteLLM Proxy + + LiteLLM Proxy will insert litellm_metadata for anthropic endpoints to track user_api_key and user_api_key_team_id + + This test ensures that the `litellm_metadata` is not present in the translated input + It ensures that `litellm.acompletion()` will receieve metadata which is a litellm specific param + """ + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hey, how's it going?"}], + "litellm_metadata": { + "user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b", + "user_api_key_alias": None, + "user_api_end_user_max_budget": None, + "litellm_api_version": "1.40.19", + "global_max_parallel_requests": None, + "user_api_key_user_id": "default_user_id", + "user_api_key_org_id": None, + "user_api_key_team_id": None, + "user_api_key_team_alias": None, + "user_api_key_team_max_budget": None, + "user_api_key_team_spend": None, + "user_api_key_spend": 0.0, + "user_api_key_max_budget": None, + "user_api_key_metadata": {}, + }, + } + translated_input = anthropic_adapter.translate_completion_input_params(kwargs=data) + + assert "litellm_metadata" not in translated_input + assert "metadata" in translated_input + assert translated_input["metadata"] == data["litellm_metadata"] + + def test_anthropic_completion_e2e(): litellm.set_verbose = True From 4c1ee1e282bb65a70a6b43a43d207fd3c955b05a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 15:25:46 -0700 Subject: [PATCH 64/99] fix add better debugging _PROXY_track_cost_callback --- litellm/proxy/proxy_server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0ac1d82e073..106b95453b6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -657,7 +657,11 @@ async def _PROXY_track_cost_callback( global prisma_client, custom_db_client try: # check if it has collected an entire stream response - verbose_proxy_logger.debug("Proxy: In track_cost_callback for: %s", kwargs) + verbose_proxy_logger.debug( + "Proxy: In track_cost_callback for: kwargs=%s and completion_response: %s", + kwargs, + completion_response, + ) verbose_proxy_logger.debug( f"kwargs stream: {kwargs.get('stream', None)} + complete streaming response: {kwargs.get('complete_streaming_response', None)}" ) From a71b60d005292d6d4cdcdf2f5ba26177c3f76acd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 15:31:30 -0700 Subject: [PATCH 65/99] Pass litellm proxy specific metadata --- litellm/llms/anthropic.py | 5 +++++ litellm/proxy/litellm_pre_call_utils.py | 3 +++ litellm/types/llms/anthropic.py | 5 ++++- litellm/types/llms/openai.py | 1 + 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index da51e887d7a..629197d51f5 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -385,6 +385,11 @@ class AnthropicConfig: if "user_id" in anthropic_message_request["metadata"]: new_kwargs["user"] = anthropic_message_request["metadata"]["user_id"] + # Pass litellm proxy specific metadata + if "litellm_metadata" in anthropic_message_request: + # metadata will be passed to litellm.acompletion(), it's a litellm_param + new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata") + ## CONVERT TOOL CHOICE if "tool_choice" in anthropic_message_request: new_kwargs["tool_choice"] = self.translate_anthropic_tool_choice_to_openai( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8909b1da3d0..7384dc30be5 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -39,6 +39,9 @@ def _get_metadata_variable_name(request: Request) -> str: """ if "thread" in request.url.path or "assistant" in request.url.path: return "litellm_metadata" + if "/v1/messages" in request.url.path: + # anthropic API has a field called metadata + return "litellm_metadata" else: return "metadata" diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 33f413eced7..b41980afddc 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -1,4 +1,4 @@ -from typing import Iterable, List, Optional, Union +from typing import Any, Dict, Iterable, List, Optional, Union from pydantic import BaseModel, validator from typing_extensions import Literal, Required, TypedDict @@ -113,6 +113,9 @@ class AnthropicMessagesRequest(TypedDict, total=False): top_k: int top_p: float + # litellm param - used for tracking litellm proxy metadata in the request + litellm_metadata: dict + class ContentTextBlockDelta(TypedDict): """ diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 294e299dbf4..35e442119de 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -436,6 +436,7 @@ class ChatCompletionRequest(TypedDict, total=False): function_call: Union[str, dict] functions: List user: str + metadata: dict # litellm specific param class ChatCompletionDeltaChunk(TypedDict, total=False): From 169da8b8d0ee00d04043bfb05c1430a1f313dce1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 15:39:21 -0700 Subject: [PATCH 66/99] docs(guardrails.md): add team-based controls to guardrails --- docs/my-website/docs/proxy/guardrails.md | 48 ++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/my-website/docs/proxy/guardrails.md b/docs/my-website/docs/proxy/guardrails.md index 053fa8cab0a..2cfa3980e7c 100644 --- a/docs/my-website/docs/proxy/guardrails.md +++ b/docs/my-website/docs/proxy/guardrails.md @@ -266,6 +266,54 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` +## Disable team from turning on/off guardrails + + +### 1. Disable team from modifying guardrails + +```bash +curl -X POST 'http://0.0.0.0:4000/team/update' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-D '{ + "team_id": "4198d93c-d375-4c83-8d5a-71e7c5473e50", + "metadata": {"guardrails": {"modify_guardrails": false}} +}' +``` + +### 2. Try to disable guardrails for a call + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \ +--data '{ +"model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "Think of 10 random colors." + } + ], + "metadata": {"guardrails": {"hide_secrets": false}} +}' +``` + +### 3. Get 403 Error + +``` +{ + "error": { + "message": { + "error": "Your team does not have permission to modify guardrails." + }, + "type": "auth_error", + "param": "None", + "code": 403 + } +} +``` + Expect to NOT see `+1 412-612-9992` in your server logs on your callback. :::info From e14ef3eeda8d921ef3feaf3fae7bfe820ec69169 Mon Sep 17 00:00:00 2001 From: Wanis Elabbar <70503629+elabbarw@users.noreply.github.com> Date: Tue, 23 Jul 2024 23:57:50 +0100 Subject: [PATCH 67/99] feat - add azure_ai llama v3.1 8B 70B and 405B --- model_prices_and_context_window.json | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f86ea8bd759..4c6dd8fdbe5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -760,6 +760,33 @@ "litellm_provider": "azure_ai", "mode": "chat" }, + "azure_ai/Meta-Llama-31-8B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.00000061, + "litellm_provider": "azure_ai", + "mode": "chat" + }, + "azure_ai/Meta-Llama-31-70B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00000268, + "output_cost_per_token": 0.00000354, + "litellm_provider": "azure_ai", + "mode": "chat" + }, + "azure_ai/Meta-Llama-31-405B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00000533, + "output_cost_per_token": 0.000016, + "litellm_provider": "azure_ai", + "mode": "chat" + }, "babbage-002": { "max_tokens": 16384, "max_input_tokens": 16384, From 78eb5164df7d02c3369673c93afb4016523ce5c2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 16:33:04 -0700 Subject: [PATCH 68/99] fix DB accept null values for api_base, user, etc --- litellm/proxy/schema.prisma | 14 +++++++------- schema.prisma | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 528d7e98df8..cf61635a0bd 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -183,12 +183,12 @@ model LiteLLM_SpendLogs { model String @default("") model_id String? @default("") // the model id stored in proxy model db model_group String? @default("") // public model_name / model_group - api_base String @default("") - user String @default("") - metadata Json @default("{}") - cache_hit String @default("") - cache_key String @default("") - request_tags Json @default("[]") + api_base String? @default("") + user String? @default("") + metadata Json? @default("{}") + cache_hit String? @default("") + cache_key String? @default("") + request_tags Json? @default("[]") team_id String? end_user String? requester_ip_address String? @@ -257,4 +257,4 @@ model LiteLLM_AuditLog { object_id String // id of the object being audited. This can be the key id, team id, user id, model id before_value Json? // value of the row updated_values Json? // value of the row after change -} \ No newline at end of file +} diff --git a/schema.prisma b/schema.prisma index 970a1197e68..8f412510410 100644 --- a/schema.prisma +++ b/schema.prisma @@ -172,7 +172,7 @@ model LiteLLM_Config { model LiteLLM_SpendLogs { request_id String @id call_type String - api_key String @default ("") + api_key String @default ("") // Hashed API Token. Not the actual Virtual Key. Equivalent to 'token' column in LiteLLM_VerificationToken spend Float @default(0.0) total_tokens Int @default(0) prompt_tokens Int @default(0) @@ -183,12 +183,12 @@ model LiteLLM_SpendLogs { model String @default("") model_id String? @default("") // the model id stored in proxy model db model_group String? @default("") // public model_name / model_group - api_base String @default("") - user String @default("") - metadata Json @default("{}") - cache_hit String @default("") - cache_key String @default("") - request_tags Json @default("[]") + api_base String? @default("") + user String? @default("") + metadata Json? @default("{}") + cache_hit String? @default("") + cache_key String? @default("") + request_tags Json? @default("[]") team_id String? end_user String? requester_ip_address String? From 83b13d34baf7d7423e67f22a4f024fa528323cdb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 16:48:50 -0700 Subject: [PATCH 69/99] =?UTF-8?q?bump:=20version=201.41.27=20=E2=86=92=201?= =?UTF-8?q?.41.28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5dc8ab62d32..8a2168d2b9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.41.27" +version = "1.41.28" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.41.27" +version = "1.41.28" version_files = [ "pyproject.toml:^version" ] From 83ef52e18005db1e0b6ee9756c1edebf5820887e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 17:07:30 -0700 Subject: [PATCH 70/99] feat(vertex_ai_llama.py): vertex ai llama3.1 api support Initial working commit for vertex ai llama 3.1 api support --- litellm/llms/vertex_ai_llama.py | 270 ++++++++++++++++++ litellm/llms/vertex_httpx.py | 2 +- litellm/main.py | 50 ++-- .../tests/test_amazing_vertex_completion.py | 46 +++ litellm/utils.py | 6 +- 5 files changed, 355 insertions(+), 19 deletions(-) create mode 100644 litellm/llms/vertex_ai_llama.py diff --git a/litellm/llms/vertex_ai_llama.py b/litellm/llms/vertex_ai_llama.py new file mode 100644 index 00000000000..4b5407faaa5 --- /dev/null +++ b/litellm/llms/vertex_ai_llama.py @@ -0,0 +1,270 @@ +# What is this? +## Handler for calling llama 3.1 API on Vertex AI +import copy +import json +import os +import time +import types +import uuid +from enum import Enum +from typing import Any, Callable, List, Optional, Tuple, Union + +import httpx # type: ignore +import requests # type: ignore + +import litellm +from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.llms.anthropic import ( + AnthropicMessagesTool, + AnthropicMessagesToolChoice, +) +from litellm.types.llms.openai import ( + ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, +) +from litellm.types.utils import ResponseFormatChunk +from litellm.utils import CustomStreamWrapper, ModelResponse, Usage + +from .base import BaseLLM +from .prompt_templates.factory import ( + construct_tool_use_system_prompt, + contains_tag, + custom_prompt, + extract_between_tags, + parse_xml_params, + prompt_factory, + response_schema_prompt, +) + + +class VertexAIError(Exception): + def __init__(self, status_code, message): + self.status_code = status_code + self.message = message + self.request = httpx.Request( + method="POST", url=" https://cloud.google.com/vertex-ai/" + ) + self.response = httpx.Response(status_code=status_code, request=self.request) + super().__init__( + self.message + ) # Call the base class constructor with the parameters it needs + + +class VertexAILlama3Config: + """ + Reference:https://docs.anthropic.com/claude/reference/messages_post + + Note that the API for Claude on Vertex differs from the Anthropic API documentation in the following ways: + + - `model` is not a valid parameter. The model is instead specified in the Google Cloud endpoint URL. + - `anthropic_version` is a required parameter and must be set to "vertex-2023-10-16". + + The class `VertexAIAnthropicConfig` provides configuration for the VertexAI's Anthropic API interface. Below are the parameters: + + - `max_tokens` Required (integer) max tokens, + - `anthropic_version` Required (string) version of anthropic for bedrock - e.g. "bedrock-2023-05-31" + - `system` Optional (string) the system prompt, conversion from openai format to this is handled in factory.py + - `temperature` Optional (float) The amount of randomness injected into the response + - `top_p` Optional (float) Use nucleus sampling. + - `top_k` Optional (int) Only sample from the top K options for each subsequent token + - `stop_sequences` Optional (List[str]) Custom text sequences that cause the model to stop generating + + Note: Please make sure to modify the default parameters as required for your use case. + """ + + max_tokens: Optional[int] = ( + 4096 # anthropic max - setting this doesn't impact response, but is required by anthropic. + ) + system: Optional[str] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[int] = None + stop_sequences: Optional[List[str]] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + anthropic_version: Optional[str] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key == "max_tokens" and value is None: + value = self.max_tokens + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self): + return [ + "max_tokens", + "tools", + "tool_choice", + "stream", + "stop", + "temperature", + "top_p", + "response_format", + ] + + def map_openai_params(self, non_default_params: dict, optional_params: dict): + for param, value in non_default_params.items(): + if param == "max_tokens": + optional_params["max_tokens"] = value + if param == "tools": + optional_params["tools"] = value + if param == "tool_choice": + _tool_choice: Optional[AnthropicMessagesToolChoice] = None + if value == "auto": + _tool_choice = {"type": "auto"} + elif value == "required": + _tool_choice = {"type": "any"} + elif isinstance(value, dict): + _tool_choice = {"type": "tool", "name": value["function"]["name"]} + + if _tool_choice is not None: + optional_params["tool_choice"] = _tool_choice + if param == "stream": + optional_params["stream"] = value + if param == "stop": + optional_params["stop_sequences"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["top_p"] = value + if param == "response_format" and "response_schema" in value: + """ + When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + - You usually want to provide a single tool + - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool + - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the modelโ€™s perspective. + """ + _tool_choice = None + _tool_choice = {"name": "json_tool_call", "type": "tool"} + + _tool = AnthropicMessagesTool( + name="json_tool_call", + input_schema={ + "type": "object", + "properties": {"values": value["response_schema"]}, # type: ignore + }, + ) + + optional_params["tools"] = [_tool] + optional_params["tool_choice"] = _tool_choice + optional_params["json_mode"] = True + + return optional_params + + +class VertexAILlama3(BaseLLM): + def __init__(self) -> None: + pass + + def create_vertex_llama3_url( + self, vertex_location: str, vertex_project: str + ) -> str: + return f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/openapi" + + def completion( + self, + model: str, + messages: list, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj, + optional_params: dict, + custom_prompt_dict: dict, + headers: Optional[dict], + timeout: Union[float, httpx.Timeout], + vertex_project=None, + vertex_location=None, + vertex_credentials=None, + litellm_params=None, + logger_fn=None, + acompletion: bool = False, + client=None, + ): + try: + import vertexai + from google.cloud import aiplatform + + from litellm.llms.openai import OpenAIChatCompletion + from litellm.llms.vertex_httpx import VertexLLM + except Exception: + + raise VertexAIError( + status_code=400, + message="""vertexai import failed please run `pip install -U "google-cloud-aiplatform>=1.38"`""", + ) + + if not ( + hasattr(vertexai, "preview") or hasattr(vertexai.preview, "language_models") + ): + raise VertexAIError( + status_code=400, + message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""", + ) + try: + + vertex_httpx_logic = VertexLLM() + + access_token, project_id = vertex_httpx_logic._ensure_access_token( + credentials=vertex_credentials, project_id=vertex_project + ) + + openai_chat_completions = OpenAIChatCompletion() + + ## Load Config + # config = litellm.VertexAILlama3.get_config() + # for k, v in config.items(): + # if k not in optional_params: + # optional_params[k] = v + + ## CONSTRUCT API BASE + stream: bool = optional_params.get("stream", False) or False + + optional_params["stream"] = stream + + api_base = self.create_vertex_llama3_url( + vertex_location=vertex_location or "us-central1", + vertex_project=vertex_project or project_id, + ) + + return openai_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + api_key=access_token, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + logging_obj=logging_obj, + optional_params=optional_params, + acompletion=acompletion, + litellm_params=litellm_params, + logger_fn=logger_fn, + client=client, + timeout=timeout, + ) + + except Exception as e: + raise VertexAIError(status_code=500, message=str(e)) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index a8de79affc8..93d8f42827c 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -1189,7 +1189,7 @@ class VertexLLM(BaseLLM): response.raise_for_status() except httpx.HTTPStatusError as err: error_code = err.response.status_code - raise VertexAIError(status_code=error_code, message=response.text) + raise VertexAIError(status_code=error_code, message=err.response.text) except httpx.TimeoutException: raise VertexAIError(status_code=408, message="Timeout error occurred.") diff --git a/litellm/main.py b/litellm/main.py index fad2e15ccd3..35fad5e029a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -120,6 +120,7 @@ from .llms.prompt_templates.factory import ( ) from .llms.text_completion_codestral import CodestralTextCompletion from .llms.triton import TritonChatCompletion +from .llms.vertex_ai_llama import VertexAILlama3 from .llms.vertex_httpx import VertexLLM from .llms.watsonx import IBMWatsonXAI from .types.llms.openai import HttpxBinaryResponseContent @@ -156,6 +157,7 @@ triton_chat_completions = TritonChatCompletion() bedrock_chat_completion = BedrockLLM() bedrock_converse_chat_completion = BedrockConverseLLM() vertex_chat_completion = VertexLLM() +vertex_llama_chat_completion = VertexAILlama3() watsonxai = IBMWatsonXAI() ####### COMPLETION ENDPOINTS ################ @@ -2064,7 +2066,26 @@ def completion( timeout=timeout, client=client, ) - + elif model.startswith("meta/"): + model_response = vertex_llama_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) else: model_response = vertex_ai.completion( model=model, @@ -2478,28 +2499,25 @@ def completion( return generator response = generator - + elif custom_llm_provider == "triton": - api_base = ( - litellm.api_base or api_base - ) + api_base = litellm.api_base or api_base model_response = triton_chat_completions.completion( - api_base=api_base, - timeout=timeout, # type: ignore - model=model, - messages=messages, - model_response=model_response, - optional_params=optional_params, - logging_obj=logging, - stream=stream, - acompletion=acompletion + api_base=api_base, + timeout=timeout, # type: ignore + model=model, + messages=messages, + model_response=model_response, + optional_params=optional_params, + logging_obj=logging, + stream=stream, + acompletion=acompletion, ) ## RESPONSE OBJECT response = model_response return response - - + elif custom_llm_provider == "cloudflare": api_key = ( api_key diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 3def5a1ec30..b9762afcbfd 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -895,6 +895,52 @@ async def test_gemini_pro_function_calling_httpx(model, sync_mode): pytest.fail("An unexpected exception occurred - {}".format(str(e))) +from litellm.tests.test_completion import response_format_tests + + +@pytest.mark.parametrize( + "model", ["vertex_ai/meta/llama3-405b-instruct-maas"] +) # "vertex_ai", +@pytest.mark.parametrize("sync_mode", [True, False]) # "vertex_ai", +@pytest.mark.asyncio +async def test_llama_3_httpx(model, sync_mode): + try: + load_vertex_ai_credentials() + litellm.set_verbose = True + + messages = [ + { + "role": "system", + "content": "Your name is Litellm Bot, you are a helpful assistant", + }, + # User asks for their name and weather in San Francisco + { + "role": "user", + "content": "Hello, what is your name and can you tell me the weather?", + }, + ] + + data = { + "model": model, + "messages": messages, + } + if sync_mode: + response = litellm.completion(**data) + else: + response = await litellm.acompletion(**data) + + response_format_tests(response=response) + + print(f"response: {response}") + except litellm.RateLimitError as e: + pass + except Exception as e: + if "429 Quota exceeded" in str(e): + pass + else: + pytest.fail("An unexpected exception occurred - {}".format(str(e))) + + def vertex_httpx_mock_reject_prompt_post(*args, **kwargs): mock_response = MagicMock() mock_response.status_code = 200 diff --git a/litellm/utils.py b/litellm/utils.py index 7f615ab61c9..8baced4c53e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5752,10 +5752,12 @@ def convert_to_model_response_object( model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) # type: ignore if "created" in response_object: - model_response_object.created = response_object["created"] + model_response_object.created = response_object["created"] or int( + time.time() + ) if "id" in response_object: - model_response_object.id = response_object["id"] + model_response_object.id = response_object["id"] or str(uuid.uuid4()) if "system_fingerprint" in response_object: model_response_object.system_fingerprint = response_object[ From 7df94100e8946b94ce74ee42dd2cce01af38b664 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 17:36:07 -0700 Subject: [PATCH 71/99] build(model_prices_and_context_window.json): add model pricing for vertex ai llama 3.1 api --- litellm/__init__.py | 2 + litellm/llms/vertex_ai_llama.py | 73 +------------------ ...odel_prices_and_context_window_backup.json | 10 +++ litellm/tests/test_optional_params.py | 13 ++++ litellm/utils.py | 12 +++ model_prices_and_context_window.json | 10 +++ 6 files changed, 50 insertions(+), 70 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 9bb9a81cd37..5eea6346c69 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -357,6 +357,7 @@ vertex_text_models: List = [] vertex_code_text_models: List = [] vertex_embedding_models: List = [] vertex_anthropic_models: List = [] +vertex_llama3_models: List = [] ai21_models: List = [] nlp_cloud_models: List = [] aleph_alpha_models: List = [] @@ -828,6 +829,7 @@ from .llms.petals import PetalsConfig from .llms.vertex_httpx import VertexGeminiConfig, GoogleAIStudioGeminiConfig from .llms.vertex_ai import VertexAIConfig, VertexAITextEmbeddingConfig from .llms.vertex_ai_anthropic import VertexAIAnthropicConfig +from .llms.vertex_ai_llama import VertexAILlama3Config from .llms.sagemaker import SagemakerConfig from .llms.ollama import OllamaConfig from .llms.ollama_chat import OllamaChatConfig diff --git a/litellm/llms/vertex_ai_llama.py b/litellm/llms/vertex_ai_llama.py index 4b5407faaa5..f33c127f742 100644 --- a/litellm/llms/vertex_ai_llama.py +++ b/litellm/llms/vertex_ai_llama.py @@ -53,39 +53,20 @@ class VertexAIError(Exception): class VertexAILlama3Config: """ - Reference:https://docs.anthropic.com/claude/reference/messages_post + Reference:https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama#streaming - Note that the API for Claude on Vertex differs from the Anthropic API documentation in the following ways: - - - `model` is not a valid parameter. The model is instead specified in the Google Cloud endpoint URL. - - `anthropic_version` is a required parameter and must be set to "vertex-2023-10-16". - - The class `VertexAIAnthropicConfig` provides configuration for the VertexAI's Anthropic API interface. Below are the parameters: + The class `VertexAILlama3Config` provides configuration for the VertexAI's Llama API interface. Below are the parameters: - `max_tokens` Required (integer) max tokens, - - `anthropic_version` Required (string) version of anthropic for bedrock - e.g. "bedrock-2023-05-31" - - `system` Optional (string) the system prompt, conversion from openai format to this is handled in factory.py - - `temperature` Optional (float) The amount of randomness injected into the response - - `top_p` Optional (float) Use nucleus sampling. - - `top_k` Optional (int) Only sample from the top K options for each subsequent token - - `stop_sequences` Optional (List[str]) Custom text sequences that cause the model to stop generating Note: Please make sure to modify the default parameters as required for your use case. """ - max_tokens: Optional[int] = ( - 4096 # anthropic max - setting this doesn't impact response, but is required by anthropic. - ) - system: Optional[str] = None - temperature: Optional[float] = None - top_p: Optional[float] = None - top_k: Optional[int] = None - stop_sequences: Optional[List[str]] = None + max_tokens: Optional[int] = None def __init__( self, max_tokens: Optional[int] = None, - anthropic_version: Optional[str] = None, ) -> None: locals_ = locals() for key, value in locals_.items(): @@ -115,61 +96,13 @@ class VertexAILlama3Config: def get_supported_openai_params(self): return [ "max_tokens", - "tools", - "tool_choice", "stream", - "stop", - "temperature", - "top_p", - "response_format", ] def map_openai_params(self, non_default_params: dict, optional_params: dict): for param, value in non_default_params.items(): if param == "max_tokens": optional_params["max_tokens"] = value - if param == "tools": - optional_params["tools"] = value - if param == "tool_choice": - _tool_choice: Optional[AnthropicMessagesToolChoice] = None - if value == "auto": - _tool_choice = {"type": "auto"} - elif value == "required": - _tool_choice = {"type": "any"} - elif isinstance(value, dict): - _tool_choice = {"type": "tool", "name": value["function"]["name"]} - - if _tool_choice is not None: - optional_params["tool_choice"] = _tool_choice - if param == "stream": - optional_params["stream"] = value - if param == "stop": - optional_params["stop_sequences"] = value - if param == "temperature": - optional_params["temperature"] = value - if param == "top_p": - optional_params["top_p"] = value - if param == "response_format" and "response_schema" in value: - """ - When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode - - You usually want to provide a single tool - - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool - - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the modelโ€™s perspective. - """ - _tool_choice = None - _tool_choice = {"name": "json_tool_call", "type": "tool"} - - _tool = AnthropicMessagesTool( - name="json_tool_call", - input_schema={ - "type": "object", - "properties": {"values": value["response_schema"]}, # type: ignore - }, - ) - - optional_params["tools"] = [_tool] - optional_params["tool_choice"] = _tool_choice - optional_params["json_mode"] = True return optional_params diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f86ea8bd759..e9e599945f8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1948,6 +1948,16 @@ "supports_function_calling": true, "supports_vision": true }, + "vertex_ai/meta/llama3-405b-instruct-maas": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "vertex_ai-llama_models", + "mode": "chat", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models" + }, "vertex_ai/imagegeneration@006": { "cost_per_image": 0.020, "litellm_provider": "vertex_ai-image-models", diff --git a/litellm/tests/test_optional_params.py b/litellm/tests/test_optional_params.py index bbfc88710f8..b8011960eca 100644 --- a/litellm/tests/test_optional_params.py +++ b/litellm/tests/test_optional_params.py @@ -128,6 +128,19 @@ def test_azure_ai_mistral_optional_params(): assert "user" not in optional_params +def test_vertex_ai_llama_3_optional_params(): + litellm.vertex_llama3_models = ["meta/llama3-405b-instruct-maas"] + litellm.drop_params = True + optional_params = get_optional_params( + model="meta/llama3-405b-instruct-maas", + user="John", + custom_llm_provider="vertex_ai", + max_tokens=10, + temperature=0.2, + ) + assert "user" not in optional_params + + def test_azure_gpt_optional_params_gpt_vision(): # for OpenAI, Azure all extra params need to get passed as extra_body to OpenAI python. We assert we actually set extra_body here optional_params = litellm.utils.get_optional_params( diff --git a/litellm/utils.py b/litellm/utils.py index 8baced4c53e..035c1c72f7a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3088,6 +3088,15 @@ def get_optional_params( non_default_params=non_default_params, optional_params=optional_params, ) + elif custom_llm_provider == "vertex_ai" and model in litellm.vertex_llama3_models: + supported_params = get_supported_openai_params( + model=model, custom_llm_provider=custom_llm_provider + ) + _check_valid_arg(supported_params=supported_params) + optional_params = litellm.VertexAILlama3Config().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + ) elif custom_llm_provider == "sagemaker": ## check if unsupported param passed in supported_params = get_supported_openai_params( @@ -4189,6 +4198,9 @@ def get_supported_openai_params( return litellm.GoogleAIStudioGeminiConfig().get_supported_openai_params() elif custom_llm_provider == "vertex_ai": if request_type == "chat_completion": + if model.startswith("meta/"): + return litellm.VertexAILlama3Config().get_supported_openai_params() + return litellm.VertexAIConfig().get_supported_openai_params() elif request_type == "embeddings": return litellm.VertexAITextEmbeddingConfig().get_supported_openai_params() diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f86ea8bd759..e9e599945f8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1948,6 +1948,16 @@ "supports_function_calling": true, "supports_vision": true }, + "vertex_ai/meta/llama3-405b-instruct-maas": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "vertex_ai-llama_models", + "mode": "chat", + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models" + }, "vertex_ai/imagegeneration@006": { "cost_per_image": 0.020, "litellm_provider": "vertex_ai-image-models", From ae693424e4890dad4eca1cb1565f1e612d426468 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 17:55:28 -0700 Subject: [PATCH 72/99] fix(__init__.py): update init --- litellm/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 5eea6346c69..5a10ae77c11 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -400,6 +400,9 @@ for key, value in model_cost.items(): elif value.get("litellm_provider") == "vertex_ai-anthropic_models": key = key.replace("vertex_ai/", "") vertex_anthropic_models.append(key) + elif value.get("litellm_provider") == "vertex_ai-llama_models": + key = key.replace("vertex_ai/", "") + vertex_llama3_models.append(key) elif value.get("litellm_provider") == "ai21": ai21_models.append(key) elif value.get("litellm_provider") == "nlp_cloud": From fb0a13c8bb2099a479146291a3b3f1a591dffeb6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 21:44:24 -0700 Subject: [PATCH 73/99] fix(anthropic.py): support openai system message being a list --- litellm/llms/anthropic.py | 13 +++++++++++-- litellm/proxy/_new_secret_config.yaml | 5 ++--- litellm/tests/test_completion.py | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 629197d51f5..d3a3c38a483 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -780,8 +780,17 @@ class AnthropicChatCompletion(BaseLLM): system_prompt = "" for idx, message in enumerate(messages): if message["role"] == "system": - system_prompt += message["content"] - system_prompt_indices.append(idx) + valid_content: bool = False + if isinstance(message["content"], str): + system_prompt += message["content"] + valid_content = True + elif isinstance(message["content"], list): + for content in message["content"]: + system_prompt += content.get("text", "") + valid_content = True + + if valid_content: + system_prompt_indices.append(idx) if len(system_prompt_indices) > 0: for idx in reversed(system_prompt_indices): messages.pop(idx) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index a1af38379ad..7e3c9a241ab 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,8 +1,7 @@ model_list: - - model_name: groq-llama3 + - model_name: anthropic-claude litellm_params: - model: groq/llama3-groq-70b-8192-tool-use-preview - api_key: os.environ/GROQ_API_KEY + model: claude-3-haiku-20240307 litellm_settings: callbacks: ["logfire"] diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index c2ce836efed..31b7b8355d6 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -346,7 +346,7 @@ def test_completion_claude_3_empty_response(): messages = [ { "role": "system", - "content": "You are 2twNLGfqk4GMOn3ffp4p.", + "content": [{"type": "text", "text": "You are 2twNLGfqk4GMOn3ffp4p."}], }, {"role": "user", "content": "Hi gm!", "name": "ishaan"}, {"role": "assistant", "content": "Good morning! How are you doing today?"}, From d5d2ffffdfb64a6b8fdeaee04bed6fea493fb587 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 23 Jul 2024 21:54:06 -0700 Subject: [PATCH 74/99] =?UTF-8?q?bump:=20version=201.41.28=20=E2=86=92=201?= =?UTF-8?q?.42.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/my-website/docs/providers/vertex.md | 79 ++++++++++++++++++++++++ pyproject.toml | 4 +- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 19442e11bc5..f8759704603 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -749,6 +749,85 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ + +## Llama 3 API + +| Model Name | Function Call | +|------------------|--------------------------------------| +| meta/llama3-405b-instruct-maas | `completion('vertex_ai/meta/llama3-405b-instruct-maas', messages)` | + +### Usage + + + + +```python +from litellm import completion +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" + +model = "meta/llama3-405b-instruct-maas" + +vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] +vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] + +response = completion( + model="vertex_ai/" + model, + messages=[{"role": "user", "content": "hi"}], + temperature=0.7, + vertex_ai_project=vertex_ai_project, + vertex_ai_location=vertex_ai_location, +) +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: anthropic-llama + litellm_params: + model: vertex_ai/meta/llama3-405b-instruct-maas + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-east-1" + - model_name: anthropic-llama + litellm_params: + model: vertex_ai/meta/llama3-405b-instruct-maas + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-west-1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "anthropic-llama", # ๐Ÿ‘ˆ the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + ## Model Garden | Model Name | Function Call | |------------------|--------------------------------------| diff --git a/pyproject.toml b/pyproject.toml index 8a2168d2b9a..10246abd755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.41.28" +version = "1.42.0" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.41.28" +version = "1.42.0" version_files = [ "pyproject.toml:^version" ] From 642f1a7bcced8ad7ff8b532833487b6a9de35de8 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Tue, 23 Jul 2024 21:45:01 -0700 Subject: [PATCH 75/99] Check existence of multiple views in 1 query instead of multiple queries. This is more efficient because it lets us check for all views in one query instead of multiple queries. --- litellm/proxy/utils.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index df3b68593cc..b08d7a30f1a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -844,6 +844,30 @@ class PrismaClient: If the view doesn't exist, one will be created. """ + + # Check to see if all of the necessary views exist and if they do, simply return + # This is more efficient because it lets us check for all views in one + # query instead of multiple queries. + try: + ret = await self.db.query_raw( + """ + SELECT SUM(1) FROM pg_views + WHERE schemaname = 'public' AND viewname IN ( + 'LiteLLM_VerificationTokenView', + 'MonthlyGlobalSpend', + 'Last30dKeysBySpend', + 'Last30dModelsBySpend', + 'MonthlyGlobalSpendPerKey', + 'Last30dTopEndUsersSpend' + ) + """ + ) + if ret[0]['sum'] == 6: + print("All necessary views exist!") # noqa + return + except Exception: + pass + try: # Try to select one row from the view await self.db.query_raw( From 609075bd1727c7c0dce7aefc4947f0321eb4f712 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 24 Jul 2024 05:29:27 +0000 Subject: [PATCH 76/99] Add Llama 3.1 for Bedrock. --- litellm/llms/bedrock_httpx.py | 2 ++ ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py index c3a563ce4de..d6c45fb9c9d 100644 --- a/litellm/llms/bedrock_httpx.py +++ b/litellm/llms/bedrock_httpx.py @@ -76,6 +76,8 @@ BEDROCK_CONVERSE_MODELS = [ "anthropic.claude-v1", "anthropic.claude-instant-v1", "ai21.jamba-instruct-v1:0", + "meta.llama3-1-8b-instruct-v1:0", + "meta.llama3-1-70b-instruct-v1:0", ] diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e9e599945f8..ef7caf4a754 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3643,6 +3643,24 @@ "litellm_provider": "bedrock", "mode": "chat" }, + "meta.llama3-1-8b-instruct-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 2048, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.0000006, + "litellm_provider": "bedrock", + "mode": "chat" + }, + "meta.llama3-1-70b-instruct-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 2048, + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "litellm_provider": "bedrock", + "mode": "chat" + }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "max_tokens": 77, "max_input_tokens": 77, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e9e599945f8..ef7caf4a754 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3643,6 +3643,24 @@ "litellm_provider": "bedrock", "mode": "chat" }, + "meta.llama3-1-8b-instruct-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 2048, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.0000006, + "litellm_provider": "bedrock", + "mode": "chat" + }, + "meta.llama3-1-70b-instruct-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 2048, + "input_cost_per_token": 0.00000265, + "output_cost_per_token": 0.0000035, + "litellm_provider": "bedrock", + "mode": "chat" + }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "max_tokens": 77, "max_input_tokens": 77, From adfd6ab900de537dda814c6573a0cd6102d0c19e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jul 2024 07:08:40 -0700 Subject: [PATCH 77/99] langsmith - support logging tags --- litellm/integrations/langsmith.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 81db798ae8b..12ed1ac7081 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -79,6 +79,7 @@ class LangsmithLogger(CustomLogger): project_name = metadata.get("project_name", self.langsmith_project) run_name = metadata.get("run_name", self.langsmith_default_run_name) run_id = metadata.get("id", None) + tags = metadata.get("tags", []) or [] verbose_logger.debug( f"Langsmith Logging - project_name: {project_name}, run_name {run_name}" ) @@ -122,6 +123,7 @@ class LangsmithLogger(CustomLogger): "session_name": project_name, "start_time": start_time, "end_time": end_time, + "tags": tags, } if run_id: From e378ab8bc9e7f7e57358c44e80a01f58e87b8761 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jul 2024 07:12:36 -0700 Subject: [PATCH 78/99] docs - logging langsmith tags --- docs/my-website/docs/observability/langsmith_integration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index d57a64f0952..f02cdb65224 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -56,7 +56,7 @@ response = litellm.completion( ``` ## Advanced -### Set Custom Project & Run names +### Set Langsmith fields - Custom Projec, Run names, tags ```python import litellm @@ -77,6 +77,7 @@ response = litellm.completion( metadata={ "run_name": "litellmRUN", # langsmith run name "project_name": "litellm-completion", # langsmith project name + "tags": ["model1", "prod-2"] # tags to log on langsmith } ) print(response) From cea8fcc3fcfcc9dfbb2f8ca4d849cf48074c8642 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jul 2024 07:12:45 -0700 Subject: [PATCH 79/99] test - logging langsmith tags --- litellm/tests/test_langsmith.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/tests/test_langsmith.py b/litellm/tests/test_langsmith.py index 7c690212e93..68182e73d52 100644 --- a/litellm/tests/test_langsmith.py +++ b/litellm/tests/test_langsmith.py @@ -36,6 +36,7 @@ async def test_async_langsmith_logging(): temperature=0.2, metadata={ "id": run_id, + "tags": ["tag1", "tag2"], "user_api_key": "6eb81e014497d89f3cc1aa9da7c2b37bda6b7fea68e4b710d33d94201e68970c", "user_api_key_alias": "ishaans-langmsith-key", "user_api_end_user_max_budget": None, From 41fda47587a497bcfb23756cf6fbfb8b35c96f65 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Jul 2024 08:04:27 -0700 Subject: [PATCH 80/99] test(test_embedding.py): fix base url --- litellm/tests/test_embedding.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_embedding.py b/litellm/tests/test_embedding.py index 940f10e8818..a5a2adb7579 100644 --- a/litellm/tests/test_embedding.py +++ b/litellm/tests/test_embedding.py @@ -196,6 +196,7 @@ def test_openai_azure_embedding(): except Exception as e: pytest.fail(f"Error occurred: {e}") + @pytest.mark.skipif( os.environ.get("CIRCLE_OIDC_TOKEN") is None, reason="Cannot run without being in CircleCI Runner", @@ -210,7 +211,7 @@ def test_openai_azure_embedding_with_oidc_and_cf(): model="azure/text-embedding-ada-002", input=["Hello"], azure_ad_token="oidc/circleci/", - api_base="https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/eastus2-litellm", + api_base="https://eastus2-litellm.openai.azure.com/", api_version="2024-06-01", ) print(response) From af3a900b4db6d8da3ff8f0c9ce2d20ef8ee3392f Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 24 Jul 2024 15:40:51 +0000 Subject: [PATCH 81/99] (tests) - Try azure AD auth directly. --- litellm/tests/test_secret_manager.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/litellm/tests/test_secret_manager.py b/litellm/tests/test_secret_manager.py index d18d0ea63d5..e7290a77d68 100644 --- a/litellm/tests/test_secret_manager.py +++ b/litellm/tests/test_secret_manager.py @@ -12,6 +12,7 @@ sys.path.insert( import pytest from litellm import get_secret from litellm.proxy.secret_managers.aws_secret_manager import load_aws_secret_manager +from litellm.llms.azure import get_azure_ad_token_from_oidc @pytest.mark.skip(reason="AWS Suspended Account") @@ -76,3 +77,16 @@ def test_oidc_circleci_v2(): ) print(f"secret_val: {redact_oidc_signature(secret_val)}") + + +@pytest.mark.skipif( + os.environ.get("CIRCLE_OIDC_TOKEN") is None, + reason="Cannot run without being in CircleCI Runner", +) +def test_oidc_circleci_with_azure(): + # TODO: Switch to our own Azure account, currently using ai.moda's account + os.environ["AZURE_TENANT_ID"] = "17c0a27a-1246-4aa1-a3b6-d294e80e783c" + os.environ["AZURE_CLIENT_ID"] = "4faf5422-b2bd-45e8-a6d7-46543a38acd0" + azure_ad_token = get_azure_ad_token_from_oidc("oidc/circleci/") + + print(f"secret_val: {redact_oidc_signature(azure_ad_token)}") From 4b89397136db8ceee828bbff433211f6e5cadca3 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 24 Jul 2024 15:42:57 +0000 Subject: [PATCH 82/99] (tests) - Skip embedding Azure AD test for now. --- litellm/tests/test_embedding.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/litellm/tests/test_embedding.py b/litellm/tests/test_embedding.py index a5a2adb7579..fb707ad5e3e 100644 --- a/litellm/tests/test_embedding.py +++ b/litellm/tests/test_embedding.py @@ -197,10 +197,11 @@ def test_openai_azure_embedding(): pytest.fail(f"Error occurred: {e}") -@pytest.mark.skipif( - os.environ.get("CIRCLE_OIDC_TOKEN") is None, - reason="Cannot run without being in CircleCI Runner", -) +# @pytest.mark.skipif( +# os.environ.get("CIRCLE_OIDC_TOKEN") is None, +# reason="Cannot run without being in CircleCI Runner", +# ) +@pytest.mark.skip(reason="Temporarily skipping this test.") def test_openai_azure_embedding_with_oidc_and_cf(): # TODO: Switch to our own Azure account, currently using ai.moda's account os.environ["AZURE_TENANT_ID"] = "17c0a27a-1246-4aa1-a3b6-d294e80e783c" From 77cf1fd600b5213422e807de277e6599a03a45dd Mon Sep 17 00:00:00 2001 From: Wanis Elabbar <70503629+elabbarw@users.noreply.github.com> Date: Wed, 24 Jul 2024 16:50:07 +0100 Subject: [PATCH 83/99] update azure_ai llamav31 prices with sources --- model_prices_and_context_window.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e8ca6f74d9e..90a6b1283c1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -767,7 +767,8 @@ "input_cost_per_token": 0.0000003, "output_cost_per_token": 0.00000061, "litellm_provider": "azure_ai", - "mode": "chat" + "mode": "chat", + "source":"https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice" }, "azure_ai/Meta-Llama-31-70B-Instruct": { "max_tokens": 128000, @@ -776,7 +777,8 @@ "input_cost_per_token": 0.00000268, "output_cost_per_token": 0.00000354, "litellm_provider": "azure_ai", - "mode": "chat" + "mode": "chat", + "source":"https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice" }, "azure_ai/Meta-Llama-31-405B-Instruct": { "max_tokens": 128000, @@ -785,7 +787,8 @@ "input_cost_per_token": 0.00000533, "output_cost_per_token": 0.000016, "litellm_provider": "azure_ai", - "mode": "chat" + "mode": "chat", + "source":"https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice" }, "babbage-002": { "max_tokens": 16384, From c364a3129e9cc4fcb45defe7b913cd51a1b64a8a Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 24 Jul 2024 16:05:48 +0000 Subject: [PATCH 84/99] (test_secret_manager.py) - Improve and add CircleCI v1 test with Amazon. --- litellm/tests/test_secret_manager.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_secret_manager.py b/litellm/tests/test_secret_manager.py index e7290a77d68..cd2f2731fc1 100644 --- a/litellm/tests/test_secret_manager.py +++ b/litellm/tests/test_secret_manager.py @@ -13,6 +13,7 @@ import pytest from litellm import get_secret from litellm.proxy.secret_managers.aws_secret_manager import load_aws_secret_manager from litellm.llms.azure import get_azure_ad_token_from_oidc +from litellm.llms.bedrock_httpx import BedrockLLM @pytest.mark.skip(reason="AWS Suspended Account") @@ -61,7 +62,7 @@ def test_oidc_github(): ) def test_oidc_circleci(): secret_val = get_secret( - "oidc/circleci/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke" + "oidc/circleci/" ) print(f"secret_val: {redact_oidc_signature(secret_val)}") @@ -90,3 +91,25 @@ def test_oidc_circleci_with_azure(): azure_ad_token = get_azure_ad_token_from_oidc("oidc/circleci/") print(f"secret_val: {redact_oidc_signature(azure_ad_token)}") + + +@pytest.mark.skipif( + os.environ.get("CIRCLE_OIDC_TOKEN") is None, + reason="Cannot run without being in CircleCI Runner", +) +def test_oidc_circle_v1_with_amazon(): + # The purpose of this test is to get logs using the older v1 of the CircleCI OIDC token + + # TODO: This is using ai.moda's IAM role, we should use LiteLLM's IAM role eventually + aws_role_name = ( + "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci-v1-assume-only" + ) + aws_web_identity_token = "oidc/circleci/" + + bllm = BedrockLLM() + creds = bllm.get_credentials( + aws_region_name="ca-west-1", + aws_web_identity_token=aws_web_identity_token, + aws_role_name=aws_role_name, + aws_session_name="assume-v1-session", + ) From a50fe3e1caf69d528d851d69a83c6e5e3845ce3e Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Wed, 24 Jul 2024 09:12:40 -0700 Subject: [PATCH 85/99] Fix test_prompt_factory flake8 warning ```shell $ poetry run flake8 litellm/tests/test_prompt_factory.py :215: SyntaxWarning: invalid escape sequence '\/' litellm/tests/test_prompt_factory.py:215:21: W605 invalid escape sequence '\/' ``` Fixed by making the string a raw string, which is equivalent: ``` $ python Python 3.11.6 (main, Oct 25 2023, 19:49:20) [Clang 14.0.0 (clang-1400.0.29.202)] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> "data:image\/jpeg;base64,1234" == r"data:image\/jpeg;base64,1234" True ``` See: https://stackoverflow.com/questions/52335970/how-to-fix-syntaxwarning-invalid-escape-sequence-in-python --- litellm/tests/test_prompt_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_prompt_factory.py b/litellm/tests/test_prompt_factory.py index 3ed80f6ff3d..0dc0dbca20b 100644 --- a/litellm/tests/test_prompt_factory.py +++ b/litellm/tests/test_prompt_factory.py @@ -212,7 +212,7 @@ def test_convert_url_to_img(): [ ("data:image/jpeg;base64,1234", "image/jpeg"), ("data:application/pdf;base64,1234", "application/pdf"), - ("data:image\/jpeg;base64,1234", "image/jpeg"), + (r"data:image\/jpeg;base64,1234", "image/jpeg"), ], ) def test_base64_image_input(url, expected_media_type): From ced03d9d7f197e036312731784785711592dbd36 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 24 Jul 2024 16:41:24 +0000 Subject: [PATCH 86/99] (test_embedding.py) - Re-enable embedding test with Azure OIDC. --- litellm/tests/test_embedding.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/litellm/tests/test_embedding.py b/litellm/tests/test_embedding.py index fb707ad5e3e..a5a2adb7579 100644 --- a/litellm/tests/test_embedding.py +++ b/litellm/tests/test_embedding.py @@ -197,11 +197,10 @@ def test_openai_azure_embedding(): pytest.fail(f"Error occurred: {e}") -# @pytest.mark.skipif( -# os.environ.get("CIRCLE_OIDC_TOKEN") is None, -# reason="Cannot run without being in CircleCI Runner", -# ) -@pytest.mark.skip(reason="Temporarily skipping this test.") +@pytest.mark.skipif( + os.environ.get("CIRCLE_OIDC_TOKEN") is None, + reason="Cannot run without being in CircleCI Runner", +) def test_openai_azure_embedding_with_oidc_and_cf(): # TODO: Switch to our own Azure account, currently using ai.moda's account os.environ["AZURE_TENANT_ID"] = "17c0a27a-1246-4aa1-a3b6-d294e80e783c" From d9539e518e2d4d82ea2b6ac737de19147790e5ea Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Jul 2024 10:08:25 -0700 Subject: [PATCH 87/99] build(docker-compose.yml): add prometheus scraper to docker compose persists prometheus data across restarts --- docker-compose.yml | 20 ++++++++++++-- litellm/tests/test_completion.py | 47 ++++++++++++++++++++++++++++++++ prometheus.yml | 7 +++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 prometheus.yml diff --git a/docker-compose.yml b/docker-compose.yml index be84462ef00..6991bf7eba0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,6 @@ services: ######################################### ## Uncomment these lines to start proxy with a config.yaml file ## # volumes: - # - ./proxy_server_config.yaml:/app/config.yaml - # command: [ "--config", "./config.yaml", "--port", "4000"] ############################################### ports: - "4000:4000" # Map the container port to the host, change the host port if necessary @@ -33,5 +31,23 @@ services: interval: 1s timeout: 5s retries: 10 + + prometheus: + image: prom/prometheus + volumes: + - prometheus_data:/prometheus + - ./prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=15d' + restart: always + +volumes: + prometheus_data: + driver: local + # ...rest of your docker-compose config if any diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 31b7b8355d6..dae5e7f805d 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2560,6 +2560,53 @@ def test_completion_anyscale_with_functions(): # test_completion_anyscale_with_functions() +def test_completion_azure_extra_headers(): + # this tests if we can pass api_key to completion, when it's not in the env. + # DO NOT REMOVE THIS TEST. No MATTER WHAT Happens! + # If you want to remove it, speak to Ishaan! + # Ishaan will be very disappointed if this test is removed -> this is a standard way to pass api_key + the router + proxy use this + from httpx import Client + from openai import AzureOpenAI + + from litellm.llms.custom_httpx.httpx_handler import HTTPHandler + + http_client = Client() + + with patch.object(http_client, "send", new=MagicMock()) as mock_client: + client = AzureOpenAI( + azure_endpoint=os.getenv("AZURE_API_BASE"), + api_version=litellm.AZURE_DEFAULT_API_VERSION, + api_key=os.getenv("AZURE_API_KEY"), + http_client=http_client, + ) + try: + response = completion( + model="azure/chatgpt-v-2", + messages=messages, + client=client, + extra_headers={ + "Authorization": "my-bad-key", + "Ocp-Apim-Subscription-Key": "hello-world-testing", + "api-key": "my-bad-key", + }, + ) + print(response) + pytest.fail("Expected this to fail") + except Exception as e: + pass + + mock_client.assert_called() + + print(f"mock_client.call_args: {mock_client.call_args}") + request = mock_client.call_args[0][0] + print(request.method) # This will print 'POST' + print(request.url) # This will print the full URL + print(request.headers) # This will print the full URL + auth_header = request.headers.get("Authorization") + print(auth_header) + assert auth_header == "my-bad-key" + + def test_completion_azure_key_completion_arg(): # this tests if we can pass api_key to completion, when it's not in the env. # DO NOT REMOVE THIS TEST. No MATTER WHAT Happens! diff --git a/prometheus.yml b/prometheus.yml new file mode 100644 index 00000000000..5cb4f90d787 --- /dev/null +++ b/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'litellm' + static_configs: + - targets: ['litellm:4000'] # Assuming Litellm exposes metrics at port 4000 From 11512c057d4c1e5c88b92e28d09484de9fbb0c74 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jul 2024 12:19:10 -0700 Subject: [PATCH 88/99] feat use UnsupportedParamsError as litellm error type --- litellm/__init__.py | 1 + litellm/exceptions.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 5a10ae77c11..956834afc3c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -888,6 +888,7 @@ from .exceptions import ( APIError, Timeout, APIConnectionError, + UnsupportedParamsError, APIResponseValidationError, UnprocessableEntityError, InternalServerError, diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 414b3e002ac..d2337b7f494 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -682,11 +682,39 @@ class JSONSchemaValidationError(APIError): ) +class UnsupportedParamsError(BadRequestError): + def __init__( + self, + message, + llm_provider: Optional[str] = None, + model: Optional[str] = None, + status_code: int = 400, + response: Optional[httpx.Response] = None, + litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, + ): + self.status_code = 400 + self.message = "litellm.UnsupportedParamsError: {}".format(message) + self.model = model + self.llm_provider = llm_provider + self.litellm_debug_info = litellm_debug_info + response = response or httpx.Response( + status_code=self.status_code, + request=httpx.Request( + method="GET", url="https://litellm.ai" + ), # mock request object + ) + self.max_retries = max_retries + self.num_retries = num_retries + + LITELLM_EXCEPTION_TYPES = [ AuthenticationError, NotFoundError, BadRequestError, UnprocessableEntityError, + UnsupportedParamsError, Timeout, PermissionDeniedError, RateLimitError, From 8ea4b73c27101974ef5789ad2fceda361fbde623 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jul 2024 12:20:14 -0700 Subject: [PATCH 89/99] add UnsupportedParamsError to litellm exceptions --- litellm/utils.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 035c1c72f7a..a6d3d860301 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -129,6 +129,7 @@ from .exceptions import ( ServiceUnavailableError, Timeout, UnprocessableEntityError, + UnsupportedParamsError, ) from .proxy._types import KeyManagementSystem from .types.llms.openai import ( @@ -225,17 +226,6 @@ last_fetched_at_keys = None # } -class UnsupportedParamsError(Exception): - def __init__(self, status_code, message): - self.status_code = status_code - self.message = message - self.request = httpx.Request(method="POST", url=" https://openai.api.com/v1/") - self.response = httpx.Response(status_code=status_code, request=self.request) - super().__init__( - self.message - ) # Call the base class constructor with the parameters it needs - - ############################################################ def print_verbose( print_statement, From 30c27b3f92a772d48e7a59bc74f6edc7b944a9af Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jul 2024 12:21:22 -0700 Subject: [PATCH 90/99] test UnsupportedParamsError --- litellm/tests/test_bad_params.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/tests/test_bad_params.py b/litellm/tests/test_bad_params.py index 9f126a1b8c0..1b20096e5d0 100644 --- a/litellm/tests/test_bad_params.py +++ b/litellm/tests/test_bad_params.py @@ -2,18 +2,19 @@ # This tests chaos monkeys - if random parts of the system are broken / things aren't sent correctly - what happens. # Expect to add more edge cases to this over time. -import sys, os +import os +import sys import traceback + import pytest sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path import litellm -from litellm import embedding, completion +from litellm import completion, embedding from litellm.utils import Message - # litellm.set_verbose = True user_message = "Hello, how are you?" messages = [{"content": user_message, "role": "user"}] @@ -74,6 +75,8 @@ def test_completion_invalid_param_cohere(): response = completion(model="command-nightly", messages=messages, seed=12) pytest.fail(f"This should have failed cohere does not support `seed` parameter") except Exception as e: + assert isinstance(e, litellm.UnsupportedParamsError) + print("got an exception=", str(e)) if " cohere does not support parameters: {'seed': 12}" in str(e): pass else: From 99d8b0ad68cd1d69896f1d0e6c81359bcccc55ab Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Jul 2024 12:43:52 -0700 Subject: [PATCH 91/99] fix(bedrock_httpx.py): fix async client check --- litellm/llms/bedrock_httpx.py | 2 +- ...odel_prices_and_context_window_backup.json | 30 +++++++++++++++++++ litellm/proxy/_new_secret_config.yaml | 11 ++++--- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py index d6c45fb9c9d..16c3f60b788 100644 --- a/litellm/llms/bedrock_httpx.py +++ b/litellm/llms/bedrock_httpx.py @@ -1731,7 +1731,7 @@ class BedrockConverseLLM(BaseLLM): headers={}, client: Optional[AsyncHTTPHandler] = None, ) -> Union[ModelResponse, CustomStreamWrapper]: - if client is None: + if client is None or not isinstance(client, AsyncHTTPHandler): _params = {} if timeout is not None: if isinstance(timeout, float) or isinstance(timeout, int): diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ef7caf4a754..08bc292c9bf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -760,6 +760,36 @@ "litellm_provider": "azure_ai", "mode": "chat" }, + "azure_ai/Meta-Llama-31-8B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0000003, + "output_cost_per_token": 0.00000061, + "litellm_provider": "azure_ai", + "mode": "chat", + "source":"https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice" + }, + "azure_ai/Meta-Llama-31-70B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00000268, + "output_cost_per_token": 0.00000354, + "litellm_provider": "azure_ai", + "mode": "chat", + "source":"https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice" + }, + "azure_ai/Meta-Llama-31-405B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00000533, + "output_cost_per_token": 0.000016, + "litellm_provider": "azure_ai", + "mode": "chat", + "source":"https://azuremarketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice" + }, "babbage-002": { "max_tokens": 16384, "max_input_tokens": 16384, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 7e3c9a241ab..bec92c1e96f 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,9 +1,8 @@ model_list: - - model_name: anthropic-claude + - model_name: "*" # all requests where model not in your config go to this deployment litellm_params: - model: claude-3-haiku-20240307 + model: "openai/*" # passes our validation check that a real provider is given + api_key: "" -litellm_settings: - callbacks: ["logfire"] - redact_user_api_key_info: true - return_response_headers: true +general_settings: + completion_model: "gpt-3.5-turbo" \ No newline at end of file From 77ffee4e2ea347b7ba0964ba105635f7772d0dff Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Jul 2024 13:07:25 -0700 Subject: [PATCH 92/99] test(test_completion.py): add basic test to confirm azure ad token flow works as expected --- litellm/tests/test_completion.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index dae5e7f805d..f62b2b7ef44 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2607,6 +2607,26 @@ def test_completion_azure_extra_headers(): assert auth_header == "my-bad-key" +def test_completion_azure_ad_token(): + # this tests if we can pass api_key to completion, when it's not in the env. + # DO NOT REMOVE THIS TEST. No MATTER WHAT Happens! + # If you want to remove it, speak to Ishaan! + # Ishaan will be very disappointed if this test is removed -> this is a standard way to pass api_key + the router + proxy use this + from httpx import Client + from openai import AzureOpenAI + + from litellm import completion + from litellm.llms.custom_httpx.httpx_handler import HTTPHandler + + response = completion( + model="azure/chatgpt-v-2", + messages=messages, + # api_key="my-fake-ad-token", + azure_ad_token=os.getenv("AZURE_API_KEY"), + ) + print(response) + + def test_completion_azure_key_completion_arg(): # this tests if we can pass api_key to completion, when it's not in the env. # DO NOT REMOVE THIS TEST. No MATTER WHAT Happens! From 65705fde2558b650de81708bad5f7e5c262036f1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Jul 2024 13:38:03 -0700 Subject: [PATCH 93/99] test(test_embedding.py): add simple azure embedding ad token test Addresses https://github.com/BerriAI/litellm/issues/4859#issuecomment-2248838617 --- litellm/tests/test_embedding.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/litellm/tests/test_embedding.py b/litellm/tests/test_embedding.py index a5a2adb7579..e6dd8bbb2b9 100644 --- a/litellm/tests/test_embedding.py +++ b/litellm/tests/test_embedding.py @@ -673,3 +673,17 @@ async def test_databricks_embeddings(sync_mode): # print(response) # local_proxy_embeddings() + + +def test_embedding_azure_ad_token(): + # this tests if we can pass api_key to completion, when it's not in the env. + # DO NOT REMOVE THIS TEST. No MATTER WHAT Happens! + # If you want to remove it, speak to Ishaan! + # Ishaan will be very disappointed if this test is removed -> this is a standard way to pass api_key + the router + proxy use this + + response = embedding( + model="azure/azure-embedding-model", + input=["good morning from litellm"], + azure_ad_token=os.getenv("AZURE_API_KEY"), + ) + print(response) From fe0b0ddaaa676272e7d6d8bc53f91fd7af9543f4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jul 2024 14:33:49 -0700 Subject: [PATCH 94/99] doc example using litellm proxy with groq --- docs/my-website/docs/providers/groq.md | 106 ++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md index bcca20b5ddf..bfb944cb430 100644 --- a/docs/my-website/docs/providers/groq.md +++ b/docs/my-website/docs/providers/groq.md @@ -1,3 +1,6 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Groq https://groq.com/ @@ -20,7 +23,7 @@ import os os.environ['GROQ_API_KEY'] = "" response = completion( - model="groq/llama2-70b-4096", + model="groq/llama3-8b-8192", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -35,7 +38,7 @@ import os os.environ['GROQ_API_KEY'] = "" response = completion( - model="groq/llama2-70b-4096", + model="groq/llama3-8b-8192", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -47,6 +50,101 @@ for chunk in response: ``` + +## Usage with LiteLLM Proxy + +### 1. Set Groq Models on config.yaml + +```yaml +model_list: + - model_name: groq-llama3-8b-8192 # Model Alias to use for requests + litellm_params: + model: groq/llama3-8b-8192 + api_key: "os.environ/GROQ_API_KEY" # ensure you have `GROQ_API_KEY` in your .env +``` + +### 2. Start Proxy + +``` +litellm --config config.yaml +``` + +### 3. Test it + +Make request to litellm proxy + + + + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "groq-llama3-8b-8192", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create(model="groq-llama3-8b-8192", messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } +]) + +print(response) + +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy + model = "groq-llama3-8b-8192", + temperature=0.1 +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + ## Supported Models - ALL Groq Models Supported! We support ALL Groq models, just set `groq/` as a prefix when sending completion requests @@ -114,7 +212,7 @@ tools = [ } ] response = litellm.completion( - model="groq/llama2-70b-4096", + model="groq/llama3-8b-8192", messages=messages, tools=tools, tool_choice="auto", # auto is default, but we'll be explicit @@ -154,7 +252,7 @@ if tool_calls: ) # extend conversation with function response print(f"messages: {messages}") second_response = litellm.completion( - model="groq/llama2-70b-4096", messages=messages + model="groq/llama3-8b-8192", messages=messages ) # get a new response from the model where it can see the function response print("second response\n", second_response) ``` From b5c5ed220910370b73937208d2e2f481eb966ec9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Jul 2024 15:02:03 -0700 Subject: [PATCH 95/99] fix(key_management_endpoints.py): if budget duration set, set budget_reset_at --- .../proxy/management_endpoints/key_management_endpoints.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 03028821f0f..0e4696e4423 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -333,6 +333,13 @@ async def update_key_fn( expires = datetime.now(timezone.utc) + timedelta(seconds=duration_s) non_default_values["expires"] = expires + if "budget_duration" in non_default_values: + duration_s = _duration_in_seconds( + duration=non_default_values["budget_duration"] + ) + key_reset_at = datetime.now(timezone.utc) + timedelta(seconds=duration_s) + non_default_values["budget_reset_at"] = key_reset_at + response = await prisma_client.update_data( token=key, data={**non_default_values, "token": key} ) From b93b2636a998081260ec9bb47ac23b307f582ff3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 24 Jul 2024 16:51:40 -0700 Subject: [PATCH 96/99] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 92328b4d5c5..3ac5f02852f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Deploy on Railway

-

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.] +

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]

OpenAI Proxy Server | Hosted Proxy (Preview) | Enterprise Tier

From f35af3bf1c631f878b9be8fc207882383af5cf83 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 24 Jul 2024 18:42:31 -0700 Subject: [PATCH 97/99] test(test_completion.py): update azure extra headers --- litellm/tests/test_completion.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index f62b2b7ef44..9061293d53f 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2573,21 +2573,17 @@ def test_completion_azure_extra_headers(): http_client = Client() with patch.object(http_client, "send", new=MagicMock()) as mock_client: - client = AzureOpenAI( - azure_endpoint=os.getenv("AZURE_API_BASE"), - api_version=litellm.AZURE_DEFAULT_API_VERSION, - api_key=os.getenv("AZURE_API_KEY"), - http_client=http_client, - ) + litellm.client_session = http_client try: response = completion( model="azure/chatgpt-v-2", messages=messages, - client=client, + api_base=os.getenv("AZURE_API_BASE"), + api_version="2023-07-01-preview", + api_key=os.getenv("AZURE_API_KEY"), extra_headers={ "Authorization": "my-bad-key", "Ocp-Apim-Subscription-Key": "hello-world-testing", - "api-key": "my-bad-key", }, ) print(response) @@ -2603,8 +2599,10 @@ def test_completion_azure_extra_headers(): print(request.url) # This will print the full URL print(request.headers) # This will print the full URL auth_header = request.headers.get("Authorization") + apim_key = request.headers.get("Ocp-Apim-Subscription-Key") print(auth_header) assert auth_header == "my-bad-key" + assert apim_key == "hello-world-testing" def test_completion_azure_ad_token(): From dd10da4d466ca4c145fe4b320f008efab81a8652 Mon Sep 17 00:00:00 2001 From: wslee Date: Wed, 10 Jul 2024 19:05:38 +0900 Subject: [PATCH 98/99] add support for friendli dedicated endpoint --- docs/my-website/docs/providers/friendliai.md | 60 ++++++++++++++++++++ docs/my-website/sidebars.js | 1 + litellm/utils.py | 5 +- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/providers/friendliai.md diff --git a/docs/my-website/docs/providers/friendliai.md b/docs/my-website/docs/providers/friendliai.md new file mode 100644 index 00000000000..137c3dde380 --- /dev/null +++ b/docs/my-website/docs/providers/friendliai.md @@ -0,0 +1,60 @@ +# FriendliAI +https://suite.friendli.ai/ + +**We support ALL FriendliAI models, just set `friendliai/` as a prefix when sending completion requests** + +## API Key +```python +# env variable +os.environ['FRIENDLI_TOKEN'] +os.environ['FRIENDLI_API_BASE'] # Optional. Set this when using dedicated endpoint. +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['FRIENDLI_TOKEN'] = "" +response = completion( + model="friendliai/mixtral-8x7b-instruct-v0-1", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['FRIENDLI_TOKEN'] = "" +response = completion( + model="friendliai/mixtral-8x7b-instruct-v0-1", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], + stream=True +) + +for chunk in response: + print(chunk) +``` + + +## Supported Models +### Serverless Endpoints +We support ALL FriendliAI AI models, just set `friendliai/` as a prefix when sending completion requests + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mixtral-8x7b-instruct | `completion(model="friendliai/mixtral-8x7b-instruct-v0-1", messages)` | +| meta-llama-3-8b-instruct | `completion(model="friendliai/meta-llama-3-8b-instruct", messages)` | +| meta-llama-3-70b-instruct | `completion(model="friendliai/meta-llama-3-70b-instruct", messages)` | + +### Dedicated Endpoints +``` +model="friendliai/$ENDPOINT_ID:$ADAPTER_ROUTE" +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index c3f7e924984..d228e09d2dc 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -158,6 +158,7 @@ const sidebars = { "providers/triton-inference-server", "providers/ollama", "providers/perplexity", + "providers/friendliai", "providers/groq", "providers/deepseek", "providers/fireworks_ai", diff --git a/litellm/utils.py b/litellm/utils.py index a6d3d860301..03bbb0e8cb0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4486,7 +4486,10 @@ def get_llm_provider( or get_secret("TOGETHER_AI_TOKEN") ) elif custom_llm_provider == "friendliai": - api_base = "https://inference.friendli.ai/v1" + api_base = ( + get_secret("FRIENDLI_API_BASE") + or "https://inference.friendli.ai/v1" + ) dynamic_api_key = ( api_key or get_secret("FRIENDLIAI_API_KEY") From 40bb165108ddb3bf3a20e5084924c42698023530 Mon Sep 17 00:00:00 2001 From: wslee Date: Mon, 15 Jul 2024 10:24:54 +0900 Subject: [PATCH 99/99] support dynamic api base --- litellm/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 03bbb0e8cb0..f35f1ce4b0c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4487,7 +4487,8 @@ def get_llm_provider( ) elif custom_llm_provider == "friendliai": api_base = ( - get_secret("FRIENDLI_API_BASE") + api_base + or get_secret("FRIENDLI_API_BASE") or "https://inference.friendli.ai/v1" ) dynamic_api_key = (