From 7c0c3529630716db4e1e63be052d85ec1821cd35 Mon Sep 17 00:00:00 2001 From: rzeta-10 Date: Mon, 23 Mar 2026 21:45:02 +0530 Subject: [PATCH] Add Akto integration documentation and update logging callback manager --- .../docs/observability/akto_integration.md | 105 ++++++++++++++++++ .../custom_logger_registry.py | 2 + litellm/litellm_core_utils/litellm_logging.py | 80 ++----------- .../logging_callback_manager.py | 10 +- 4 files changed, 119 insertions(+), 78 deletions(-) create mode 100644 docs/my-website/docs/observability/akto_integration.md diff --git a/docs/my-website/docs/observability/akto_integration.md b/docs/my-website/docs/observability/akto_integration.md new file mode 100644 index 00000000000..c651930c7f8 --- /dev/null +++ b/docs/my-website/docs/observability/akto_integration.md @@ -0,0 +1,105 @@ +# Akto - LLM Traffic Monitoring & API Security + +## What is Akto? + +[Akto](https://www.akto.io/) is an API security platform that provides monitoring, testing, and guardrails for AI/ML workloads. For LLM applications, Akto ingests request/response traffic for security analysis, vulnerability detection, and compliance monitoring. + +## Usage with LiteLLM Proxy (LLM Gateway) + +**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` + +```yaml +model_list: + - model_name: gpt-5.4 + litellm_params: + model: gpt-5.4 + +litellm_settings: + success_callback: ["akto"] + failure_callback: ["akto"] +``` + +**Step 2**: Set required environment variables + +```shell +export AKTO_DATA_INGESTION_API_BASE="http://your-akto-instance:8080" +export AKTO_API_KEY="your-akto-api-key" + +# Optional +export AKTO_ACCOUNT_ID="1000000" # default: 1000000 +export AKTO_VXLAN_ID="0" # default: 0 +``` + +**Step 3**: Start the proxy, make a test request + +Start proxy + +```shell +litellm --config config.yaml --debug +``` + +Test Request + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-5.4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + +## What's Logged to Akto? + +When LiteLLM logs to Akto, it sends the full HTTP transaction in Akto's MIRRORING format: + +### For Every LLM Call +- **Request**: Messages, model, tools, tool calls +- **Response**: Full model response (choices, usage) +- **Headers**: All proxy request headers (sensitive headers like `Authorization`, `Cookie` are stripped) +- **Metadata**: User ID, team ID, API route, client IP +- **Status**: HTTP status code (200 for success, 500 for failures) +- **Timing**: Request timestamp + +### For Errors +- **Status Code**: Extracted from the exception (e.g., 429 for rate limits, 500 for server errors) +- **Request Context**: The original request that caused the error + +## Environment Variables + +| Variable | Required | Description | +|----------|----------|-------------| +| `AKTO_DATA_INGESTION_API_BASE` | Yes | Akto data ingestion API base URL | +| `AKTO_API_KEY` | Yes | Akto API key for authentication | +| `AKTO_ACCOUNT_ID` | No | Akto account ID (default: `1000000`) | +| `AKTO_VXLAN_ID` | No | Akto VXLAN ID (default: `0`) | + +## Troubleshooting + +### 1. Missing API Key +``` +Error: Missing keys=['AKTO_DATA_INGESTION_API_BASE'] in environment. +``` + +Set your Akto environment variables: +```shell +export AKTO_DATA_INGESTION_API_BASE="http://your-akto-instance:8080" +export AKTO_API_KEY="your-api-key" +``` + +### 2. Events Not Appearing +- Check that your API key is correct +- Verify network connectivity to the Akto data ingestion service +- Check LiteLLM logs for `Akto logging error` or `Akto ingestion returned` warnings + +### 3. Health Check +Verify the Akto integration is healthy: +```shell +curl 'http://localhost:4000/health/services?service=akto' \ + -H 'Authorization: Bearer your-litellm-key' +``` diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index f873bfeece5..5ff30d76718 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -38,6 +38,7 @@ from litellm.integrations.langsmith import LangsmithLogger from litellm.integrations.litellm_agent import LiteLLMAgentModelResolver from litellm.integrations.literal_ai import LiteralAILogger from litellm.integrations.mlflow import MlflowLogger +from litellm.integrations.akto.akto_logger import AktoLogger from litellm.integrations.openmeter import OpenMeterLogger from litellm.integrations.opentelemetry import OpenTelemetry from litellm.integrations.opik.opik import OpikLogger @@ -102,6 +103,7 @@ class CustomLoggerRegistry: "focus": FocusLogger, "vantage": VantageLogger, "posthog": PostHogLogger, + "akto": AktoLogger, } try: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4ec01bf5623..303e2d3525b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -162,7 +162,6 @@ from ..integrations.s3 import S3Logger from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger -from ..integrations.akto.akto_logger import AktoLogger from .exception_mapping_utils import _get_response_headers from .initialize_dynamic_callback_params import ( initialize_standard_callback_dynamic_params as _initialize_standard_callback_dynamic_params, @@ -242,7 +241,6 @@ greenscaleLogger = None lunaryLogger = None supabaseClient = None deepevalLogger = None -aktoLogger = None callback_list: Optional[List[str]] = [] user_logger_fn = None additional_details: Optional[Dict[str, str]] = {} @@ -2389,26 +2387,6 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, ) - if callback == "akto" and is_sync_request: - global aktoLogger - if aktoLogger is None: - aktoLogger = AktoLogger() - if self.stream and complete_streaming_response is None: - pass # skip partial stream chunks - else: - if self.stream and complete_streaming_response: - self.model_call_details[ - "complete_response" - ] = self.model_call_details.get( - "complete_streaming_response", {} - ) - result = self.model_call_details["complete_response"] - aktoLogger.log_success_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) if ( isinstance(callback, CustomLogger) and is_sync_request @@ -2724,31 +2702,6 @@ class Logging(LiteLLMLoggingBaseClass): end_time=end_time, ) - if callback == "akto": - global aktoLogger - if aktoLogger is None: - aktoLogger = AktoLogger() - if self.stream is True: - if ( - "async_complete_streaming_response" - in self.model_call_details - ): - await aktoLogger.async_log_success_event( - kwargs=self.model_call_details, - response_obj=self.model_call_details[ - "async_complete_streaming_response" - ], - start_time=start_time, - end_time=end_time, - ) - else: - await aktoLogger.async_log_success_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) - if isinstance(callback, CustomLogger): # custom logger class model_call_details: Dict = self.model_call_details ################################## @@ -3058,16 +3011,6 @@ class Logging(LiteLLMLoggingBaseClass): print_verbose=print_verbose, callback_func=callback, ) - if callback == "akto" and is_sync_request: - global aktoLogger - if aktoLogger is None: - aktoLogger = AktoLogger() - aktoLogger.log_failure_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) if ( isinstance(callback, CustomLogger) and is_sync_request @@ -3193,16 +3136,6 @@ class Logging(LiteLLMLoggingBaseClass): ) if not should_run: continue - if callback == "akto": - global aktoLogger - if aktoLogger is None: - aktoLogger = AktoLogger() - await aktoLogger.async_log_failure_event( - kwargs=self.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) if isinstance(callback, CustomLogger): # custom logger class await callback.async_log_failure_event( kwargs=self.model_call_details, @@ -3619,7 +3552,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 """ Globally sets the callback client """ - global sentry_sdk_instance, capture_exception, add_breadcrumb, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, supabaseClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, logfireLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger, deepevalLogger, aktoLogger + global sentry_sdk_instance, capture_exception, add_breadcrumb, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, supabaseClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, logfireLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger, deepevalLogger try: for callback in callback_list: @@ -3709,9 +3642,6 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 elif callback == "greenscale": greenscaleLogger = GreenscaleLogger() print_verbose("Initialized Greenscale Logger") - elif callback == "akto": - aktoLogger = AktoLogger() - print_verbose("Initialized Akto Logger") elif callable(callback): customLogger = CustomLogger() except Exception as e: @@ -3757,6 +3687,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_openmeter_logger) return _openmeter_logger # type: ignore elif logging_integration == "akto": + from litellm.integrations.akto.akto_logger import AktoLogger + for callback in _in_memory_loggers: if isinstance(callback, AktoLogger): return callback # type: ignore @@ -4406,6 +4338,12 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 for callback in _in_memory_loggers: if isinstance(callback, OpenMeterLogger): return callback + elif logging_integration == "akto": + from litellm.integrations.akto.akto_logger import AktoLogger + + for callback in _in_memory_loggers: + if isinstance(callback, AktoLogger): + return callback elif logging_integration == "braintrust": from litellm.integrations.braintrust_logging import BraintrustLogger diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 6942337892b..c5c150274cc 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -77,9 +77,9 @@ class LoggingCallbackManager: """ Add a success callback to `litellm.success_callback`. Auto-routes async callbacks to litellm._async_success_callback. - Special-cases 'dynamodb', 'openmeter', and 'akto' as async callbacks. + Special-cases 'dynamodb' and 'openmeter' as async callbacks. """ - if isinstance(callback, str) and callback in ("dynamodb", "openmeter", "akto"): + if isinstance(callback, str) and callback in ("dynamodb", "openmeter"): self._safe_add_callback_to_list( callback=callback, parent_list=litellm._async_success_callback ) @@ -99,11 +99,7 @@ class LoggingCallbackManager: Add a failure callback to `litellm.failure_callback`. Auto-routes async callbacks to litellm._async_failure_callback. """ - if isinstance(callback, str) and callback in ("akto",): - self._safe_add_callback_to_list( - callback=callback, parent_list=litellm._async_failure_callback - ) - elif not isinstance(callback, str) and self._is_async_callable(callback): + if not isinstance(callback, str) and self._is_async_callable(callback): self._safe_add_callback_to_list( callback=callback, parent_list=litellm._async_failure_callback )