litellm/litellm/proxy/common_utils/debug_utils.py

# Start tracing memory allocations
import asyncio
import gc
import json
import os
import socket
import sys
import tracemalloc
from collections import Counter
from collections.abc import Mapping, Sequence
from typing import Any, Final, NamedTuple, Protocol, TypedDict

from fastapi import APIRouter, Depends, HTTPException, Query
from typing_extensions import ReadOnly

from litellm import get_secret_str
from litellm._logging import verbose_proxy_logger
from litellm.constants import PYTHON_GC_THRESHOLD
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth

router: Final = APIRouter()


# Configure garbage collection thresholds from environment variables
def configure_gc_thresholds():
    """Configure Python garbage collection thresholds from environment variables."""
    gc_threshold_env: Final = PYTHON_GC_THRESHOLD
    if gc_threshold_env:
        try:
            # Parse threshold string like "1000,50,50"
            thresholds: Final = [int(x.strip()) for x in gc_threshold_env.split(",")]
            if len(thresholds) == 3:
                gc.set_threshold(*thresholds)
                verbose_proxy_logger.info("GC thresholds set to: %s", thresholds)
            else:
                verbose_proxy_logger.warning(
                    "GC threshold not set: %s. Expected format: 'gen0,gen1,gen2'", gc_threshold_env
                )
        except ValueError as e:
            verbose_proxy_logger.warning("Failed to parse GC threshold: %s. Error: %s", gc_threshold_env, e)

    # Log current thresholds
    current_thresholds: Final = gc.get_threshold()
    verbose_proxy_logger.info(
        "Current GC thresholds: gen0=%s, gen1=%s, gen2=%s",
        current_thresholds[0],
        current_thresholds[1],
        current_thresholds[2],
    )


# Initialize GC configuration
configure_gc_thresholds()


@router.get(
    "/debug/asyncio-tasks",
    dependencies=[Depends(user_api_key_auth)],
)
async def get_active_tasks_stats():
    """
    Returns:
      total_active_tasks: int
      by_name: { coroutine_name: count }
    """
    MAX_TASKS_TO_CHECK: Final = 5000
    # Gather all tasks in this event loop (including this endpoint’s own task).
    all_tasks: Final = asyncio.all_tasks()

    # Filter out tasks that are already done.
    active_tasks: Final = [t for t in all_tasks if not t.done()]

    # Count how many active tasks exist, grouped by coroutine function name.
    counter: Final = Counter()
    for idx, task in enumerate(active_tasks):
        # reasonable max circuit breaker
        if idx >= MAX_TASKS_TO_CHECK:
            break
        coro = task.get_coro()
        # Derive a human‐readable name from the coroutine:
        name = getattr(coro, "__qualname__", None) or getattr(coro, "__name__", None) or repr(coro)
        counter[name] += 1

    return {
        "total_active_tasks": len(active_tasks),
        "by_name": dict(counter),
    }


if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
    try:
        import objgraph

        print("growth of objects")  # noqa: T201
        objgraph.show_growth()
        print("\n\nMost common types")  # noqa: T201
        objgraph.show_most_common_types()
        roots: Final = objgraph.get_leaking_objects()
        print("\n\nLeaking objects")  # noqa: T201
        objgraph.show_most_common_types(objects=roots)
    except ImportError:
        raise ImportError("objgraph not found. Please install objgraph to use this feature.")

    tracemalloc.start(10)

    @router.get(
        "/memory-usage",
        dependencies=[Depends(user_api_key_auth)],
        include_in_schema=False,
    )
    async def memory_usage():
        # Take a snapshot of the current memory usage
        snapshot: Final = tracemalloc.take_snapshot()
        top_stats: Final = snapshot.statistics("lineno")
        verbose_proxy_logger.debug("TOP STATS: %s", top_stats)

        # Get the top 50 memory usage lines
        top_50: Final = top_stats[:50]
        result: Final = []
        for stat in top_50:
            result.append(f"{stat.traceback.format(limit=10)}: {stat.size / 1024} KiB")

        return {"top_50_memory_usage": result}


@router.get("/memory-usage-in-mem-cache", include_in_schema=False)
async def memory_usage_in_mem_cache(
    _: UserAPIKeyAuth = Depends(user_api_key_auth),
):
    # returns the size of all in-memory caches on the proxy server
    """
    1. user_api_key_cache
    2. router_cache
    3. proxy_logging_cache
    4. internal_usage_cache
    """
    from litellm.proxy.proxy_server import (
        llm_router,
        proxy_logging_obj,
        user_api_key_cache,
    )

    if llm_router is None:
        num_items_in_llm_router_cache = 0
    else:
        num_items_in_llm_router_cache = len(llm_router.cache.in_memory_cache.cache_dict) + len(
            llm_router.cache.in_memory_cache.ttl_dict
        )

    num_items_in_user_api_key_cache: Final = (
        len(user_api_key_cache.in_memory_cache.cache_dict)
        + len(user_api_key_cache.in_memory_cache.ttl_dict)
        + len(user_api_key_cache.key_object_cache.in_memory_cache.cache_dict)
        + len(user_api_key_cache.key_object_cache.in_memory_cache.ttl_dict)
    )

    num_items_in_proxy_logging_obj_cache: Final = len(
        proxy_logging_obj.internal_usage_cache.dual_cache.in_memory_cache.cache_dict
    ) + len(proxy_logging_obj.internal_usage_cache.dual_cache.in_memory_cache.ttl_dict)

    return {
        "num_items_in_user_api_key_cache": num_items_in_user_api_key_cache,
        "num_items_in_llm_router_cache": num_items_in_llm_router_cache,
        "num_items_in_proxy_logging_obj_cache": num_items_in_proxy_logging_obj_cache,
    }


@router.get("/memory-usage-in-mem-cache-items", include_in_schema=False)
async def memory_usage_in_mem_cache_items(
    _: UserAPIKeyAuth = Depends(user_api_key_auth),
):
    # returns the size of all in-memory caches on the proxy server
    """
    1. user_api_key_cache
    2. router_cache
    3. proxy_logging_cache
    4. internal_usage_cache
    """
    from litellm.proxy.proxy_server import (
        llm_router,
        proxy_logging_obj,
        user_api_key_cache,
    )

    if llm_router is None:
        llm_router_in_memory_cache_dict = {}
        llm_router_in_memory_ttl_dict = {}
    else:
        llm_router_in_memory_cache_dict = llm_router.cache.in_memory_cache.cache_dict
        llm_router_in_memory_ttl_dict = llm_router.cache.in_memory_cache.ttl_dict

    return {
        "user_api_key_cache": user_api_key_cache.in_memory_cache.cache_dict,
        "user_api_key_ttl": user_api_key_cache.in_memory_cache.ttl_dict,
        "user_key_object_cache": user_api_key_cache.key_object_cache.in_memory_cache.cache_dict,
        "user_key_object_ttl": user_api_key_cache.key_object_cache.in_memory_cache.ttl_dict,
        "llm_router_cache": llm_router_in_memory_cache_dict,
        "llm_router_ttl": llm_router_in_memory_ttl_dict,
        "proxy_logging_obj_cache": proxy_logging_obj.internal_usage_cache.dual_cache.in_memory_cache.cache_dict,
        "proxy_logging_obj_ttl": proxy_logging_obj.internal_usage_cache.dual_cache.in_memory_cache.ttl_dict,
    }


class _ProcessMemoryInfo(Protocol):
    """The resident and virtual sizes psutil reports for a process."""

    @property
    def rss(self) -> int: ...

    @property
    def vms(self) -> int: ...


class _ProcessHandle(Protocol):
    """The psutil process handle members this module reads."""

    def memory_info(self) -> _ProcessMemoryInfo: ...

    def memory_percent(self) -> float: ...


class _ProcessMemoryUsage(NamedTuple):
    """Memory usage of a single worker process."""

    resident_megabytes: float
    virtual_megabytes: float
    percent: float


def _process_memory_usage(process: _ProcessHandle) -> _ProcessMemoryUsage:
    """Read resident/virtual megabytes and system memory share for ``process``."""
    memory_info: Final = process.memory_info()
    return _ProcessMemoryUsage(
        resident_megabytes=memory_info.rss / (1024 * 1024),
        virtual_megabytes=memory_info.vms / (1024 * 1024),
        percent=process.memory_percent(),
    )


PROC_STATM_PATH: Final = "/proc/self/statm"
PROC_MEMINFO_PATH: Final = "/proc/meminfo"
PSUTIL_MISSING_ERROR: Final = "Install psutil for memory monitoring: pip install psutil"


class _ProcMemoryInfo(NamedTuple):
    rss: int
    vms: int


class _ProcFilesystemProcess:
    """Memory of the running process read from the Linux proc filesystem, for images without psutil."""

    def __init__(
        self,
        statm_path: str = PROC_STATM_PATH,
        meminfo_path: str = PROC_MEMINFO_PATH,
        page_size: int | None = None,
    ) -> None:
        self._statm_path: Final = statm_path
        self._meminfo_path: Final = meminfo_path
        self._page_size: Final = os.sysconf("SC_PAGE_SIZE") if page_size is None else page_size

    def memory_info(self) -> _ProcMemoryInfo:
        with open(self._statm_path, encoding="ascii") as statm:
            size_pages, resident_pages = statm.read().split()[:2]
        return _ProcMemoryInfo(rss=int(resident_pages) * self._page_size, vms=int(size_pages) * self._page_size)

    def memory_percent(self) -> float:
        with open(self._meminfo_path, encoding="ascii") as meminfo:
            total_kilobytes: Final = next(int(line.split()[1]) for line in meminfo if line.startswith("MemTotal:"))
        return self.memory_info().rss / (total_kilobytes * 1024) * 100


def _process_handle() -> _ProcessHandle | None:
    try:
        import psutil
    except ImportError:
        return _ProcFilesystemProcess() if os.path.exists(PROC_STATM_PATH) else None
    return psutil.Process()


def _health_status(memory_percent: float) -> str:
    if memory_percent > 80:
        return "critical"
    if memory_percent > 60:
        return "warning"
    return "healthy"


class _SummaryProcessMemory(TypedDict, total=False):
    summary: ReadOnly[str]
    ram_usage_mb: ReadOnly[float]
    system_memory_percent: ReadOnly[float]
    error: ReadOnly[str]


def _summary_process_memory(process: _ProcessHandle | None) -> tuple[_SummaryProcessMemory, str]:
    if process is None:
        missing: Final[_SummaryProcessMemory] = {"error": PSUTIL_MISSING_ERROR}
        return missing, "healthy"
    try:
        usage: Final = _process_memory_usage(process)
    except Exception as e:
        unreadable: Final[_SummaryProcessMemory] = {"error": str(e)}
        return unreadable, "healthy"
    memory: Final[_SummaryProcessMemory] = {
        "summary": f"{usage.resident_megabytes:.1f} MB ({usage.percent:.1f}% of system memory)",
        "ram_usage_mb": round(usage.resident_megabytes, 2),
        "system_memory_percent": round(usage.percent, 2),
    }
    return memory, _health_status(usage.percent)


@router.get("/debug/memory/summary", include_in_schema=False)
async def get_memory_summary(
    _: UserAPIKeyAuth = Depends(user_api_key_auth),
) -> dict[str, Any]:
    """
    Get simplified memory usage summary for the proxy.

    Returns:
    - worker_pid: Process ID
    - hostname: Host (the pod on Kubernetes) the worker runs on
    - status: Overall health based on memory usage
    - memory: Process memory usage and RAM info
    - caches: Cache item counts and descriptions
    - garbage_collector: GC status and pending object counts

    Example usage:
    curl http://localhost:4000/debug/memory/summary -H "Authorization: Bearer sk-1234"

    For detailed analysis, call GET /debug/memory/details
    For cache management, use the cache management endpoints
    """
    from litellm.proxy.proxy_server import (
        llm_router,
        proxy_logging_obj,
        user_api_key_cache,
    )

    process_memory, health_status = _summary_process_memory(_process_handle())

    # Get cache information
    caches: Final[dict[str, object]] = {}
    total_cache_items = 0

    try:
        # User API key cache
        user_cache_items: Final = len(user_api_key_cache.in_memory_cache.cache_dict) + len(
            user_api_key_cache.key_object_cache.in_memory_cache.cache_dict
        )
        total_cache_items += user_cache_items
        caches["user_api_keys"] = {
            "count": user_cache_items,
            "count_readable": f"{user_cache_items:,}",
            "what_it_stores": "Validated API keys for faster authentication",
        }

        # Router cache
        if llm_router is not None:
            router_cache_items: Final = len(llm_router.cache.in_memory_cache.cache_dict)
            total_cache_items += router_cache_items
            caches["llm_responses"] = {
                "count": router_cache_items,
                "count_readable": f"{router_cache_items:,}",
                "what_it_stores": "LLM responses for identical requests",
            }

        # Proxy logging cache
        logging_cache_items: Final = len(proxy_logging_obj.internal_usage_cache.dual_cache.in_memory_cache.cache_dict)
        total_cache_items += logging_cache_items
        caches["usage_tracking"] = {
            "count": logging_cache_items,
            "count_readable": f"{logging_cache_items:,}",
            "what_it_stores": "Usage metrics before database write",
        }

    except Exception as e:
        caches["error"] = str(e)

    # Get garbage collector stats
    gc_enabled: Final = gc.isenabled()
    objects_pending: Final = gc.get_count()[0]
    uncollectable: Final = len(gc.garbage)

    gc_info: Final = {
        "status": "enabled" if gc_enabled else "disabled",
        "objects_awaiting_collection": objects_pending,
    }

    # Add warning if garbage collection issues detected
    if uncollectable > 0:
        gc_info["warning"] = f"{uncollectable} uncollectable objects (possible memory leak)"

    return {
        "worker_pid": os.getpid(),
        "hostname": socket.gethostname(),
        "status": health_status,
        "memory": process_memory,
        "caches": {
            "total_items": total_cache_items,
            "breakdown": caches,
        },
        "garbage_collector": gc_info,
    }


def _get_gc_statistics() -> Mapping[str, object]:
    """Get garbage collector statistics."""
    return {
        "enabled": gc.isenabled(),
        "thresholds": {
            "generation_0": gc.get_threshold()[0],
            "generation_1": gc.get_threshold()[1],
            "generation_2": gc.get_threshold()[2],
            "explanation": "Number of allocations before automatic collection for each generation",
        },
        "current_counts": {
            "generation_0": gc.get_count()[0],
            "generation_1": gc.get_count()[1],
            "generation_2": gc.get_count()[2],
            "explanation": "Current number of allocated objects in each generation",
        },
        "collection_history": [
            {
                "generation": i,
                "total_collections": stat["collections"],
                "total_collected": stat["collected"],
                "uncollectable": stat["uncollectable"],
            }
            for i, stat in enumerate(gc.get_stats())
        ],
    }


class _ObjectTypeCount(TypedDict):
    """One row of the tracked-object histogram."""

    type: ReadOnly[str]
    count: ReadOnly[int]
    count_readable: ReadOnly[str]


def _type_name_counts(objects: Sequence[object]) -> Counter[str]:
    """Count ``objects`` by the name of their type."""
    return Counter(type(obj).__name__ for obj in objects)


def _get_object_type_counts(top_n: int) -> tuple[int, list[_ObjectTypeCount]]:
    """Count objects by type and return total count and top N types."""
    type_counts: Final = _type_name_counts(gc.get_objects())

    top_object_types: Final[list[_ObjectTypeCount]] = [
        {"type": obj_type, "count": count, "count_readable": f"{count:,}"}
        for obj_type, count in type_counts.most_common(top_n)
    ]

    return sum(type_counts.values()), top_object_types


def _type_names(objects: Sequence[object]) -> Sequence[str]:
    """The type name of each object in ``objects``."""
    return [type(obj).__name__ for obj in objects]


def _get_uncollectable_objects_info() -> Mapping[str, object]:
    """Get information about uncollectable objects (potential memory leaks)."""
    uncollectable: Final = gc.garbage
    return {
        "count": len(uncollectable),
        "sample_types": _type_names(uncollectable[:10]),
        "warning": (
            "If count > 0, you may have reference cycles preventing garbage collection"
            if len(uncollectable) > 0
            else None
        ),
    }


def _get_cache_memory_stats(
    user_api_key_cache, llm_router, proxy_logging_obj, redis_usage_cache
) -> Mapping[str, object]:
    """Calculate memory usage for all caches."""
    cache_stats: Final[dict[str, object]] = {}
    try:
        # User API key cache
        key_object_in_memory_cache: Final = user_api_key_cache.key_object_cache.in_memory_cache
        user_cache_size: Final = sys.getsizeof(user_api_key_cache.in_memory_cache.cache_dict) + sys.getsizeof(
            key_object_in_memory_cache.cache_dict
        )
        user_ttl_size: Final = sys.getsizeof(user_api_key_cache.in_memory_cache.ttl_dict) + sys.getsizeof(
            key_object_in_memory_cache.ttl_dict
        )
        cache_stats["user_api_key_cache"] = {
            "num_items": len(user_api_key_cache.in_memory_cache.cache_dict)
            + len(key_object_in_memory_cache.cache_dict),
            "cache_dict_size_bytes": user_cache_size,
            "ttl_dict_size_bytes": user_ttl_size,
            "total_size_mb": round((user_cache_size + user_ttl_size) / (1024 * 1024), 2),
        }

        # Router cache
        if llm_router is not None:
            router_cache_size: Final = sys.getsizeof(llm_router.cache.in_memory_cache.cache_dict)
            router_ttl_size: Final = sys.getsizeof(llm_router.cache.in_memory_cache.ttl_dict)
            cache_stats["llm_router_cache"] = {
                "num_items": len(llm_router.cache.in_memory_cache.cache_dict),
                "cache_dict_size_bytes": router_cache_size,
                "ttl_dict_size_bytes": router_ttl_size,
                "total_size_mb": round((router_cache_size + router_ttl_size) / (1024 * 1024), 2),
            }

        # Proxy logging cache
        logging_cache_size = sys.getsizeof(proxy_logging_obj.internal_usage_cache.dual_cache.in_memory_cache.cache_dict)
        logging_ttl_size = sys.getsizeof(proxy_logging_obj.internal_usage_cache.dual_cache.in_memory_cache.ttl_dict)
        cache_stats["proxy_logging_cache"] = {
            "num_items": len(proxy_logging_obj.internal_usage_cache.dual_cache.in_memory_cache.cache_dict),
            "cache_dict_size_bytes": logging_cache_size,
            "ttl_dict_size_bytes": logging_ttl_size,
            "total_size_mb": round((logging_cache_size + logging_ttl_size) / (1024 * 1024), 2),
        }

        # Redis cache info
        if redis_usage_cache is not None:
            cache_stats["redis_usage_cache"] = {
                "enabled": True,
                "cache_type": type(redis_usage_cache).__name__,
            }
            # Try to get Redis connection pool info if available
            try:
                if hasattr(redis_usage_cache, "redis_client") and redis_usage_cache.redis_client:
                    if hasattr(redis_usage_cache.redis_client, "connection_pool"):
                        pool_info: Final = redis_usage_cache.redis_client.connection_pool
                        cache_stats["redis_usage_cache"]["connection_pool"] = {
                            "max_connections": (
                                pool_info.max_connections if hasattr(pool_info, "max_connections") else None
                            ),
                            "connection_class": (
                                pool_info.connection_class.__name__ if hasattr(pool_info, "connection_class") else None
                            ),
                        }
            except Exception as e:
                verbose_proxy_logger.debug("Error getting Redis pool info: %s", e)
        else:
            cache_stats["redis_usage_cache"] = {"enabled": False}

    except Exception as e:
        verbose_proxy_logger.debug("Error calculating cache stats: %s", e)
        cache_stats["error"] = str(e)

    return cache_stats


def _get_router_memory_stats(llm_router) -> Mapping[str, object]:
    """Get memory usage statistics for LiteLLM router."""
    litellm_router_memory: dict[str, object] = {}
    try:
        if llm_router is not None:
            # Model list memory size
            if hasattr(llm_router, "model_list") and llm_router.model_list:
                model_list_size: Final = sys.getsizeof(llm_router.model_list)
                litellm_router_memory["model_list"] = {
                    "num_models": len(llm_router.model_list),
                    "size_bytes": model_list_size,
                    "size_mb": round(model_list_size / (1024 * 1024), 4),
                }

            # Model names set
            if hasattr(llm_router, "model_names") and llm_router.model_names:
                model_names_size: Final = sys.getsizeof(llm_router.model_names)
                litellm_router_memory["model_names_set"] = {
                    "num_model_groups": len(llm_router.model_names),
                    "size_bytes": model_names_size,
                    "size_mb": round(model_names_size / (1024 * 1024), 4),
                }

            # Deployment names list
            if hasattr(llm_router, "deployment_names") and llm_router.deployment_names:
                deployment_names_size: Final = sys.getsizeof(llm_router.deployment_names)
                litellm_router_memory["deployment_names"] = {
                    "num_deployments": len(llm_router.deployment_names),
                    "size_bytes": deployment_names_size,
                    "size_mb": round(deployment_names_size / (1024 * 1024), 4),
                }

            # Deployment latency map
            if hasattr(llm_router, "deployment_latency_map") and llm_router.deployment_latency_map:
                latency_map_size: Final = sys.getsizeof(llm_router.deployment_latency_map)
                litellm_router_memory["deployment_latency_map"] = {
                    "num_tracked_deployments": len(llm_router.deployment_latency_map),
                    "size_bytes": latency_map_size,
                    "size_mb": round(latency_map_size / (1024 * 1024), 4),
                }

            # Fallback configuration
            if hasattr(llm_router, "fallbacks") and llm_router.fallbacks:
                fallbacks_size: Final = sys.getsizeof(llm_router.fallbacks)
                litellm_router_memory["fallbacks"] = {
                    "num_fallback_configs": len(llm_router.fallbacks),
                    "size_bytes": fallbacks_size,
                    "size_mb": round(fallbacks_size / (1024 * 1024), 4),
                }

            # Total router object size
            router_obj_size: Final = sys.getsizeof(llm_router)
            litellm_router_memory["router_object"] = {
                "size_bytes": router_obj_size,
                "size_mb": round(router_obj_size / (1024 * 1024), 4),
            }

        else:
            litellm_router_memory = {"note": "Router not initialized"}
    except Exception as e:
        verbose_proxy_logger.debug("Error getting router memory info: %s", e)
        litellm_router_memory = {"error": str(e)}

    return litellm_router_memory


def _get_process_memory_info(worker_pid: int, include_process_info: bool) -> Mapping[str, object] | None:
    """Get process-level memory information using psutil."""
    if not include_process_info:
        return None

    try:
        import psutil

        process: Final = psutil.Process()
        usage: Final = _process_memory_usage(process)
        ram_usage_mb: Final = round(usage.resident_megabytes, 2)
        virtual_memory_mb: Final = round(usage.virtual_megabytes, 2)
        memory_percent: Final = round(usage.percent, 2)

        return {
            "pid": worker_pid,
            "summary": f"Worker PID {worker_pid} using {ram_usage_mb:.1f} MB of RAM ({memory_percent:.1f}% of system memory)",
            "ram_usage": {
                "megabytes": ram_usage_mb,
                "description": "Actual physical RAM used by this process",
            },
            "virtual_memory": {
                "megabytes": virtual_memory_mb,
                "description": "Total virtual memory allocated (includes swapped memory)",
            },
            "system_memory_percent": {
                "percent": memory_percent,
                "description": "Percentage of total system RAM being used",
            },
            "open_file_handles": {
                "count": (process.num_fds() if hasattr(process, "num_fds") else "N/A (Windows)"),
                "description": "Number of open file descriptors/handles",
            },
            "threads": {
                "count": process.num_threads(),
                "description": "Number of active threads in this process",
            },
        }
    except ImportError:
        return {
            "pid": worker_pid,
            "error": "psutil not installed. Install with: pip install psutil",
        }
    except Exception as e:
        verbose_proxy_logger.debug("Error getting process info: %s", e)
        return {"pid": worker_pid, "error": str(e)}


@router.get("/debug/memory/details", include_in_schema=False)
async def get_memory_details(
    _: UserAPIKeyAuth = Depends(user_api_key_auth),
    top_n: int = Query(20, description="Number of top object types to return"),
    include_process_info: bool = Query(True, description="Include process memory info"),
) -> dict[str, Any]:
    """
    Get detailed memory diagnostics for deep debugging.

    Returns:
    - worker_pid: Process ID
    - process_memory: RAM usage, virtual memory, file handles, threads
    - garbage_collector: GC thresholds, counts, collection history
    - objects: Total tracked objects and top object types
    - uncollectable: Objects that can't be garbage collected (potential leaks)
    - cache_memory: Memory usage of user_api_key, router, and logging caches
    - router_memory: Memory usage of router components (model_list, deployment_names, etc.)

    Query Parameters:
    - top_n: Number of top object types to return (default: 20)
    - include_process_info: Include process-level memory info using psutil (default: true)

    Example usage:
    curl "http://localhost:4000/debug/memory/details?top_n=30" -H "Authorization: Bearer sk-1234"

    All memory sizes are reported in both bytes and MB.
    """
    from litellm.proxy.proxy_server import (
        llm_router,
        proxy_logging_obj,
        redis_usage_cache,
        user_api_key_cache,
    )

    worker_pid: Final = os.getpid()

    # Collect all diagnostics using helper functions
    gc_stats: Final = _get_gc_statistics()
    total_objects, top_object_types = _get_object_type_counts(top_n)
    uncollectable_info: Final = _get_uncollectable_objects_info()
    cache_stats: Final = _get_cache_memory_stats(user_api_key_cache, llm_router, proxy_logging_obj, redis_usage_cache)
    litellm_router_memory: Final = _get_router_memory_stats(llm_router)
    process_info: Final = _get_process_memory_info(worker_pid, include_process_info)

    return {
        "worker_pid": worker_pid,
        "process_memory": process_info,
        "garbage_collector": gc_stats,
        "objects": {
            "total_tracked": total_objects,
            "total_tracked_readable": f"{total_objects:,}",
            "top_types": top_object_types,
        },
        "uncollectable": uncollectable_info,
        "cache_memory": cache_stats,
        "router_memory": litellm_router_memory,
    }


@router.post("/debug/memory/gc/configure", include_in_schema=False)
async def configure_gc_thresholds_endpoint(
    _: UserAPIKeyAuth = Depends(user_api_key_auth),
    generation_0: int = Query(700, description="Generation 0 threshold (default: 700)"),
    generation_1: int = Query(10, description="Generation 1 threshold (default: 10)"),
    generation_2: int = Query(10, description="Generation 2 threshold (default: 10)"),
) -> dict[str, Any]:
    """
    Configure Python garbage collection thresholds.

    Lower thresholds mean more frequent GC cycles (less memory, more CPU overhead).
    Higher thresholds mean less frequent GC cycles (more memory, less CPU overhead).

    Returns:
    - message: Confirmation message
    - previous_thresholds: Old threshold values
    - new_thresholds: New threshold values
    - objects_awaiting_collection: Current object count in gen-0
    - tip: Hint about when next collection will occur

    Query Parameters:
    - generation_0: Number of allocations before gen-0 collection (default: 700)
    - generation_1: Number of gen-0 collections before gen-1 collection (default: 10)
    - generation_2: Number of gen-1 collections before gen-2 collection (default: 10)

    Example for more aggressive collection:
    curl -X POST "http://localhost:4000/debug/memory/gc/configure?generation_0=500" -H "Authorization: Bearer sk-1234"

    Example for less aggressive collection:
    curl -X POST "http://localhost:4000/debug/memory/gc/configure?generation_0=1000" -H "Authorization: Bearer sk-1234"

    Monitor memory usage with GET /debug/memory/summary after changes.
    """
    # Get current thresholds for logging
    old_thresholds: Final = gc.get_threshold()

    # Set new thresholds with error handling
    try:
        gc.set_threshold(generation_0, generation_1, generation_2)
        verbose_proxy_logger.info(
            "GC thresholds updated from %s to (%s, %s, %s)", old_thresholds, generation_0, generation_1, generation_2
        )
    except Exception as e:
        verbose_proxy_logger.error("Failed to set GC thresholds: %s", e)
        raise HTTPException(status_code=500, detail=f"Failed to set GC thresholds: {e}")

    # Get current object count to show immediate impact
    current_count: Final = gc.get_count()[0]

    return {
        "message": "GC thresholds updated",
        "previous_thresholds": f"{old_thresholds[0]}, {old_thresholds[1]}, {old_thresholds[2]}",
        "new_thresholds": f"{generation_0}, {generation_1}, {generation_2}",
        "objects_awaiting_collection": current_count,
        "tip": f"Next collection will run after {generation_0 - current_count} more allocations",
    }


@router.get(
    "/otel-spans",
    dependencies=[Depends(user_api_key_auth)],
    include_in_schema=False,
)
async def get_otel_spans():
    from litellm.proxy.proxy_server import open_telemetry_logger

    if open_telemetry_logger is None:
        return {
            "otel_spans": [],
            "spans_grouped_by_parent": {},
            "most_recent_parent": None,
        }

    otel_exporter: Final = open_telemetry_logger.OTEL_EXPORTER
    if hasattr(otel_exporter, "get_finished_spans"):
        recorded_spans = otel_exporter.get_finished_spans()
    else:
        recorded_spans = []

    print("Spans: ", recorded_spans)  # noqa: T201

    most_recent_parent = None
    most_recent_start_time = 1000000
    spans_grouped_by_parent: Final = {}
    for span in recorded_spans:
        if span.parent is not None:
            parent_trace_id = span.parent.trace_id
            if parent_trace_id not in spans_grouped_by_parent:
                spans_grouped_by_parent[parent_trace_id] = []
            spans_grouped_by_parent[parent_trace_id].append(span.name)

            # check time of span
            if span.start_time > most_recent_start_time:
                most_recent_parent = parent_trace_id
                most_recent_start_time = span.start_time

    # these are otel spans - get the span name
    span_names: Final = [span.name for span in recorded_spans]
    return {
        "otel_spans": span_names,
        "spans_grouped_by_parent": spans_grouped_by_parent,
        "most_recent_parent": most_recent_parent,
    }


# Helper functions for debugging
def init_verbose_loggers():
    try:
        worker_config: Final = get_secret_str("WORKER_CONFIG")
        # if not, assume it's a json string
        if worker_config is None:
            return
        if os.path.isfile(worker_config):
            return
        _settings: Final = json.loads(worker_config)
        if not isinstance(_settings, dict):
            return

        debug: Final = _settings.get("debug", None)
        detailed_debug: Final = _settings.get("detailed_debug", None)
        if debug is True:  # this needs to be first, so users can see Router init debugg
            import logging

            from litellm._logging import (
                verbose_logger,
                verbose_proxy_logger,
                verbose_router_logger,
            )

            # this must ALWAYS remain logging.INFO, DO NOT MODIFY THIS
            verbose_logger.setLevel(level=logging.INFO)  # sets package logs to info
            verbose_router_logger.setLevel(level=logging.INFO)  # set router logs to info
            verbose_proxy_logger.setLevel(level=logging.INFO)  # set proxy logs to info
        if detailed_debug is True:
            import logging

            from litellm._logging import (
                verbose_logger,
                verbose_proxy_logger,
                verbose_router_logger,
            )

            verbose_logger.setLevel(level=logging.DEBUG)  # set package log to debug
            verbose_router_logger.setLevel(level=logging.DEBUG)  # set router logs to debug
            verbose_proxy_logger.setLevel(level=logging.DEBUG)  # set proxy logs to debug
        elif debug is False and detailed_debug is False:
            # users can control proxy debugging using env variable = 'LITELLM_LOG'
            litellm_log_setting: Final = os.environ.get("LITELLM_LOG", "")
            if litellm_log_setting is not None:
                if litellm_log_setting.upper() == "INFO":
                    import logging

                    from litellm._logging import (
                        verbose_proxy_logger,
                        verbose_router_logger,
                    )

                    # this must ALWAYS remain logging.INFO, DO NOT MODIFY THIS

                    verbose_router_logger.setLevel(level=logging.INFO)  # set router logs to info
                    verbose_proxy_logger.setLevel(level=logging.INFO)  # set proxy logs to info
                elif litellm_log_setting.upper() == "DEBUG":
                    import logging

                    from litellm._logging import (
                        verbose_proxy_logger,
                        verbose_router_logger,
                    )

                    verbose_router_logger.setLevel(level=logging.DEBUG)  # set router logs to info
                    verbose_proxy_logger.setLevel(level=logging.DEBUG)  # set proxy logs to debug
    except Exception as e:
        import logging

        logging.warning("Failed to init verbose loggers: %s", e)