litellm/enterprise/litellm_enterprise/proxy/enterprise_routes.py

from fastapi import APIRouter
from fastapi.responses import Response
from litellm_enterprise.enterprise_callbacks.send_emails.endpoints import (
    router as email_events_router,
)

from . import ui_crud_endpoints  # side-effect: registers extra UI settings
from .audit_logging_endpoints import router as audit_logging_router
from .liteadmin import router as liteadmin_router
from .management_endpoints import management_endpoints_router
from .utils import should_block_robots

__all__ = ["router", "ui_crud_endpoints"]

router = APIRouter()
router.include_router(email_events_router)
router.include_router(audit_logging_router)
router.include_router(liteadmin_router)
router.include_router(management_endpoints_router)


@router.get("/robots.txt")
async def get_robots():
    """
    Block all web crawlers from indexing the proxy server endpoints
    This is useful for ensuring that the API endpoints aren't indexed by search engines
    """
    if should_block_robots():
        return Response(content="User-agent: *\nDisallow: /", media_type="text/plain")
    else:
        return Response(status_code=404)