name: "Redis Chaos E2E"

on:
  workflow_dispatch:
  workflow_call:
    inputs:
      ref:
        description: "Commit SHA or ref to test. Defaults to the ref the workflow was triggered on"
        required: false
        type: string

permissions:
  contents: read

jobs:
  redis-chaos-e2e:
    runs-on: ubuntu-latest-16-cores
    timeout-minutes: 30
    services:
      postgres:
        image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36
        env:
          POSTGRES_USER: llmproxy
          POSTGRES_PASSWORD: dbpassword9090
          POSTGRES_DB: litellm
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U llmproxy"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
      valkey:
        image: valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa
        ports:
          - 6379:6379
        options: >-
          --health-cmd "valkey-cli ping"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
    env:
      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
      LITELLM_MASTER_KEY: sk-redis-chaos-e2e
      LITELLM_LOG: WARNING
      JSON_LOGS: "true"
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false
          ref: ${{ inputs.ref || github.sha }}

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Set up uv
        uses: ./.github/actions/setup-uv-with-retries
        with:
          version: "0.10.9"

      - name: Cache the Rust build
        uses: ./.github/actions/cache-cargo-build

      - name: Install dependencies
        run: |
          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy

      - name: Cache Prisma binaries
        uses: ./.github/actions/cache-prisma-binaries

      - name: Generate Prisma client
        run: |
          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma

      - name: Start a multi-worker proxy on the chaos config
        run: |
          nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_chaos_ci_config.yml --port 4000 --num_workers 4 > proxy.log 2>&1 &
          echo "E2E_PROXY_PID=$!" >> "$GITHUB_ENV"
          echo "E2E_PROXY_LOG=$(pwd)/proxy.log" >> "$GITHUB_ENV"
          for _ in $(seq 1 90); do
            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
              exit 0
            fi
            sleep 2
          done
          echo "proxy never became live"
          tail -n 100 proxy.log
          exit 1

      - name: Run the Redis chaos load test
        env:
          E2E_REDIS_CHAOS: "1"
          LITELLM_PROXY_URL: http://localhost:4000
          REDIS_HOST: 127.0.0.1
          REDIS_PORT: "6379"
        run: |
          uv run --no-sync pytest tests/e2e/load/test_redis_chaos_e2e.py -v --tb=short -rA -s

      - name: Show proxy log on failure
        if: failure()
        run: tail -n 300 proxy.log