litellm/.github/workflows/llm-translation-testing.yml

name: LLM Translation Test Results for Release Candidates

on:
  workflow_dispatch:
    inputs:
      release_candidate_tag:
        description: 'Release candidate tag/version'
        required: true
        type: string
  push:
    tags:
      - 'v*-rc*'  # Triggers on release candidate tags like v1.0.0-rc1
  
jobs:
  fetch-llm-translation-results:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
      
      - name: Install CircleCI CLI
        run: |
          curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/main/install.sh | sudo bash
      
      - name: Create test results directory
        run: mkdir -p test-results
      
      - name: Fetch LLM Translation Test Results from CircleCI
        env:
          CIRCLE_TOKEN: ${{ secrets.CIRCLE_TOKEN }}
        run: |
          # Get the actual commit SHA after checkout
          COMMIT_SHA=$(git rev-parse HEAD)
          echo "Fetching CircleCI results for commit: $COMMIT_SHA"
          
          # Search for pipelines across all branches for this commit
          PIPELINE_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
            "https://circleci.com/api/v2/project/github/BerriAI/litellm/pipeline" | \
            jq -r ".items[] | select(.vcs.revision == \"$COMMIT_SHA\") | .id" | head -1)
          
          # If not found, try searching recent pipelines more broadly
          if [ -z "$PIPELINE_INFO" ]; then
            echo "Trying broader search for recent pipelines..."
            PIPELINE_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
              "https://circleci.com/api/v2/project/github/BerriAI/litellm/pipeline" | \
              jq -r ".items[0:20][] | select(.vcs.revision == \"$COMMIT_SHA\") | .id" | head -1)
          fi
          
          if [ -z "$PIPELINE_INFO" ]; then
            echo "No CircleCI pipeline found for commit $COMMIT_SHA"
            echo "Checking recent pipelines..."
            curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
              "https://circleci.com/api/v2/project/github/BerriAI/litellm/pipeline" | \
              jq -r ".items[0:5][] | \"Pipeline: \(.id) | Commit: \(.vcs.revision) | Branch: \(.vcs.branch) | Status: \(.state)\""
            echo "Creating placeholder test results..."
            echo '<?xml version="1.0" encoding="utf-8"?>' > test-results/junit.xml
            echo '<testsuite name="llm_translation" tests="0" failures="0" errors="0" skipped="0" time="0">' >> test-results/junit.xml
            echo '<system-out>No CircleCI results found for this commit</system-out>' >> test-results/junit.xml
            echo '</testsuite>' >> test-results/junit.xml
            exit 0
          fi
          
          echo "Found pipeline: $PIPELINE_INFO"
          
          # Get workflow info - look for any workflow that might contain tests
          WORKFLOW_ID=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
            "https://circleci.com/api/v2/pipeline/$PIPELINE_INFO/workflow" | \
            jq -r '.items[] | select(.name | test("test|Test|TEST")) | .id' | head -1)
          
          if [ -z "$WORKFLOW_ID" ]; then
            echo "No test workflow found, trying any workflow..."
            WORKFLOW_ID=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
              "https://circleci.com/api/v2/pipeline/$PIPELINE_INFO/workflow" | \
              jq -r '.items[0].id')
          fi
          
          if [ -z "$WORKFLOW_ID" ]; then
            echo "No workflow found in pipeline"
            exit 1
          fi
          
          echo "Found workflow: $WORKFLOW_ID"
          
          # Get job info for llm_translation tests
          JOB_INFO=$(curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
            "https://circleci.com/api/v2/workflow/$WORKFLOW_ID/job" | \
            jq -r '.items[] | select(.name | test("llm_translation|llm-translation")) | select(.status == "success" or .status == "failed") | .job_number' | head -1)
          
          if [ -z "$JOB_INFO" ]; then
            echo "No completed llm_translation job found, checking all jobs:"
            curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
              "https://circleci.com/api/v2/workflow/$WORKFLOW_ID/job" | \
              jq -r '.items[] | "Job: \(.name) | Status: \(.status) | Number: \(.job_number)"'
            echo "Creating placeholder test results..."
            echo '<?xml version="1.0" encoding="utf-8"?>' > test-results/junit.xml
            echo '<testsuite name="llm_translation" tests="0" failures="0" errors="0" skipped="0" time="0">' >> test-results/junit.xml
            echo '<system-out>No llm_translation job found in CircleCI</system-out>' >> test-results/junit.xml
            echo '</testsuite>' >> test-results/junit.xml
            exit 0
          fi
          
          echo "Found job: $JOB_INFO"
          
          # Download artifacts
          ARTIFACT_COUNT=0
          curl -s -H "Circle-Token: $CIRCLE_TOKEN" \
            "https://circleci.com/api/v2/project/github/BerriAI/litellm/$JOB_INFO/artifacts" | \
            jq -r '.items[] | select(.path | test("junit|coverage|report|xml|html")) | .url' | \
            while read -r artifact_url; do
              if [ -n "$artifact_url" ]; then
                filename=$(basename "$artifact_url" | sed 's/[?&].*//')
                echo "Downloading artifact: $filename from $artifact_url"
                if curl -s -H "Circle-Token: $CIRCLE_TOKEN" -o "test-results/$filename" "$artifact_url"; then
                  echo "Successfully downloaded $filename"
                  ARTIFACT_COUNT=$((ARTIFACT_COUNT + 1))
                else
                  echo "Failed to download $filename"
                fi
              fi
            done
          
          # If no artifacts found, create placeholder
          if [ ! -f "test-results/junit.xml" ] && [ "$ARTIFACT_COUNT" -eq 0 ]; then
            echo "No test artifacts found, creating placeholder..."
            echo '<?xml version="1.0" encoding="utf-8"?>' > test-results/junit.xml
            echo '<testsuite name="llm_translation" tests="0" failures="0" errors="0" skipped="0" time="0">' >> test-results/junit.xml
            echo '<system-out>Test artifacts not available from CircleCI</system-out>' >> test-results/junit.xml
            echo '</testsuite>' >> test-results/junit.xml
          else
            echo "Successfully retrieved $ARTIFACT_COUNT artifacts"
          fi
        continue-on-error: true
      
      - name: Generate test summary
        run: |
          echo "# LLM Translation Testing Results" > test-results/summary.md
          echo "" >> test-results/summary.md
          echo "**Release Candidate:** ${{ github.event.inputs.release_candidate_tag || github.ref_name }}" >> test-results/summary.md
          echo "**Fetched Date:** $(date)" >> test-results/summary.md
          echo "**Commit:** ${{ github.sha }}" >> test-results/summary.md
          echo "**Source:** CircleCI Pipeline" >> test-results/summary.md
          echo "" >> test-results/summary.md
          
          # Parse junit.xml for test statistics if it exists
          if [ -f "test-results/junit.xml" ]; then
            python -c "
          import xml.etree.ElementTree as ET
          try:
              tree = ET.parse('test-results/junit.xml')
              root = tree.getroot()
              tests = root.get('tests', '0')
              failures = root.get('failures', '0')
              errors = root.get('errors', '0')
              skipped = root.get('skipped', '0')
              time = root.get('time', '0')
              
              print(f'**Total Tests:** {tests}')
              print(f'**Passed:** {int(tests) - int(failures) - int(errors) - int(skipped)}')
              print(f'**Failed:** {failures}')
              print(f'**Errors:** {errors}')
              print(f'**Skipped:** {skipped}')
              print(f'**Duration:** {time} seconds')
          except Exception as e:
              print(f'Could not parse test results: {e}')
          " >> test-results/summary.md
          fi
          
          echo "" >> test-results/summary.md
          echo "## Test Files Covered" >> test-results/summary.md
          ls tests/llm_translation/*.py | sed 's/^/- /' >> test-results/summary.md
      
      - name: Upload test artifacts
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: llm-translation-test-artifacts-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
          path: test-results/
          retention-days: 30