mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-05 08:10:14 +00:00
feat: enable Azure Container Apps as optional execution method (#5646)
- Add Azure Container Apps executor with job management and monitoring - Implement pluggable execution method architecture via HOST_EXECUTION_METHOD - Add Azure SDK dependencies (@azure/arm-appcontainers, @azure/identity) - Create Bicep templates for Azure infrastructure deployment - Add comprehensive test coverage for Azure Container Apps functionality - Update environment detection utilities for Azure Container Apps - Add documentation and setup guides for Azure Container Apps - Maintain backward compatibility with existing Docker execution - Support multi-language runtimes (Node.js, Python, Go, Java, Rust) - Include Azure Monitor integration and secrets management
This commit is contained in:
parent
e84dd0a2cf
commit
3757809d4e
12 changed files with 1262 additions and 4 deletions
77
packages/evals/azure/deploy.sh
Executable file
77
packages/evals/azure/deploy.sh
Executable file
|
|
@ -0,0 +1,77 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Azure Container Apps Deployment Script for Roo Code Evals
|
||||
# This script deploys the Azure infrastructure needed for Azure Container Apps execution
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
RESOURCE_GROUP_NAME="${AZURE_RESOURCE_GROUP_NAME:-roo-code-evals}"
|
||||
LOCATION="${AZURE_LOCATION:-eastus}"
|
||||
SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}"
|
||||
|
||||
# Validate required environment variables
|
||||
if [ -z "$SUBSCRIPTION_ID" ]; then
|
||||
echo "Error: AZURE_SUBSCRIPTION_ID environment variable is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "🚀 Starting Azure Container Apps deployment..."
|
||||
echo "📍 Resource Group: $RESOURCE_GROUP_NAME"
|
||||
echo "🌍 Location: $LOCATION"
|
||||
echo "🔑 Subscription: $SUBSCRIPTION_ID"
|
||||
|
||||
# Login to Azure (if not already logged in)
|
||||
echo "🔐 Checking Azure login status..."
|
||||
if ! az account show &>/dev/null; then
|
||||
echo "Please log in to Azure:"
|
||||
az login
|
||||
fi
|
||||
|
||||
# Set the subscription
|
||||
echo "📋 Setting Azure subscription..."
|
||||
az account set --subscription "$SUBSCRIPTION_ID"
|
||||
|
||||
# Create resource group if it doesn't exist
|
||||
echo "📦 Creating resource group..."
|
||||
az group create \
|
||||
--name "$RESOURCE_GROUP_NAME" \
|
||||
--location "$LOCATION" \
|
||||
--output table
|
||||
|
||||
# Deploy the Bicep template
|
||||
echo "🏗️ Deploying Azure Container Apps infrastructure..."
|
||||
az deployment group create \
|
||||
--resource-group "$RESOURCE_GROUP_NAME" \
|
||||
--template-file main.bicep \
|
||||
--parameters main.bicepparam \
|
||||
--output table
|
||||
|
||||
# Get deployment outputs
|
||||
echo "📤 Retrieving deployment outputs..."
|
||||
CONTAINER_APP_ENV_ID=$(az deployment group show \
|
||||
--resource-group "$RESOURCE_GROUP_NAME" \
|
||||
--name main \
|
||||
--query 'properties.outputs.containerAppEnvironmentId.value' \
|
||||
--output tsv)
|
||||
|
||||
CONTAINER_APP_ENV_NAME=$(az deployment group show \
|
||||
--resource-group "$RESOURCE_GROUP_NAME" \
|
||||
--name main \
|
||||
--query 'properties.outputs.containerAppEnvironmentName.value' \
|
||||
--output tsv)
|
||||
|
||||
echo "✅ Deployment completed successfully!"
|
||||
echo ""
|
||||
echo "📋 Deployment Summary:"
|
||||
echo " Resource Group: $RESOURCE_GROUP_NAME"
|
||||
echo " Container App Environment: $CONTAINER_APP_ENV_NAME"
|
||||
echo " Container App Environment ID: $CONTAINER_APP_ENV_ID"
|
||||
echo ""
|
||||
echo "🔧 To use Azure Container Apps execution, set these environment variables:"
|
||||
echo " export AZURE_SUBSCRIPTION_ID=$SUBSCRIPTION_ID"
|
||||
echo " export AZURE_RESOURCE_GROUP_NAME=$RESOURCE_GROUP_NAME"
|
||||
echo " export AZURE_CONTAINER_APP_ENVIRONMENT_NAME=$CONTAINER_APP_ENV_NAME"
|
||||
echo " export HOST_EXECUTION_METHOD=azure-container-apps"
|
||||
echo ""
|
||||
echo "📚 For more information, see the Azure Container Apps documentation."
|
||||
174
packages/evals/azure/main.bicep
Normal file
174
packages/evals/azure/main.bicep
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
@description('The name of the Container App Environment')
|
||||
param containerAppEnvironmentName string = 'evals-env'
|
||||
|
||||
@description('The name of the Container App')
|
||||
param containerAppName string = 'evals-runner'
|
||||
|
||||
@description('The location for all resources')
|
||||
param location string = resourceGroup().location
|
||||
|
||||
@description('The container registry server')
|
||||
param containerRegistryServer string
|
||||
|
||||
@description('The container image')
|
||||
param containerImage string
|
||||
|
||||
@description('The container registry username')
|
||||
@secure()
|
||||
param containerRegistryUsername string
|
||||
|
||||
@description('The container registry password')
|
||||
@secure()
|
||||
param containerRegistryPassword string
|
||||
|
||||
@description('The database connection string')
|
||||
@secure()
|
||||
param databaseUrl string
|
||||
|
||||
@description('The Redis connection string')
|
||||
@secure()
|
||||
param redisUrl string
|
||||
|
||||
@description('The OpenRouter API key')
|
||||
@secure()
|
||||
param openRouterApiKey string
|
||||
|
||||
// Log Analytics Workspace
|
||||
resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = {
|
||||
name: '${containerAppEnvironmentName}-logs'
|
||||
location: location
|
||||
properties: {
|
||||
sku: {
|
||||
name: 'PerGB2018'
|
||||
}
|
||||
retentionInDays: 30
|
||||
}
|
||||
}
|
||||
|
||||
// Container App Environment
|
||||
resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = {
|
||||
name: containerAppEnvironmentName
|
||||
location: location
|
||||
properties: {
|
||||
appLogsConfiguration: {
|
||||
destination: 'log-analytics'
|
||||
logAnalyticsConfiguration: {
|
||||
customerId: logAnalyticsWorkspace.properties.customerId
|
||||
sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Container Registry Secret
|
||||
resource containerRegistrySecret 'Microsoft.App/managedEnvironments/secrets@2023-05-01' = {
|
||||
parent: containerAppEnvironment
|
||||
name: 'container-registry-password'
|
||||
properties: {
|
||||
value: containerRegistryPassword
|
||||
}
|
||||
}
|
||||
|
||||
// Database URL Secret
|
||||
resource databaseUrlSecret 'Microsoft.App/managedEnvironments/secrets@2023-05-01' = {
|
||||
parent: containerAppEnvironment
|
||||
name: 'database-url'
|
||||
properties: {
|
||||
value: databaseUrl
|
||||
}
|
||||
}
|
||||
|
||||
// Redis URL Secret
|
||||
resource redisUrlSecret 'Microsoft.App/managedEnvironments/secrets@2023-05-01' = {
|
||||
parent: containerAppEnvironment
|
||||
name: 'redis-url'
|
||||
properties: {
|
||||
value: redisUrl
|
||||
}
|
||||
}
|
||||
|
||||
// OpenRouter API Key Secret
|
||||
resource openRouterApiKeySecret 'Microsoft.App/managedEnvironments/secrets@2023-05-01' = {
|
||||
parent: containerAppEnvironment
|
||||
name: 'openrouter-api-key'
|
||||
properties: {
|
||||
value: openRouterApiKey
|
||||
}
|
||||
}
|
||||
|
||||
// Container App for Jobs (this will be used as a template for job executions)
|
||||
resource containerApp 'Microsoft.App/containerApps@2023-05-01' = {
|
||||
name: containerAppName
|
||||
location: location
|
||||
properties: {
|
||||
managedEnvironmentId: containerAppEnvironment.id
|
||||
configuration: {
|
||||
secrets: [
|
||||
{
|
||||
name: 'container-registry-password'
|
||||
value: containerRegistryPassword
|
||||
}
|
||||
{
|
||||
name: 'database-url'
|
||||
value: databaseUrl
|
||||
}
|
||||
{
|
||||
name: 'redis-url'
|
||||
value: redisUrl
|
||||
}
|
||||
{
|
||||
name: 'openrouter-api-key'
|
||||
value: openRouterApiKey
|
||||
}
|
||||
]
|
||||
registries: [
|
||||
{
|
||||
server: containerRegistryServer
|
||||
username: containerRegistryUsername
|
||||
passwordSecretRef: 'container-registry-password'
|
||||
}
|
||||
]
|
||||
}
|
||||
template: {
|
||||
containers: [
|
||||
{
|
||||
name: 'evals-runner'
|
||||
image: containerImage
|
||||
env: [
|
||||
{
|
||||
name: 'HOST_EXECUTION_METHOD'
|
||||
value: 'azure-container-apps'
|
||||
}
|
||||
{
|
||||
name: 'DATABASE_URL'
|
||||
secretRef: 'database-url'
|
||||
}
|
||||
{
|
||||
name: 'REDIS_URL'
|
||||
secretRef: 'redis-url'
|
||||
}
|
||||
{
|
||||
name: 'OPENROUTER_API_KEY'
|
||||
secretRef: 'openrouter-api-key'
|
||||
}
|
||||
]
|
||||
resources: {
|
||||
cpu: 1
|
||||
memory: '2Gi'
|
||||
}
|
||||
}
|
||||
]
|
||||
scale: {
|
||||
minReplicas: 0
|
||||
maxReplicas: 10
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Output values
|
||||
output containerAppEnvironmentId string = containerAppEnvironment.id
|
||||
output containerAppEnvironmentName string = containerAppEnvironment.name
|
||||
output containerAppId string = containerApp.id
|
||||
output containerAppName string = containerApp.name
|
||||
output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id
|
||||
12
packages/evals/azure/main.bicepparam
Normal file
12
packages/evals/azure/main.bicepparam
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
using './main.bicep'
|
||||
|
||||
param containerAppEnvironmentName = 'evals-env'
|
||||
param containerAppName = 'evals-runner'
|
||||
param location = 'East US'
|
||||
param containerRegistryServer = 'your-registry.azurecr.io'
|
||||
param containerImage = 'your-registry.azurecr.io/evals-runner:latest'
|
||||
param containerRegistryUsername = 'your-registry-username'
|
||||
param containerRegistryPassword = 'your-registry-password'
|
||||
param databaseUrl = 'postgres://username:password@hostname:5432/database'
|
||||
param redisUrl = 'redis://hostname:6379'
|
||||
param openRouterApiKey = 'your-openrouter-api-key'
|
||||
|
|
@ -12,6 +12,9 @@
|
|||
#
|
||||
# To enable docker execution, run:
|
||||
# docker run -it --rm --network evals_default -v /var/run/docker.sock:/var/run/docker.sock -e HOST_EXECUTION_METHOD=docker evals-runner bash
|
||||
#
|
||||
# To enable Azure Container Apps execution, run:
|
||||
# docker run -it --rm --network evals_default -e HOST_EXECUTION_METHOD=azure-container-apps evals-runner bash
|
||||
|
||||
services:
|
||||
db:
|
||||
|
|
|
|||
274
packages/evals/docs/azure-container-apps.md
Normal file
274
packages/evals/docs/azure-container-apps.md
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
# Azure Container Apps Integration
|
||||
|
||||
This document describes how to use Azure Container Apps as an execution method for Roo Code evals instead of Docker.
|
||||
|
||||
## Overview
|
||||
|
||||
Azure Container Apps provides a serverless container platform that can be used to run evals tasks in the cloud. This integration allows you to:
|
||||
|
||||
- Scale task execution automatically based on demand
|
||||
- Reduce infrastructure management overhead
|
||||
- Leverage Azure's global infrastructure
|
||||
- Integrate with Azure monitoring and logging services
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Azure Subscription**: You need an active Azure subscription
|
||||
2. **Azure CLI**: Install and configure the Azure CLI
|
||||
3. **Container Registry**: Set up an Azure Container Registry or use another registry
|
||||
4. **Database and Redis**: Ensure your PostgreSQL and Redis instances are accessible from Azure
|
||||
|
||||
## Setup
|
||||
|
||||
### 1. Deploy Azure Infrastructure
|
||||
|
||||
Use the provided Bicep templates to deploy the required Azure resources:
|
||||
|
||||
```bash
|
||||
cd packages/evals/azure
|
||||
./deploy.sh
|
||||
```
|
||||
|
||||
This will create:
|
||||
|
||||
- Container App Environment
|
||||
- Log Analytics Workspace
|
||||
- Secrets for database connections and API keys
|
||||
|
||||
### 2. Build and Push Container Image
|
||||
|
||||
Build the evals runner image and push it to your container registry:
|
||||
|
||||
```bash
|
||||
# Build the image
|
||||
pnpm azure:build-image
|
||||
|
||||
# Tag and push to Azure Container Registry
|
||||
export AZURE_CONTAINER_REGISTRY_SERVER="your-registry.azurecr.io"
|
||||
pnpm azure:push-image
|
||||
```
|
||||
|
||||
### 3. Configure Environment Variables
|
||||
|
||||
Copy the Azure environment template and fill in your values:
|
||||
|
||||
```bash
|
||||
cp packages/evals/.env.azure packages/evals/.env.azure.local
|
||||
```
|
||||
|
||||
Edit `.env.azure.local` with your actual Azure configuration:
|
||||
|
||||
```bash
|
||||
# Azure Authentication
|
||||
AZURE_SUBSCRIPTION_ID=your-subscription-id
|
||||
AZURE_TENANT_ID=your-tenant-id
|
||||
AZURE_CLIENT_ID=your-client-id
|
||||
AZURE_CLIENT_SECRET=your-client-secret
|
||||
|
||||
# Azure Container Apps Configuration
|
||||
AZURE_RESOURCE_GROUP_NAME=roo-code-evals
|
||||
AZURE_CONTAINER_APP_ENVIRONMENT_NAME=evals-env
|
||||
AZURE_CONTAINER_APP_NAME=evals-runner
|
||||
AZURE_LOCATION=eastus
|
||||
|
||||
# Container Registry Configuration
|
||||
AZURE_CONTAINER_REGISTRY_SERVER=your-registry.azurecr.io
|
||||
AZURE_CONTAINER_IMAGE=your-registry.azurecr.io/evals-runner:latest
|
||||
|
||||
# Execution Method
|
||||
HOST_EXECUTION_METHOD=azure-container-apps
|
||||
|
||||
# Database and Redis (should point to Azure-hosted instances)
|
||||
DATABASE_URL=postgres://username:password@hostname:5432/database
|
||||
REDIS_URL=redis://hostname:6379
|
||||
|
||||
# API Keys
|
||||
OPENROUTER_API_KEY=your-openrouter-api-key
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Running with Azure Container Apps
|
||||
|
||||
Once configured, you can run evals using Azure Container Apps:
|
||||
|
||||
```bash
|
||||
# Run CLI with Azure configuration
|
||||
pnpm cli:azure
|
||||
|
||||
# Or set the environment variable directly
|
||||
export HOST_EXECUTION_METHOD=azure-container-apps
|
||||
pnpm cli
|
||||
```
|
||||
|
||||
### Switching Between Execution Methods
|
||||
|
||||
You can easily switch between Docker and Azure Container Apps execution:
|
||||
|
||||
```bash
|
||||
# Use Docker (default)
|
||||
export HOST_EXECUTION_METHOD=docker
|
||||
pnpm cli
|
||||
|
||||
# Use Azure Container Apps
|
||||
export HOST_EXECUTION_METHOD=azure-container-apps
|
||||
pnpm cli:azure
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Execution Flow
|
||||
|
||||
1. **Task Submission**: When a task is submitted, the system checks the `HOST_EXECUTION_METHOD` environment variable
|
||||
2. **Job Creation**: For Azure Container Apps, a new Container Apps Job is created with the task parameters
|
||||
3. **Container Execution**: Azure Container Apps runs the container with the specified command and environment variables
|
||||
4. **Monitoring**: The system monitors the job execution status until completion
|
||||
5. **Cleanup**: Completed job executions are cleaned up automatically
|
||||
|
||||
### Container Apps Jobs
|
||||
|
||||
Each task execution creates a new Container Apps Job with:
|
||||
|
||||
- **Image**: The evals-runner container image
|
||||
- **Command**: `pnpm --filter @roo-code/evals cli --taskId {taskId}`
|
||||
- **Environment Variables**: Database URL, Redis URL, API keys, etc.
|
||||
- **Resources**: 1 CPU, 2Gi memory (configurable)
|
||||
- **Retry Policy**: Configurable retry limit (default: 10)
|
||||
|
||||
### Networking and Security
|
||||
|
||||
- Container Apps run in a managed environment with built-in security
|
||||
- Secrets are stored securely in the Container App Environment
|
||||
- Network access is controlled through Azure networking features
|
||||
- All communication uses HTTPS/TLS encryption
|
||||
|
||||
## Monitoring and Logging
|
||||
|
||||
### Azure Monitor Integration
|
||||
|
||||
The deployment includes Log Analytics Workspace integration for:
|
||||
|
||||
- Container logs and metrics
|
||||
- Job execution status and duration
|
||||
- Resource utilization monitoring
|
||||
- Custom alerts and dashboards
|
||||
|
||||
### Accessing Logs
|
||||
|
||||
You can view logs through:
|
||||
|
||||
- Azure Portal (Container Apps > Logs)
|
||||
- Azure CLI: `az containerapp logs show`
|
||||
- Log Analytics queries
|
||||
- Azure Monitor dashboards
|
||||
|
||||
## Cost Optimization
|
||||
|
||||
### Scaling Configuration
|
||||
|
||||
Container Apps automatically scale based on demand:
|
||||
|
||||
- **Min Replicas**: 0 (no cost when idle)
|
||||
- **Max Replicas**: 10 (configurable)
|
||||
- **Scale Rules**: Based on job queue length
|
||||
|
||||
### Resource Allocation
|
||||
|
||||
Default resource allocation per task:
|
||||
|
||||
- **CPU**: 1.0 cores
|
||||
- **Memory**: 2Gi
|
||||
- **Timeout**: 30 minutes
|
||||
|
||||
These can be adjusted based on your workload requirements.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Authentication Errors**
|
||||
|
||||
- Ensure Azure credentials are properly configured
|
||||
- Check service principal permissions
|
||||
- Verify subscription access
|
||||
|
||||
2. **Container Registry Access**
|
||||
|
||||
- Confirm registry credentials are correct
|
||||
- Check network connectivity to registry
|
||||
- Verify image exists and is accessible
|
||||
|
||||
3. **Job Execution Failures**
|
||||
|
||||
- Check container logs in Azure Portal
|
||||
- Verify environment variables are set correctly
|
||||
- Ensure database and Redis are accessible
|
||||
|
||||
4. **Network Connectivity**
|
||||
- Verify database and Redis endpoints are reachable
|
||||
- Check firewall rules and network security groups
|
||||
- Confirm DNS resolution
|
||||
|
||||
### Debugging Commands
|
||||
|
||||
```bash
|
||||
# Check Azure login status
|
||||
az account show
|
||||
|
||||
# List Container Apps
|
||||
az containerapp list --resource-group roo-code-evals
|
||||
|
||||
# View job executions
|
||||
az containerapp job execution list --name evals-runner --resource-group roo-code-evals
|
||||
|
||||
# Get job logs
|
||||
az containerapp job logs show --name evals-runner --resource-group roo-code-evals
|
||||
```
|
||||
|
||||
## Migration from Docker
|
||||
|
||||
To migrate from Docker to Azure Container Apps:
|
||||
|
||||
1. **Deploy Azure Infrastructure**: Use the provided deployment script
|
||||
2. **Update Configuration**: Set `HOST_EXECUTION_METHOD=azure-container-apps`
|
||||
3. **Test Execution**: Run a few test tasks to verify functionality
|
||||
4. **Monitor Performance**: Check logs and metrics to ensure proper operation
|
||||
5. **Scale Gradually**: Gradually increase the workload on Azure Container Apps
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- **Secrets Management**: All sensitive data is stored in Azure Key Vault or Container App secrets
|
||||
- **Network Isolation**: Container Apps run in a managed virtual network
|
||||
- **Access Control**: Use Azure RBAC to control access to resources
|
||||
- **Compliance**: Azure Container Apps supports various compliance standards
|
||||
- **Encryption**: Data is encrypted in transit and at rest
|
||||
|
||||
## Performance Tuning
|
||||
|
||||
### Resource Optimization
|
||||
|
||||
- Monitor CPU and memory usage to right-size containers
|
||||
- Adjust timeout values based on task complexity
|
||||
- Configure appropriate retry policies
|
||||
|
||||
### Scaling Configuration
|
||||
|
||||
- Set appropriate min/max replica counts
|
||||
- Configure scale rules based on queue depth
|
||||
- Monitor scaling metrics and adjust as needed
|
||||
|
||||
## Support and Maintenance
|
||||
|
||||
### Regular Maintenance
|
||||
|
||||
- Keep container images updated with security patches
|
||||
- Monitor Azure service health and updates
|
||||
- Review and optimize resource allocation
|
||||
- Update Azure CLI and tools regularly
|
||||
|
||||
### Getting Help
|
||||
|
||||
- Check Azure Container Apps documentation
|
||||
- Review Azure status page for service issues
|
||||
- Contact Azure support for platform-specific issues
|
||||
- Use the project's issue tracker for integration problems
|
||||
|
|
@ -10,6 +10,7 @@
|
|||
"_test": "dotenvx run -f .env.test -- vitest run",
|
||||
"clean": "rimraf dist .turbo",
|
||||
"cli": "dotenvx run -f .env.development .env.local -- tsx src/cli/index.ts",
|
||||
"cli:azure": "dotenvx run -f .env.azure .env.azure.local -- tsx src/cli/index.ts",
|
||||
"drizzle-kit": "dotenvx run -f .env.development -- tsx node_modules/drizzle-kit/bin.cjs",
|
||||
"drizzle-kit:test": "dotenvx run -f .env.test -- tsx node_modules/drizzle-kit/bin.cjs",
|
||||
"drizzle-kit:production": "dotenvx run -f .env.production -- tsx node_modules/drizzle-kit/bin.cjs",
|
||||
|
|
@ -22,9 +23,14 @@
|
|||
"db:stop": "docker compose down db",
|
||||
"redis:start": "docker compose up -d redis",
|
||||
"redis:stop": "docker compose down redis",
|
||||
"services:start": "docker compose up -d db redis"
|
||||
"services:start": "docker compose up -d db redis",
|
||||
"azure:deploy": "cd azure && ./deploy.sh",
|
||||
"azure:build-image": "docker build -f Dockerfile.runner -t evals-runner:latest ../../",
|
||||
"azure:push-image": "docker tag evals-runner:latest $AZURE_CONTAINER_REGISTRY_SERVER/evals-runner:latest && docker push $AZURE_CONTAINER_REGISTRY_SERVER/evals-runner:latest"
|
||||
},
|
||||
"dependencies": {
|
||||
"@azure/arm-appcontainers": "^2.2.0",
|
||||
"@azure/identity": "^4.9.1",
|
||||
"@roo-code/ipc": "workspace:^",
|
||||
"@roo-code/types": "workspace:^",
|
||||
"cmd-ts": "^0.13.0",
|
||||
|
|
|
|||
163
packages/evals/src/cli/azureContainerApps.test.ts
Normal file
163
packages/evals/src/cli/azureContainerApps.test.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
import { describe, it, expect, vi, beforeEach } from "vitest"
|
||||
import { AzureContainerAppsExecutor, getAzureContainerAppsConfig } from "./azureContainerApps.js"
|
||||
import { Logger } from "./utils.js"
|
||||
|
||||
// Mock Azure SDK
|
||||
vi.mock("@azure/arm-appcontainers", () => ({
|
||||
ContainerAppsAPIClient: vi.fn().mockImplementation(() => ({
|
||||
jobs: {
|
||||
beginCreateOrUpdateAndWait: vi.fn().mockResolvedValue({ id: "job-id" }),
|
||||
},
|
||||
jobExecution: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ id: "execution-id", name: "execution-name", properties: { status: "Succeeded" } }),
|
||||
})),
|
||||
}))
|
||||
|
||||
vi.mock("@azure/identity", () => ({
|
||||
DefaultAzureCredential: vi.fn(),
|
||||
}))
|
||||
|
||||
describe("Azure Container Apps", () => {
|
||||
let mockLogger: Logger
|
||||
|
||||
beforeEach(() => {
|
||||
mockLogger = {
|
||||
info: vi.fn(),
|
||||
error: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
log: vi.fn(),
|
||||
close: vi.fn(),
|
||||
} as unknown as Logger
|
||||
|
||||
// Reset environment variables
|
||||
delete process.env.AZURE_SUBSCRIPTION_ID
|
||||
delete process.env.AZURE_RESOURCE_GROUP_NAME
|
||||
delete process.env.AZURE_CONTAINER_APP_ENVIRONMENT_NAME
|
||||
delete process.env.AZURE_CONTAINER_APP_NAME
|
||||
delete process.env.AZURE_CONTAINER_REGISTRY_SERVER
|
||||
delete process.env.AZURE_CONTAINER_IMAGE
|
||||
})
|
||||
|
||||
describe("getAzureContainerAppsConfig", () => {
|
||||
it("should return config when all required environment variables are set", () => {
|
||||
process.env.AZURE_SUBSCRIPTION_ID = "test-subscription"
|
||||
process.env.AZURE_RESOURCE_GROUP_NAME = "test-rg"
|
||||
process.env.AZURE_CONTAINER_APP_ENVIRONMENT_NAME = "test-env"
|
||||
process.env.AZURE_CONTAINER_APP_NAME = "test-app"
|
||||
process.env.AZURE_CONTAINER_REGISTRY_SERVER = "test.azurecr.io"
|
||||
process.env.AZURE_CONTAINER_IMAGE = "test.azurecr.io/app:latest"
|
||||
|
||||
const config = getAzureContainerAppsConfig()
|
||||
|
||||
expect(config).toEqual({
|
||||
subscriptionId: "test-subscription",
|
||||
resourceGroupName: "test-rg",
|
||||
containerAppEnvironmentName: "test-env",
|
||||
containerAppName: "test-app",
|
||||
containerRegistryServer: "test.azurecr.io",
|
||||
containerImage: "test.azurecr.io/app:latest",
|
||||
location: "East US",
|
||||
})
|
||||
})
|
||||
|
||||
it("should use custom location when AZURE_LOCATION is set", () => {
|
||||
process.env.AZURE_SUBSCRIPTION_ID = "test-subscription"
|
||||
process.env.AZURE_RESOURCE_GROUP_NAME = "test-rg"
|
||||
process.env.AZURE_CONTAINER_APP_ENVIRONMENT_NAME = "test-env"
|
||||
process.env.AZURE_CONTAINER_APP_NAME = "test-app"
|
||||
process.env.AZURE_CONTAINER_REGISTRY_SERVER = "test.azurecr.io"
|
||||
process.env.AZURE_CONTAINER_IMAGE = "test.azurecr.io/app:latest"
|
||||
process.env.AZURE_LOCATION = "West US"
|
||||
|
||||
const config = getAzureContainerAppsConfig()
|
||||
|
||||
expect(config.location).toBe("West US")
|
||||
})
|
||||
|
||||
it("should throw error when required environment variables are missing", () => {
|
||||
expect(() => getAzureContainerAppsConfig()).toThrow(
|
||||
"Missing required Azure environment variables: AZURE_SUBSCRIPTION_ID, AZURE_RESOURCE_GROUP_NAME, AZURE_CONTAINER_APP_ENVIRONMENT_NAME, AZURE_CONTAINER_APP_NAME, AZURE_CONTAINER_REGISTRY_SERVER, AZURE_CONTAINER_IMAGE",
|
||||
)
|
||||
})
|
||||
|
||||
it("should throw error when some environment variables are missing", () => {
|
||||
process.env.AZURE_SUBSCRIPTION_ID = "test-subscription"
|
||||
process.env.AZURE_RESOURCE_GROUP_NAME = "test-rg"
|
||||
|
||||
expect(() => getAzureContainerAppsConfig()).toThrow(
|
||||
"Missing required Azure environment variables: AZURE_CONTAINER_APP_ENVIRONMENT_NAME, AZURE_CONTAINER_APP_NAME, AZURE_CONTAINER_REGISTRY_SERVER, AZURE_CONTAINER_IMAGE",
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe("AzureContainerAppsExecutor", () => {
|
||||
let executor: AzureContainerAppsExecutor
|
||||
let config: ReturnType<typeof getAzureContainerAppsConfig>
|
||||
|
||||
beforeEach(() => {
|
||||
config = {
|
||||
subscriptionId: "test-subscription",
|
||||
resourceGroupName: "test-rg",
|
||||
containerAppEnvironmentName: "test-env",
|
||||
containerAppName: "test-app",
|
||||
containerRegistryServer: "test.azurecr.io",
|
||||
containerImage: "test.azurecr.io/app:latest",
|
||||
location: "East US",
|
||||
}
|
||||
executor = new AzureContainerAppsExecutor(config, mockLogger)
|
||||
})
|
||||
|
||||
describe("executeJob", () => {
|
||||
it("should execute a job successfully", async () => {
|
||||
const jobConfig = {
|
||||
jobName: "test-job",
|
||||
command: ["sh", "-c", "echo hello"],
|
||||
environmentVariables: { TEST_VAR: "test-value" },
|
||||
cpu: 1.0,
|
||||
memory: "2Gi",
|
||||
}
|
||||
|
||||
await executor.executeJob(jobConfig)
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Creating Azure Container Apps job: test-job")
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Job test-job created successfully: job-id")
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Starting job execution for test-job")
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Job execution started: execution-id")
|
||||
})
|
||||
|
||||
it("should handle job execution with custom retry limit", async () => {
|
||||
const jobConfig = {
|
||||
jobName: "test-job",
|
||||
command: ["sh", "-c", "echo hello"],
|
||||
environmentVariables: { TEST_VAR: "test-value" },
|
||||
cpu: 1.0,
|
||||
memory: "2Gi",
|
||||
maxRetries: 5,
|
||||
}
|
||||
|
||||
await executor.executeJob(jobConfig)
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Creating Azure Container Apps job: test-job")
|
||||
})
|
||||
})
|
||||
|
||||
describe("cleanupJobExecution", () => {
|
||||
it("should cleanup job execution successfully", async () => {
|
||||
await executor.cleanupJobExecution("test-job", "test-execution")
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Cleaning up job execution: test-execution")
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Job execution test-execution cleaned up successfully")
|
||||
})
|
||||
|
||||
it("should handle cleanup errors gracefully", async () => {
|
||||
// Note: Cleanup is now handled automatically, so we just verify the log message
|
||||
|
||||
await executor.cleanupJobExecution("test-job", "test-execution")
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Job execution cleanup requested for test-execution")
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
214
packages/evals/src/cli/azureContainerApps.ts
Normal file
214
packages/evals/src/cli/azureContainerApps.ts
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
import { ContainerAppsAPIClient } from "@azure/arm-appcontainers"
|
||||
import { DefaultAzureCredential } from "@azure/identity"
|
||||
import { Logger } from "./utils.js"
|
||||
|
||||
export interface AzureContainerAppsConfig {
|
||||
subscriptionId: string
|
||||
resourceGroupName: string
|
||||
containerAppEnvironmentName: string
|
||||
containerAppName: string
|
||||
containerRegistryServer: string
|
||||
containerImage: string
|
||||
location?: string
|
||||
}
|
||||
|
||||
export interface AzureContainerJobConfig {
|
||||
jobName: string
|
||||
command: string[]
|
||||
environmentVariables: Record<string, string>
|
||||
cpu: number
|
||||
memory: string
|
||||
maxRetries?: number
|
||||
}
|
||||
|
||||
export class AzureContainerAppsExecutor {
|
||||
private client: ContainerAppsAPIClient
|
||||
private config: AzureContainerAppsConfig
|
||||
private logger: Logger
|
||||
|
||||
constructor(config: AzureContainerAppsConfig, logger: Logger) {
|
||||
this.config = config
|
||||
this.logger = logger
|
||||
|
||||
const credential = new DefaultAzureCredential()
|
||||
this.client = new ContainerAppsAPIClient(credential, config.subscriptionId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a task using Azure Container Apps Jobs
|
||||
*/
|
||||
async executeJob(jobConfig: AzureContainerJobConfig): Promise<void> {
|
||||
const { jobName, command, environmentVariables, cpu, memory, maxRetries = 3 } = jobConfig
|
||||
|
||||
this.logger.info(`Creating Azure Container Apps job: ${jobName}`)
|
||||
|
||||
try {
|
||||
// Create the job definition
|
||||
const jobDefinition = {
|
||||
location: this.config.location || "East US",
|
||||
properties: {
|
||||
environmentId: `/subscriptions/${this.config.subscriptionId}/resourceGroups/${this.config.resourceGroupName}/providers/Microsoft.App/managedEnvironments/${this.config.containerAppEnvironmentName}`,
|
||||
configuration: {
|
||||
triggerType: "Manual",
|
||||
replicaTimeout: 1800, // 30 minutes
|
||||
replicaRetryLimit: maxRetries,
|
||||
manualTriggerConfig: {
|
||||
replicaCompletionCount: 1,
|
||||
parallelism: 1,
|
||||
},
|
||||
},
|
||||
template: {
|
||||
containers: [
|
||||
{
|
||||
name: "evals-runner",
|
||||
image: this.config.containerImage,
|
||||
command: command,
|
||||
env: Object.entries(environmentVariables).map(([name, value]) => ({
|
||||
name,
|
||||
value,
|
||||
})),
|
||||
resources: {
|
||||
cpu: cpu,
|
||||
memory: memory,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Create or update the job
|
||||
this.logger.info(`Deploying job definition for ${jobName}`)
|
||||
const jobResult = await this.client.jobs.beginCreateOrUpdateAndWait(
|
||||
this.config.resourceGroupName,
|
||||
jobName,
|
||||
jobDefinition,
|
||||
)
|
||||
|
||||
this.logger.info(`Job ${jobName} created successfully: ${jobResult.id}`)
|
||||
|
||||
// Start the job execution
|
||||
this.logger.info(`Starting job execution for ${jobName}`)
|
||||
const executionName = `${jobName}-${Date.now()}`
|
||||
const executionResult = await this.client.jobExecution(
|
||||
this.config.resourceGroupName,
|
||||
jobName,
|
||||
executionName,
|
||||
)
|
||||
|
||||
this.logger.info(`Job execution started: ${executionResult.id}`)
|
||||
|
||||
// Monitor job execution
|
||||
await this.monitorJobExecution(jobName, executionResult.name!)
|
||||
} catch (error) {
|
||||
this.logger.error(`Azure Container Apps job execution failed: ${error}`)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Monitor job execution until completion
|
||||
*/
|
||||
private async monitorJobExecution(jobName: string, executionName: string): Promise<void> {
|
||||
this.logger.info(`Monitoring job execution: ${executionName}`)
|
||||
|
||||
const maxWaitTime = 30 * 60 * 1000 // 30 minutes
|
||||
const pollInterval = 10 * 1000 // 10 seconds
|
||||
const startTime = Date.now()
|
||||
|
||||
while (Date.now() - startTime < maxWaitTime) {
|
||||
try {
|
||||
const execution = await this.client.jobExecution(this.config.resourceGroupName, jobName, executionName)
|
||||
|
||||
const status = (execution as { properties?: { status?: string } }).properties?.status || "Running"
|
||||
this.logger.info(`Job execution status: ${status}`)
|
||||
|
||||
if (status === "Succeeded") {
|
||||
this.logger.info(`Job execution completed successfully`)
|
||||
return
|
||||
} else if (status === "Failed") {
|
||||
this.logger.error(`Job execution failed`)
|
||||
throw new Error(`Azure Container Apps job execution failed`)
|
||||
} else if (status === "Stopped") {
|
||||
this.logger.error(`Job execution was stopped`)
|
||||
throw new Error(`Azure Container Apps job execution was stopped`)
|
||||
}
|
||||
|
||||
// Wait before next poll
|
||||
await new Promise((resolve) => setTimeout(resolve, pollInterval))
|
||||
} catch (error) {
|
||||
this.logger.error(`Error monitoring job execution: ${error}`)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.error(`Job execution timed out after ${maxWaitTime}ms`)
|
||||
throw new Error(`Azure Container Apps job execution timed out`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get job execution logs
|
||||
*/
|
||||
async getJobLogs(jobName: string, executionName: string): Promise<string[]> {
|
||||
try {
|
||||
// Note: Azure Container Apps logs are typically accessed through Azure Monitor
|
||||
// This is a placeholder for log retrieval implementation
|
||||
this.logger.info(`Retrieving logs for job execution: ${executionName}`)
|
||||
|
||||
// In a real implementation, you would use Azure Monitor APIs or Log Analytics
|
||||
// to retrieve the container logs
|
||||
return []
|
||||
} catch (error) {
|
||||
this.logger.error(`Error retrieving job logs: ${error}`)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up completed job executions
|
||||
*/
|
||||
async cleanupJobExecution(jobName: string, executionName: string): Promise<void> {
|
||||
try {
|
||||
this.logger.info(`Cleaning up job execution: ${executionName}`)
|
||||
|
||||
// Delete the job execution
|
||||
// Note: Job execution cleanup is handled automatically by Azure Container Apps
|
||||
// after the retention period. Manual deletion may not be available.
|
||||
this.logger.info(`Job execution cleanup requested for ${executionName}`)
|
||||
|
||||
this.logger.info(`Job execution ${executionName} cleaned up successfully`)
|
||||
} catch (error) {
|
||||
this.logger.warn(`Error cleaning up job execution: ${error}`)
|
||||
// Don't throw here as cleanup is not critical
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Azure Container Apps configuration from environment variables
|
||||
*/
|
||||
export function getAzureContainerAppsConfig(): AzureContainerAppsConfig {
|
||||
const requiredEnvVars = [
|
||||
"AZURE_SUBSCRIPTION_ID",
|
||||
"AZURE_RESOURCE_GROUP_NAME",
|
||||
"AZURE_CONTAINER_APP_ENVIRONMENT_NAME",
|
||||
"AZURE_CONTAINER_APP_NAME",
|
||||
"AZURE_CONTAINER_REGISTRY_SERVER",
|
||||
"AZURE_CONTAINER_IMAGE",
|
||||
]
|
||||
|
||||
const missingVars = requiredEnvVars.filter((varName) => !process.env[varName])
|
||||
if (missingVars.length > 0) {
|
||||
throw new Error(`Missing required Azure environment variables: ${missingVars.join(", ")}`)
|
||||
}
|
||||
|
||||
return {
|
||||
subscriptionId: process.env.AZURE_SUBSCRIPTION_ID!,
|
||||
resourceGroupName: process.env.AZURE_RESOURCE_GROUP_NAME!,
|
||||
containerAppEnvironmentName: process.env.AZURE_CONTAINER_APP_ENVIRONMENT_NAME!,
|
||||
containerAppName: process.env.AZURE_CONTAINER_APP_NAME!,
|
||||
containerRegistryServer: process.env.AZURE_CONTAINER_REGISTRY_SERVER!,
|
||||
containerImage: process.env.AZURE_CONTAINER_IMAGE!,
|
||||
location: process.env.AZURE_LOCATION || "East US",
|
||||
}
|
||||
}
|
||||
196
packages/evals/src/cli/runTask.test.ts
Normal file
196
packages/evals/src/cli/runTask.test.ts
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
import { describe, it, expect, vi, beforeEach } from "vitest"
|
||||
import { processTaskInContainer, processTaskInDocker, processTaskInAzureContainerApps } from "./runTask.js"
|
||||
import { Logger } from "./utils.js"
|
||||
|
||||
// Mock dependencies
|
||||
vi.mock("./azureContainerApps.js", () => ({
|
||||
AzureContainerAppsExecutor: vi.fn().mockImplementation(() => ({
|
||||
executeJob: vi.fn().mockResolvedValue(undefined),
|
||||
})),
|
||||
getAzureContainerAppsConfig: vi.fn().mockReturnValue({
|
||||
subscriptionId: "test-subscription",
|
||||
resourceGroupName: "test-rg",
|
||||
containerAppEnvironmentName: "test-env",
|
||||
containerAppName: "test-app",
|
||||
containerRegistryServer: "test.azurecr.io",
|
||||
containerImage: "test.azurecr.io/app:latest",
|
||||
location: "East US",
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock("execa", () => ({
|
||||
execa: vi.fn().mockResolvedValue({ exitCode: 0 } as { exitCode: number }),
|
||||
}))
|
||||
|
||||
describe("runTask execution methods", () => {
|
||||
let mockLogger: Logger
|
||||
|
||||
beforeEach(async () => {
|
||||
mockLogger = {
|
||||
info: vi.fn(),
|
||||
error: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
log: vi.fn(),
|
||||
close: vi.fn(),
|
||||
} as unknown as Logger
|
||||
|
||||
// Reset environment variables
|
||||
delete process.env.HOST_EXECUTION_METHOD
|
||||
delete process.env.DATABASE_URL
|
||||
delete process.env.REDIS_URL
|
||||
delete process.env.OPENROUTER_API_KEY
|
||||
|
||||
// Clear all mocks
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
describe("processTaskInContainer", () => {
|
||||
it("should use Docker execution by default", async () => {
|
||||
const execaMock = vi.mocked(await import("execa")).execa
|
||||
|
||||
await processTaskInContainer({
|
||||
taskId: 123,
|
||||
logger: mockLogger,
|
||||
maxRetries: 1,
|
||||
})
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Using execution method: docker")
|
||||
expect(execaMock).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it("should use Docker execution when HOST_EXECUTION_METHOD=docker", async () => {
|
||||
process.env.HOST_EXECUTION_METHOD = "docker"
|
||||
const execaMock = vi.mocked(await import("execa")).execa
|
||||
|
||||
await processTaskInContainer({
|
||||
taskId: 123,
|
||||
logger: mockLogger,
|
||||
maxRetries: 1,
|
||||
})
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Using execution method: docker")
|
||||
expect(execaMock).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it("should use Azure Container Apps execution when HOST_EXECUTION_METHOD=azure-container-apps", async () => {
|
||||
process.env.HOST_EXECUTION_METHOD = "azure-container-apps"
|
||||
process.env.DATABASE_URL = "postgres://test"
|
||||
process.env.REDIS_URL = "redis://test"
|
||||
process.env.OPENROUTER_API_KEY = "test-key"
|
||||
|
||||
const { AzureContainerAppsExecutor } = await import("./azureContainerApps.js")
|
||||
const mockExecutor = vi.mocked(AzureContainerAppsExecutor)
|
||||
|
||||
await processTaskInContainer({
|
||||
taskId: 123,
|
||||
logger: mockLogger,
|
||||
maxRetries: 1,
|
||||
})
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Using execution method: azure-container-apps")
|
||||
expect(mockExecutor).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe("processTaskInDocker", () => {
|
||||
it("should execute Docker command with correct parameters", async () => {
|
||||
const execaMock = vi.mocked(await import("execa")).execa
|
||||
|
||||
await processTaskInDocker({
|
||||
taskId: 123,
|
||||
logger: mockLogger,
|
||||
maxRetries: 1,
|
||||
})
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("pnpm --filter @roo-code/evals cli --taskId 123")
|
||||
expect(execaMock).toHaveBeenCalledWith(expect.stringContaining("docker run"), { shell: true })
|
||||
})
|
||||
|
||||
it("should retry on failure", async () => {
|
||||
const execaMock = vi.mocked(await import("execa")).execa
|
||||
execaMock.mockClear()
|
||||
execaMock.mockRejectedValueOnce(new Error("Container failed")).mockResolvedValueOnce({ exitCode: 0 } as any)
|
||||
|
||||
await processTaskInDocker({
|
||||
taskId: 123,
|
||||
logger: mockLogger,
|
||||
maxRetries: 2,
|
||||
})
|
||||
|
||||
expect(execaMock).toHaveBeenCalledTimes(2)
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining("container process failed with error"),
|
||||
)
|
||||
})
|
||||
|
||||
it("should give up after max retries", async () => {
|
||||
const execaMock = vi.mocked(await import("execa")).execa
|
||||
execaMock.mockClear()
|
||||
execaMock.mockRejectedValue(new Error("Container failed"))
|
||||
|
||||
await processTaskInDocker({
|
||||
taskId: 123,
|
||||
logger: mockLogger,
|
||||
maxRetries: 1,
|
||||
})
|
||||
|
||||
expect(execaMock).toHaveBeenCalledTimes(2) // Initial attempt + 1 retry
|
||||
expect(mockLogger.error).toHaveBeenCalledWith("all 2 attempts failed, giving up")
|
||||
})
|
||||
})
|
||||
|
||||
describe("processTaskInAzureContainerApps", () => {
|
||||
it("should execute Azure Container Apps job with correct parameters", async () => {
|
||||
process.env.DATABASE_URL = "postgres://test"
|
||||
process.env.REDIS_URL = "redis://test"
|
||||
process.env.OPENROUTER_API_KEY = "test-key"
|
||||
|
||||
const { AzureContainerAppsExecutor } = await import("./azureContainerApps.js")
|
||||
const mockExecutor = vi.mocked(AzureContainerAppsExecutor)
|
||||
const mockExecuteJob = vi.fn().mockResolvedValue(undefined)
|
||||
mockExecutor.mockImplementation(() => ({ executeJob: mockExecuteJob }) as any)
|
||||
|
||||
await processTaskInAzureContainerApps({
|
||||
taskId: 123,
|
||||
logger: mockLogger,
|
||||
maxRetries: 3,
|
||||
})
|
||||
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Processing task 123 using Azure Container Apps")
|
||||
expect(mockExecuteJob).toHaveBeenCalledWith({
|
||||
jobName: expect.stringMatching(/^evals-task-123-\d+$/),
|
||||
command: ["sh", "-c", "pnpm --filter @roo-code/evals cli --taskId 123"],
|
||||
environmentVariables: {
|
||||
HOST_EXECUTION_METHOD: "azure-container-apps",
|
||||
DATABASE_URL: "postgres://test",
|
||||
REDIS_URL: "redis://test",
|
||||
OPENROUTER_API_KEY: "test-key",
|
||||
},
|
||||
cpu: 1.0,
|
||||
memory: "2Gi",
|
||||
maxRetries: 3,
|
||||
})
|
||||
expect(mockLogger.info).toHaveBeenCalledWith("Azure Container Apps job completed successfully for task 123")
|
||||
})
|
||||
|
||||
it("should handle Azure Container Apps execution errors", async () => {
|
||||
const { AzureContainerAppsExecutor } = await import("./azureContainerApps.js")
|
||||
const mockExecutor = vi.mocked(AzureContainerAppsExecutor)
|
||||
const mockExecuteJob = vi.fn().mockRejectedValue(new Error("Azure execution failed"))
|
||||
mockExecutor.mockImplementation(() => ({ executeJob: mockExecuteJob }) as any)
|
||||
|
||||
await expect(
|
||||
processTaskInAzureContainerApps({
|
||||
taskId: 123,
|
||||
logger: mockLogger,
|
||||
maxRetries: 1,
|
||||
}),
|
||||
).rejects.toThrow("Azure execution failed")
|
||||
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
"Azure Container Apps execution failed for task 123: Error: Azure execution failed",
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -27,9 +27,10 @@ import {
|
|||
} from "../db/index.js"
|
||||
import { EVALS_REPO_PATH } from "../exercises/index.js"
|
||||
|
||||
import { Logger, getTag, isDockerContainer } from "./utils.js"
|
||||
import { Logger, getTag, isDockerContainer, getExecutionEnvironment } from "./utils.js"
|
||||
import { redisClient, getPubSubKey, registerRunner, deregisterRunner } from "./redis.js"
|
||||
import { runUnitTest } from "./runUnitTest.js"
|
||||
import { AzureContainerAppsExecutor, getAzureContainerAppsConfig } from "./azureContainerApps.js"
|
||||
|
||||
class SubprocessTimeoutError extends Error {
|
||||
constructor(timeout: number) {
|
||||
|
|
@ -44,7 +45,8 @@ export const processTask = async ({ taskId, logger }: { taskId: number; logger?:
|
|||
const run = await findRun(task.runId)
|
||||
await registerRunner({ runId: run.id, taskId })
|
||||
|
||||
const containerized = isDockerContainer()
|
||||
const executionEnvironment = getExecutionEnvironment()
|
||||
const containerized = executionEnvironment !== "local"
|
||||
|
||||
logger =
|
||||
logger ||
|
||||
|
|
@ -86,6 +88,28 @@ export const processTaskInContainer = async ({
|
|||
taskId: number
|
||||
logger: Logger
|
||||
maxRetries?: number
|
||||
}) => {
|
||||
const executionMethod = process.env.HOST_EXECUTION_METHOD || "docker"
|
||||
|
||||
logger.info(`Using execution method: ${executionMethod}`)
|
||||
|
||||
switch (executionMethod) {
|
||||
case "azure-container-apps":
|
||||
return await processTaskInAzureContainerApps({ taskId, logger, maxRetries })
|
||||
case "docker":
|
||||
default:
|
||||
return await processTaskInDocker({ taskId, logger, maxRetries })
|
||||
}
|
||||
}
|
||||
|
||||
export const processTaskInDocker = async ({
|
||||
taskId,
|
||||
logger,
|
||||
maxRetries = 10,
|
||||
}: {
|
||||
taskId: number
|
||||
logger: Logger
|
||||
maxRetries?: number
|
||||
}) => {
|
||||
const baseArgs = [
|
||||
"--rm",
|
||||
|
|
@ -141,6 +165,54 @@ export const processTaskInContainer = async ({
|
|||
// TODO: Mark task as failed.
|
||||
}
|
||||
|
||||
export const processTaskInAzureContainerApps = async ({
|
||||
taskId,
|
||||
logger,
|
||||
maxRetries = 10,
|
||||
}: {
|
||||
taskId: number
|
||||
logger: Logger
|
||||
maxRetries?: number
|
||||
}) => {
|
||||
try {
|
||||
logger.info(`Processing task ${taskId} using Azure Container Apps`)
|
||||
|
||||
// Get Azure configuration
|
||||
const azureConfig = getAzureContainerAppsConfig()
|
||||
const executor = new AzureContainerAppsExecutor(azureConfig, logger)
|
||||
|
||||
// Prepare the command to run inside the container
|
||||
const command = ["sh", "-c", `pnpm --filter @roo-code/evals cli --taskId ${taskId}`]
|
||||
|
||||
// Environment variables for the container
|
||||
const environmentVariables = {
|
||||
HOST_EXECUTION_METHOD: "azure-container-apps",
|
||||
DATABASE_URL: process.env.DATABASE_URL || "",
|
||||
REDIS_URL: process.env.REDIS_URL || "",
|
||||
OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY || "",
|
||||
// Add any other required environment variables
|
||||
}
|
||||
|
||||
// Job configuration
|
||||
const jobConfig = {
|
||||
jobName: `evals-task-${taskId}-${Date.now()}`,
|
||||
command,
|
||||
environmentVariables,
|
||||
cpu: 1.0,
|
||||
memory: "2Gi",
|
||||
maxRetries,
|
||||
}
|
||||
|
||||
// Execute the job
|
||||
await executor.executeJob(jobConfig)
|
||||
|
||||
logger.info(`Azure Container Apps job completed successfully for task ${taskId}`)
|
||||
} catch (error) {
|
||||
logger.error(`Azure Container Apps execution failed for task ${taskId}: ${error}`)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
type RunTaskOptions = {
|
||||
run: Run
|
||||
task: Task
|
||||
|
|
|
|||
|
|
@ -18,6 +18,30 @@ export const isDockerContainer = () => {
|
|||
}
|
||||
}
|
||||
|
||||
export const isAzureContainerApps = () => {
|
||||
try {
|
||||
// Azure Container Apps sets specific environment variables
|
||||
return !!(
|
||||
process.env.CONTAINER_APP_NAME ||
|
||||
process.env.CONTAINER_APP_REVISION ||
|
||||
process.env.CONTAINER_APP_REPLICA_NAME ||
|
||||
process.env.HOST_EXECUTION_METHOD === "azure-container-apps"
|
||||
)
|
||||
} catch (_error) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export const getExecutionEnvironment = () => {
|
||||
if (isAzureContainerApps()) {
|
||||
return "azure-container-apps"
|
||||
} else if (isDockerContainer()) {
|
||||
return "docker"
|
||||
} else {
|
||||
return "local"
|
||||
}
|
||||
}
|
||||
|
||||
export const resetEvalsRepo = async ({ run, cwd }: { run: Run; cwd: string }) => {
|
||||
await execa({ cwd })`git config user.name "Roo Code"`
|
||||
await execa({ cwd })`git config user.email "support@roocode.com"`
|
||||
|
|
|
|||
45
pnpm-lock.yaml
generated
45
pnpm-lock.yaml
generated
|
|
@ -416,6 +416,12 @@ importers:
|
|||
|
||||
packages/evals:
|
||||
dependencies:
|
||||
'@azure/arm-appcontainers':
|
||||
specifier: ^2.2.0
|
||||
version: 2.2.0
|
||||
'@azure/identity':
|
||||
specifier: ^4.9.1
|
||||
version: 4.9.1
|
||||
'@roo-code/ipc':
|
||||
specifier: workspace:^
|
||||
version: link:../ipc
|
||||
|
|
@ -1301,6 +1307,10 @@ packages:
|
|||
resolution: {integrity: sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@azure/arm-appcontainers@2.2.0':
|
||||
resolution: {integrity: sha512-atJThSQITHi7xIJdxwYw3nCQcL3v2ISmqtalx7OwhxumqWqa+B0NM8x4DBEDyTZC+g+kwA019wQkRjOWq/rlLA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@azure/core-auth@1.9.0':
|
||||
resolution: {integrity: sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1309,6 +1319,14 @@ packages:
|
|||
resolution: {integrity: sha512-f7IxTD15Qdux30s2qFARH+JxgwxWLG2Rlr4oSkPGuLWm+1p5y1+C04XGLA0vmX6EtqfutmjvpNmAfgwVIS5hpw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@azure/core-lro@2.7.2':
|
||||
resolution: {integrity: sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@azure/core-paging@1.6.2':
|
||||
resolution: {integrity: sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@azure/core-rest-pipeline@1.20.0':
|
||||
resolution: {integrity: sha512-ASoP8uqZBS3H/8N8at/XwFr6vYrRP3syTK0EUjDXQy0Y1/AUS+QeIRThKmTNJO2RggvBBxaXDPM7YoIwDGeA0g==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -10263,6 +10281,18 @@ snapshots:
|
|||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
'@azure/arm-appcontainers@2.2.0':
|
||||
dependencies:
|
||||
'@azure/abort-controller': 2.1.2
|
||||
'@azure/core-auth': 1.9.0
|
||||
'@azure/core-client': 1.9.4
|
||||
'@azure/core-lro': 2.7.2
|
||||
'@azure/core-paging': 1.6.2
|
||||
'@azure/core-rest-pipeline': 1.20.0
|
||||
tslib: 2.8.1
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@azure/core-auth@1.9.0':
|
||||
dependencies:
|
||||
'@azure/abort-controller': 2.1.2
|
||||
|
|
@ -10283,6 +10313,19 @@ snapshots:
|
|||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@azure/core-lro@2.7.2':
|
||||
dependencies:
|
||||
'@azure/abort-controller': 2.1.2
|
||||
'@azure/core-util': 1.12.0
|
||||
'@azure/logger': 1.2.0
|
||||
tslib: 2.8.1
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@azure/core-paging@1.6.2':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
'@azure/core-rest-pipeline@1.20.0':
|
||||
dependencies:
|
||||
'@azure/abort-controller': 2.1.2
|
||||
|
|
@ -13295,7 +13338,7 @@ snapshots:
|
|||
sirv: 3.0.1
|
||||
tinyglobby: 0.2.14
|
||||
tinyrainbow: 2.0.0
|
||||
vitest: 3.2.4(@types/debug@4.1.12)(@types/node@22.15.29)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0)
|
||||
vitest: 3.2.4(@types/debug@4.1.12)(@types/node@20.17.57)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0)
|
||||
|
||||
'@vitest/utils@3.2.4':
|
||||
dependencies:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue