From 3822681c6298b78c6ddf4c926cea09f637f69cc4 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Fri, 21 Nov 2025 01:10:03 +0000 Subject: [PATCH] feat: add PostHog feature flag for default mode experiment - Add PostHog feature flag support to TelemetryService - Implement default mode experiment (architect vs code) - Add feature flag checking to PostHogTelemetryClient - Initialize default mode based on feature flag for new users - Add tests for mode selection logic - Add documentation for the experiment The feature flag "default-mode-experiment" controls whether new users see "code" or "architect" as their default mode. Existing users are not affected. --- docs/DEFAULT_MODE_EXPERIMENT.md | 93 +++++++++++++++++++ .../telemetry/src/PostHogTelemetryClient.ts | 2 + packages/telemetry/src/TelemetryService.ts | 22 +++++ src/core/webview/ClineProvider.ts | 52 ++++++++++- .../__tests__/modes-feature-flag.spec.ts | 54 +++++++++++ src/shared/modes.ts | 36 ++++++- 6 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 docs/DEFAULT_MODE_EXPERIMENT.md create mode 100644 src/shared/__tests__/modes-feature-flag.spec.ts diff --git a/docs/DEFAULT_MODE_EXPERIMENT.md b/docs/DEFAULT_MODE_EXPERIMENT.md new file mode 100644 index 0000000000..4894a10542 --- /dev/null +++ b/docs/DEFAULT_MODE_EXPERIMENT.md @@ -0,0 +1,93 @@ +# Default Mode Feature Flag Experiment + +This document describes the PostHog feature flag experiment for controlling the default mode shown to new users. + +## Overview + +The default mode experiment allows us to A/B test whether new users should see **Code** mode or **Architect** mode as their default starting mode. + +## Feature Flag + +- **Flag Key**: `default-mode-experiment` +- **Location**: `src/shared/modes.ts` - `DEFAULT_MODE_FEATURE_FLAG` constant + +## How It Works + +1. When a new user first uses Roo Code (no mode has been set), the system checks the PostHog feature flag +2. Based on the flag value, the default mode is set: + + - `undefined` or `null`: Falls back to **Code** mode (control) + - `"architect"` (string): Sets **Architect** mode + - `"code"` (string): Sets **Code** mode + - `true` (boolean): Sets **Architect** mode (experiment variant) + - `false` (boolean): Sets **Code** mode (control variant) + +3. The mode is only set once for new users - existing users with a mode already set are not affected + +## Implementation Details + +### Key Files + +- `packages/telemetry/src/PostHogTelemetryClient.ts`: Added `getFeatureFlag()` method +- `packages/telemetry/src/TelemetryService.ts`: Added feature flag checking +- `src/shared/modes.ts`: Added feature flag constants and logic +- `src/core/webview/ClineProvider.ts`: Integrated feature flag checking on initialization + +### Code Flow + +``` +ClineProvider constructor + ↓ +initializeDefaultModeForNewUsers() + ↓ +Check if mode is already set + ↓ (only for new users) +TelemetryService.getFeatureFlag(DEFAULT_MODE_FEATURE_FLAG) + ↓ +PostHogTelemetryClient.getFeatureFlag() + ↓ +getDefaultModeFromFeatureFlag(flagValue) + ↓ +Set mode in global state +``` + +### Feature Flag Values + +Configure the feature flag in PostHog with one of these values: + +- **String values**: `"architect"` or `"code"` +- **Boolean values**: `true` (architect) or `false` (code) +- **Rollout**: Use PostHog's percentage rollout to A/B test + +## Testing + +Tests are located in: + +- `src/shared/__tests__/modes-feature-flag.spec.ts` - Tests for mode selection logic +- `packages/telemetry/src/__tests__/PostHogTelemetryClient.featureFlags.test.ts` - Tests for feature flag fetching + +Run tests: + +```bash +cd src && npx vitest run shared/__tests__/modes-feature-flag.spec.ts +``` + +## Metrics to Track + +Track in PostHog: + +- New user signups with each variant +- Task completion rates by default mode +- Mode switching behavior (do users stay in default mode or switch?) +- Time to first task completion +- User retention by initial mode + +## Rollback + +If issues arise, set the feature flag to `false` or `"code"` to revert all new users to Code mode. + +## Future Enhancements + +- Add telemetry event when default mode is set via feature flag +- Track which users were part of the experiment +- Consider adding more mode options to the experiment diff --git a/packages/telemetry/src/PostHogTelemetryClient.ts b/packages/telemetry/src/PostHogTelemetryClient.ts index d7f632f137..400d546559 100644 --- a/packages/telemetry/src/PostHogTelemetryClient.ts +++ b/packages/telemetry/src/PostHogTelemetryClient.ts @@ -5,6 +5,8 @@ import { TelemetryEventName, type TelemetryEvent } from "@roo-code/types" import { BaseTelemetryClient } from "./BaseTelemetryClient" +export type FeatureFlagValue = string | boolean | undefined + /** * PostHogTelemetryClient handles telemetry event tracking for the Roo Code extension. * Uses PostHog analytics to track user interactions and system events. diff --git a/packages/telemetry/src/TelemetryService.ts b/packages/telemetry/src/TelemetryService.ts index 8f4fbe0974..67e10d231e 100644 --- a/packages/telemetry/src/TelemetryService.ts +++ b/packages/telemetry/src/TelemetryService.ts @@ -6,6 +6,7 @@ import { TelemetryEventName, type TelemetrySetting, } from "@roo-code/types" +import type { FeatureFlagValue } from "./PostHogTelemetryClient" /** * TelemetryService wrapper class that defers initialization. @@ -243,6 +244,27 @@ export class TelemetryService { }) } + /** + * Checks a PostHog feature flag value + * @param flagKey The feature flag key to check + * @returns The feature flag value or undefined if not available + */ + public async getFeatureFlag(flagKey: string): Promise { + if (!this.isReady) { + return undefined + } + + // Only check PostHog client since it supports feature flags + const posthogClient = this.clients.find((client) => "getFeatureFlag" in client) + + if (posthogClient && "getFeatureFlag" in posthogClient) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return await (posthogClient as any).getFeatureFlag(flagKey) + } + + return undefined + } + /** * Checks if telemetry is currently enabled * @returns Whether telemetry is enabled diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index ff97d5f030..6f841324b3 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -52,7 +52,13 @@ import { findLast } from "../../shared/array" import { supportPrompt } from "../../shared/support-prompt" import { GlobalFileNames } from "../../shared/globalFileNames" import type { ExtensionMessage, ExtensionState, MarketplaceInstalledMetadata } from "../../shared/ExtensionMessage" -import { Mode, defaultModeSlug, getModeBySlug } from "../../shared/modes" +import { + Mode, + defaultModeSlug, + getModeBySlug, + DEFAULT_MODE_FEATURE_FLAG, + getDefaultModeFromFeatureFlag, +} from "../../shared/modes" import { experimentDefault } from "../../shared/experiments" import { formatLanguage } from "../../shared/language" import { WebviewMessage } from "../../shared/WebviewMessage" @@ -170,6 +176,9 @@ export class ClineProvider this.mdmService = mdmService this.updateGlobalState("codebaseIndexModels", EMBEDDING_MODEL_PROFILES) + // Initialize default mode based on feature flag for new users + this.initializeDefaultModeForNewUsers() + // Start configuration loading (which might trigger indexing) in the background. // Don't await, allowing activation to continue immediately. @@ -293,6 +302,47 @@ export class ClineProvider } } + /** + * Initialize default mode for new users based on PostHog feature flag + * This is called once during provider construction + */ + private async initializeDefaultModeForNewUsers(): Promise { + try { + // Only check if mode has never been set (new user) + const currentMode = this.getGlobalState("mode") + if (currentMode !== undefined) { + // User has already set a mode, don't override it + return + } + + // Check the feature flag + const featureFlagValue = await TelemetryService.instance.getFeatureFlag(DEFAULT_MODE_FEATURE_FLAG) + + // Get the mode based on the feature flag + const modeToSet = getDefaultModeFromFeatureFlag(featureFlagValue) + + // Set the mode in global state + await this.updateGlobalState("mode", modeToSet) + + this.log( + `[initializeDefaultModeForNewUsers] Set default mode to '${modeToSet}' based on feature flag value: ${featureFlagValue}`, + ) + } catch (error) { + // If anything fails, fallback to the default mode slug + this.log( + `[initializeDefaultModeForNewUsers] Error checking feature flag: ${ + error instanceof Error ? error.message : String(error) + }. Using default mode: ${defaultModeSlug}`, + ) + + // Only set if mode is still undefined + const currentMode = this.getGlobalState("mode") + if (currentMode === undefined) { + await this.updateGlobalState("mode", defaultModeSlug) + } + } + } + /** * Override EventEmitter's on method to match TaskProviderLike interface */ diff --git a/src/shared/__tests__/modes-feature-flag.spec.ts b/src/shared/__tests__/modes-feature-flag.spec.ts new file mode 100644 index 0000000000..0793279fa7 --- /dev/null +++ b/src/shared/__tests__/modes-feature-flag.spec.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from "vitest" +import { getDefaultModeFromFeatureFlag, DEFAULT_MODE_FEATURE_FLAG, modes } from "../modes" + +describe("Default Mode Feature Flag", () => { + describe("DEFAULT_MODE_FEATURE_FLAG", () => { + it("should have the correct feature flag key", () => { + expect(DEFAULT_MODE_FEATURE_FLAG).toBe("default-mode-experiment") + }) + }) + + describe("getDefaultModeFromFeatureFlag", () => { + it("should return 'code' mode when feature flag is undefined", () => { + const result = getDefaultModeFromFeatureFlag(undefined) + expect(result).toBe("code") + expect(result).toBe(modes[1].slug) + }) + + it("should return 'code' mode when feature flag is null", () => { + const result = getDefaultModeFromFeatureFlag(null as any) + expect(result).toBe("code") + }) + + it("should return 'architect' mode when feature flag is the string 'architect'", () => { + const result = getDefaultModeFromFeatureFlag("architect") + expect(result).toBe("architect") + expect(result).toBe(modes[0].slug) + }) + + it("should return 'architect' mode when feature flag is the string 'ARCHITECT' (case insensitive)", () => { + const result = getDefaultModeFromFeatureFlag("ARCHITECT") + expect(result).toBe("architect") + }) + + it("should return 'code' mode when feature flag is the string 'code'", () => { + const result = getDefaultModeFromFeatureFlag("code") + expect(result).toBe("code") + }) + + it("should return 'code' mode when feature flag is an unknown string", () => { + const result = getDefaultModeFromFeatureFlag("unknown-mode") + expect(result).toBe("code") + }) + + it("should return 'architect' mode when feature flag is true (experiment variant)", () => { + const result = getDefaultModeFromFeatureFlag(true) + expect(result).toBe("architect") + }) + + it("should return 'code' mode when feature flag is false (control variant)", () => { + const result = getDefaultModeFromFeatureFlag(false) + expect(result).toBe("code") + }) + }) +}) diff --git a/src/shared/modes.ts b/src/shared/modes.ts index f68d25c682..11ee2f4ca5 100644 --- a/src/shared/modes.ts +++ b/src/shared/modes.ts @@ -63,8 +63,40 @@ export function getToolsForMode(groups: readonly GroupEntry[]): string[] { // Main modes configuration as an ordered array export const modes = DEFAULT_MODES -// Export the default mode slug -export const defaultModeSlug = modes[0].slug +/** + * Feature flag key for default mode experiment + * This feature flag controls whether new users see 'code' or 'architect' as their default mode + */ +export const DEFAULT_MODE_FEATURE_FLAG = "default-mode-experiment" + +/** + * Gets the default mode slug based on PostHog feature flag + * @param featureFlagValue The value returned from PostHog feature flag check + * @returns The mode slug to use as default + */ +export function getDefaultModeFromFeatureFlag(featureFlagValue: string | boolean | undefined): Mode { + // If feature flag is not available or telemetry is disabled, use 'code' mode (index 1) + if (featureFlagValue === undefined || featureFlagValue === null) { + return modes[1].slug as Mode // 'code' mode + } + + // If feature flag is a string, compare it + if (typeof featureFlagValue === "string") { + const normalizedValue = featureFlagValue.toLowerCase() + if (normalizedValue === "architect") { + return modes[0].slug as Mode // 'architect' mode + } + // Default to 'code' for any other string value + return modes[1].slug as Mode + } + + // If feature flag is a boolean + // true = architect (experiment variant), false = code (control variant) + return featureFlagValue ? (modes[0].slug as Mode) : (modes[1].slug as Mode) +} + +// Export the default mode slug (defaults to 'code' mode which is at index 1) +export const defaultModeSlug = modes[1].slug // Helper functions export function getModeBySlug(slug: string, customModes?: ModeConfig[]): ModeConfig | undefined {