feat: Add Google Cloud and Azure TTS integration

- Add TTS provider interface and types with pricing constants
- Implement Google Cloud TTS provider with voice selection and cost tracking
- Implement Azure TTS provider with SSML support and region configuration
- Create default OS TTS provider wrapping existing say module
- Add provider factory for managing multiple TTS providers
- Update TTS utility to support provider switching and usage tracking
- Add monthly usage tracking for free tier management
- Update webview message handler to support TTS provider operations
- Add TTS-related settings to global settings schema
This commit is contained in:
Roo Code 2025-11-04 23:56:11 +00:00
parent 54745fc234
commit 82a3e7b3a7
8 changed files with 969 additions and 28 deletions

View file

@ -131,7 +131,18 @@ export const globalSettingsSchema = z.object({
.optional(),
ttsEnabled: z.boolean().optional(),
ttsProvider: z.enum(["default", "google-cloud", "azure"]).optional(),
ttsSpeed: z.number().optional(),
ttsSelectedVoice: z.string().optional(),
ttsGoogleVoice: z.string().optional(),
ttsAzureVoice: z.string().optional(),
ttsMonthlyUsage: z
.object({
google: z.number().optional(),
azure: z.number().optional(),
lastResetDate: z.string().optional(),
})
.optional(),
soundEnabled: z.boolean().optional(),
soundVolume: z.number().optional(),
@ -245,6 +256,9 @@ export const SECRET_STATE_KEYS = [
// Global secrets that are part of GlobalSettings (not ProviderSettings)
export const GLOBAL_SECRET_KEYS = [
"openRouterImageApiKey", // For image generation
"googleCloudTtsApiKey", // For Google Cloud TTS
"azureTtsApiKey", // For Microsoft Azure TTS
"azureTtsRegion", // Azure TTS region/endpoint
] as const
// Type for the actual secret storage keys

View file

@ -43,7 +43,17 @@ import { getTheme } from "../../integrations/theme/getTheme"
import { discoverChromeHostUrl, tryChromeHostUrl } from "../../services/browser/browserDiscovery"
import { searchWorkspaceFiles } from "../../services/search/file-search"
import { fileExistsAtPath } from "../../utils/fs"
import { playTts, setTtsEnabled, setTtsSpeed, stopTts } from "../../utils/tts"
import {
playTts,
setTtsEnabled,
setTtsSpeed,
stopTts,
initializeTts,
getAvailableVoices,
getCurrentProviderName,
getTtsUsageStats,
isProviderConfigured,
} from "../../utils/tts"
import { searchCommits } from "../../utils/git"
import { exportSettings, importSettingsWithFeedback } from "../config/importExport"
import { getOpenAiModels } from "../../api/providers/openai"
@ -427,6 +437,9 @@ export const webviewMessageHandler = async (
switch (message.type) {
case "webviewDidLaunch":
// Initialize TTS system with context proxy
initializeTts(provider.contextProxy)
// Load custom modes first
const customModes = await provider.customModesManager.getCustomModes()
await updateGlobalState("customModes", customModes)
@ -1302,6 +1315,73 @@ export const webviewMessageHandler = async (
case "stopTts":
stopTts()
break
case "ttsProvider":
// Handle TTS provider selection
if (message.text) {
await updateGlobalState("ttsProvider", message.text)
await provider.postStateToWebview()
}
break
case "ttsGoogleVoice":
// Handle Google Cloud TTS voice selection
if (message.text) {
await updateGlobalState("ttsGoogleVoice", message.text)
await provider.postStateToWebview()
}
break
case "ttsAzureVoice":
// Handle Azure TTS voice selection
if (message.text) {
await updateGlobalState("ttsAzureVoice", message.text)
await provider.postStateToWebview()
}
break
case "ttsAzureRegion":
// Handle Azure region selection
if (message.text) {
await updateGlobalState("ttsAzureRegion", message.text)
await provider.postStateToWebview()
}
break
case "getTtsVoices":
// Get available voices for the current provider
try {
const voices = await getAvailableVoices()
await provider.postMessageToWebview({
type: "ttsVoices",
voices: voices,
})
} catch (error) {
console.error("Failed to get TTS voices:", error)
await provider.postMessageToWebview({
type: "ttsVoices",
voices: [],
})
}
break
case "getTtsUsageStats":
// Get TTS usage statistics
try {
const stats = getTtsUsageStats()
await provider.postMessageToWebview({
type: "ttsUsageStats",
stats: stats,
})
} catch (error) {
console.error("Failed to get TTS usage stats:", error)
}
break
case "checkTtsProviderConfigured":
// Check if a specific provider is configured
if (message.text) {
const isConfigured = isProviderConfigured(message.text)
await provider.postMessageToWebview({
type: "ttsProviderConfigured",
provider: message.text,
configured: isConfigured,
})
}
break
case "diffEnabled":
const diffEnabled = message.bool ?? true
await updateGlobalState("diffEnabled", diffEnabled)

View file

@ -1,3 +1,7 @@
import { TtsProviderFactory } from "./tts/provider-factory"
import { TtsProviderInterface, TtsVoice } from "./tts/types"
import { ContextProxy } from "../core/config/ContextProxy"
interface Say {
speak: (text: string, voice?: string, speed?: number, callback?: (err?: string) => void) => void
stop: () => void
@ -14,16 +18,66 @@ type QueueItem = {
}
let isTtsEnabled = false
export const setTtsEnabled = (enabled: boolean) => (isTtsEnabled = enabled)
let speed = 1.0
export const setTtsSpeed = (newSpeed: number) => (speed = newSpeed)
let sayInstance: Say | undefined = undefined
let queue: QueueItem[] = []
let currentProvider: TtsProviderInterface | null = null
let providerFactory: TtsProviderFactory | null = null
let contextProxy: ContextProxy | null = null
/**
* Initialize the TTS system with a context proxy
*/
export const initializeTts = (proxy: ContextProxy) => {
contextProxy = proxy
providerFactory = TtsProviderFactory.getInstance(proxy)
}
/**
* Set whether TTS is enabled
*/
export const setTtsEnabled = (enabled: boolean) => {
isTtsEnabled = enabled
}
/**
* Set the TTS speed
*/
export const setTtsSpeed = (newSpeed: number) => {
speed = newSpeed
}
/**
* Get available voices for the current provider
*/
export const getAvailableVoices = async (): Promise<TtsVoice[]> => {
if (!providerFactory) {
console.error("TTS not initialized. Call initializeTts() first.")
return []
}
try {
const provider = providerFactory.getCurrentProvider()
return await provider.getVoices()
} catch (error) {
console.error("Failed to get available voices:", error)
return []
}
}
/**
* Get the current TTS provider name
*/
export const getCurrentProviderName = (): string => {
if (!contextProxy) {
return "default"
}
return contextProxy.getValue("ttsProvider" as any) || "default"
}
/**
* Play TTS for a message
*/
export const playTts = async (message: string, options: PlayTtsOptions = {}) => {
if (!isTtsEnabled) {
return
@ -32,50 +86,205 @@ export const playTts = async (message: string, options: PlayTtsOptions = {}) =>
try {
queue.push({ message, options })
await processQueue()
} catch (error) {}
} catch (error) {
console.error("TTS playback error:", error)
}
}
/**
* Stop TTS playback
*/
export const stopTts = () => {
sayInstance?.stop()
sayInstance = undefined
// Stop the current provider's playback if it's the default provider
const providerName = getCurrentProviderName()
if (providerName === "default" && sayInstance) {
sayInstance.stop()
sayInstance = undefined
}
// For cloud providers, we may need to add audio player cancellation
// This would require tracking the audio playback process
queue = []
}
/**
* Process the TTS queue
*/
const processQueue = async (): Promise<void> => {
if (!isTtsEnabled || sayInstance) {
return
}
const item = queue.shift()
if (!item) {
return
}
if (!providerFactory) {
console.error("TTS not initialized. Call initializeTts() first.")
await processQueue()
return
}
try {
const { message: nextUtterance, options } = item
const provider = providerFactory.getCurrentProvider()
const providerName = getCurrentProviderName()
await new Promise<void>((resolve, reject) => {
const say: Say = require("say")
sayInstance = say
options.onStart?.()
say.speak(nextUtterance, undefined, speed, (err) => {
options.onStop?.()
if (err) {
reject(new Error(err))
} else {
resolve()
}
sayInstance = undefined
})
})
// Handle different providers
if (providerName === "default") {
// Use the existing Say-based implementation for default provider
await playWithDefaultProvider(nextUtterance, options)
} else {
// Use cloud providers (Google Cloud or Azure)
await playWithCloudProvider(provider, providerName, nextUtterance, options)
}
await processQueue()
} catch (error: any) {
console.error("TTS processing error:", error)
sayInstance = undefined
await processQueue()
}
}
/**
* Play TTS using the default OS provider
*/
const playWithDefaultProvider = async (text: string, options: PlayTtsOptions): Promise<void> => {
return new Promise<void>((resolve, reject) => {
const say: Say = require("say")
sayInstance = say
options.onStart?.()
say.speak(text, undefined, speed, (err) => {
options.onStop?.()
if (err) {
reject(new Error(err))
} else {
resolve()
}
sayInstance = undefined
})
})
}
/**
* Play TTS using a cloud provider
*/
const playWithCloudProvider = async (
provider: TtsProviderInterface,
providerName: string,
text: string,
options: PlayTtsOptions,
): Promise<void> => {
if (!contextProxy || !providerFactory) {
throw new Error("TTS not initialized")
}
// Get the selected voice for this provider
let voiceId = "default"
if (providerName === "google-cloud") {
voiceId = contextProxy.getValue("ttsGoogleVoice" as any) || "en-US-Wavenet-D"
} else if (providerName === "azure") {
voiceId = contextProxy.getValue("ttsAzureVoice" as any) || "en-US-JennyNeural"
}
options.onStart?.()
try {
// Check if within free tier
const usage = providerFactory.getMonthlyUsage(providerName)
const isWithinFreeTier = await provider.isWithinFreeTier(usage.characters)
if (!isWithinFreeTier) {
console.warn(
`TTS: Monthly free tier exceeded for ${providerName}. Current usage: ${usage.characters} characters`,
)
// Optionally fall back to default provider or show a warning to the user
}
// Calculate cost
const cost = provider.calculateCost(text)
if (cost > 0) {
console.debug(`TTS cost for ${text.length} characters: $${cost.toFixed(6)}`)
}
// Synthesize speech
const audioBuffer = await provider.synthesizeSpeech(text, voiceId, speed)
// Play the audio
await playAudioBuffer(audioBuffer)
// Update usage tracking
await providerFactory.updateUsageTracking(text.length)
} finally {
options.onStop?.()
}
}
/**
* Play an audio buffer using system commands
*/
const playAudioBuffer = async (audioBuffer: Buffer): Promise<void> => {
const fs = require("fs").promises
const path = require("path")
const { exec } = require("child_process")
const { promisify } = require("util")
const execAsync = promisify(exec)
const os = require("os")
const tempFile = path.join(os.tmpdir(), `tts-${Date.now()}.mp3`)
await fs.writeFile(tempFile, audioBuffer)
try {
const platform = process.platform
let command: string
if (platform === "darwin") {
// macOS
command = `afplay "${tempFile}"`
} else if (platform === "win32") {
// Windows
command = `powershell -c "(New-Object Media.SoundPlayer '${tempFile}').PlaySync()"`
} else {
// Linux - try multiple players
command = `aplay "${tempFile}" || mpg123 "${tempFile}" || ffplay -nodisp -autoexit "${tempFile}"`
}
await execAsync(command)
} finally {
// Clean up temp file
await fs.unlink(tempFile).catch(() => {})
}
}
/**
* Get TTS usage statistics for the current month
*/
export const getTtsUsageStats = () => {
if (!providerFactory || !contextProxy) {
return null
}
const providerName = getCurrentProviderName()
const usage = providerFactory.getMonthlyUsage(providerName)
const provider = providerFactory.getCurrentProvider()
return {
provider: providerName,
monthlyUsage: usage,
isWithinFreeTier: provider.isWithinFreeTier(usage.characters),
}
}
/**
* Check if a specific provider is configured
*/
export const isProviderConfigured = (providerName: string): boolean => {
if (!providerFactory) {
return false
}
return providerFactory.isProviderAvailable(providerName)
}

View file

@ -0,0 +1,145 @@
import { TtsProviderInterface } from "./types"
import { DefaultTtsProvider } from "./providers/default-tts"
import { GoogleCloudTtsProvider } from "./providers/google-cloud-tts"
import { AzureTtsProvider } from "./providers/azure-tts"
import { ContextProxy } from "../../core/config/ContextProxy"
export class TtsProviderFactory {
private static instance: TtsProviderFactory
private providers: Map<string, TtsProviderInterface> = new Map()
private currentProvider: TtsProviderInterface | null = null
private constructor(private contextProxy: ContextProxy) {
this.initializeProviders()
}
static getInstance(contextProxy: ContextProxy): TtsProviderFactory {
if (!TtsProviderFactory.instance) {
TtsProviderFactory.instance = new TtsProviderFactory(contextProxy)
}
return TtsProviderFactory.instance
}
private initializeProviders() {
// Initialize all available providers
this.providers.set("default", new DefaultTtsProvider(this.contextProxy))
this.providers.set("google-cloud", new GoogleCloudTtsProvider(this.contextProxy))
this.providers.set("azure", new AzureTtsProvider(this.contextProxy))
}
/**
* Get the current TTS provider based on user settings
*/
getCurrentProvider(): TtsProviderInterface {
const providerName = this.contextProxy.getValue("ttsProvider" as any) || "default"
const provider = this.providers.get(providerName)
if (!provider) {
// Fall back to default if the selected provider is not available
console.warn(`TTS provider '${providerName}' not found, falling back to default`)
return this.providers.get("default")!
}
// Check if the provider is properly configured
if (!provider.isConfigured() && providerName !== "default") {
console.warn(`TTS provider '${providerName}' is not configured, falling back to default`)
return this.providers.get("default")!
}
this.currentProvider = provider
return provider
}
/**
* Get a specific provider by name
*/
getProvider(name: string): TtsProviderInterface | undefined {
return this.providers.get(name)
}
/**
* Get all available providers
*/
getAllProviders(): Map<string, TtsProviderInterface> {
return this.providers
}
/**
* Check if a provider is available and configured
*/
isProviderAvailable(name: string): boolean {
const provider = this.providers.get(name)
return provider ? provider.isConfigured() : false
}
/**
* Update the usage tracking for the current month
*/
async updateUsageTracking(charactersUsed: number): Promise<void> {
const currentMonth = new Date().toISOString().slice(0, 7) // YYYY-MM format
const providerName = this.contextProxy.getValue("ttsProvider" as any) || "default"
// Only track usage for cloud providers
if (providerName === "google-cloud" || providerName === "azure") {
const usageKey =
`tts${providerName.charAt(0).toUpperCase() + providerName.slice(1).replace("-", "")}MonthlyUsage` as any
const currentUsage = this.contextProxy.getValue(usageKey) || { month: currentMonth, characters: 0 }
// Reset usage if it's a new month
if (currentUsage.month !== currentMonth) {
currentUsage.month = currentMonth
currentUsage.characters = 0
}
// Update the character count
currentUsage.characters += charactersUsed
// Save the updated usage
await this.contextProxy.setValue(usageKey, currentUsage)
}
}
/**
* Get the current month's usage for a provider
*/
getMonthlyUsage(providerName: string): { month: string; characters: number } {
const currentMonth = new Date().toISOString().slice(0, 7)
if (providerName === "google-cloud" || providerName === "azure") {
const usageKey =
`tts${providerName.charAt(0).toUpperCase() + providerName.slice(1).replace("-", "")}MonthlyUsage` as any
const usage = this.contextProxy.getValue(usageKey) || { month: currentMonth, characters: 0 }
// Reset if it's a new month
if (usage.month !== currentMonth) {
return { month: currentMonth, characters: 0 }
}
return usage
}
return { month: currentMonth, characters: 0 }
}
/**
* Check if the user is within the free tier for the current provider
*/
async isWithinFreeTier(): Promise<boolean> {
const providerName = this.contextProxy.getValue("ttsProvider" as any) || "default"
const provider = this.getCurrentProvider()
if (providerName === "default") {
return true // Default TTS is always free
}
const usage = this.getMonthlyUsage(providerName)
return provider.isWithinFreeTier(usage.characters)
}
/**
* Reset the instance (useful for testing)
*/
static reset(): void {
TtsProviderFactory.instance = null as any
}
}

View file

@ -0,0 +1,142 @@
import axios from "axios"
import { TtsProviderInterface, TtsVoice, TTS_PRICING } from "../types"
import { ContextProxy } from "../../../core/config/ContextProxy"
export class AzureTtsProvider implements TtsProviderInterface {
private apiKey: string | undefined
private region: string | undefined
private baseUrl: string
constructor(private contextProxy: ContextProxy) {
this.apiKey = this.contextProxy.getValue("azureTtsApiKey" as any)
this.region = this.contextProxy.getValue("azureTtsRegion" as any) || "eastus"
this.baseUrl = `https://${this.region}.tts.speech.microsoft.com/cognitiveservices`
}
async getVoices(): Promise<TtsVoice[]> {
if (!this.isConfigured()) {
return []
}
try {
const response = await axios.get(`${this.baseUrl}/voices/list`, {
headers: {
"Ocp-Apim-Subscription-Key": this.apiKey!,
},
})
return response.data.map((voice: any) => ({
id: voice.ShortName,
name: `${voice.DisplayName} (${voice.LocalName})`,
languageCode: voice.Locale,
gender: voice.Gender,
premium: voice.VoiceType === "Neural",
}))
} catch (error) {
console.error("Failed to fetch Azure TTS voices:", error)
return []
}
}
async synthesizeSpeech(text: string, voiceId: string, speed: number): Promise<Buffer> {
if (!this.isConfigured()) {
throw new Error("Azure TTS is not configured")
}
try {
// Convert speed to Azure's rate format (e.g., "+20%" or "-10%")
const rate = speed === 1 ? "default" : `${Math.round((speed - 1) * 100)}%`
const ssml = `
<speak version='1.0' xml:lang='en-US'>
<voice xml:lang='en-US' name='${voiceId}'>
<prosody rate='${rate}'>
${this.escapeXml(text)}
</prosody>
</voice>
</speak>`
const response = await axios.post(`${this.baseUrl}/v1`, ssml, {
headers: {
"Ocp-Apim-Subscription-Key": this.apiKey!,
"Content-Type": "application/ssml+xml",
"X-Microsoft-OutputFormat": "audio-16khz-128kbitrate-mono-mp3",
},
responseType: "arraybuffer",
})
return Buffer.from(response.data)
} catch (error) {
console.error("Failed to synthesize speech with Azure TTS:", error)
throw error
}
}
calculateCost(text: string): number {
const characterCount = text.length
// Assume neural voices by default (can be improved by checking voice type)
const pricePerMillion = TTS_PRICING.azure.neural
return (characterCount / 1_000_000) * pricePerMillion
}
isConfigured(): boolean {
return !!this.apiKey
}
/**
* Check if the user has exceeded their free tier for the current month
*/
async isWithinFreeTier(charactersUsed: number): Promise<boolean> {
return charactersUsed < TTS_PRICING.azure.freeMonthlyCharacters
}
/**
* Escape XML special characters for SSML
*/
private escapeXml(text: string): string {
return text
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;")
.replace(/'/g, "&apos;")
}
/**
* Play audio buffer using the system's audio player
*/
async playAudio(audioBuffer: Buffer): Promise<void> {
// Save to temporary file and play using system command
const fs = require("fs").promises
const path = require("path")
const { exec } = require("child_process")
const { promisify } = require("util")
const execAsync = promisify(exec)
const os = require("os")
const tempFile = path.join(os.tmpdir(), `tts-${Date.now()}.mp3`)
await fs.writeFile(tempFile, audioBuffer)
try {
// Use different commands based on platform
const platform = process.platform
let command: string
if (platform === "darwin") {
// macOS
command = `afplay "${tempFile}"`
} else if (platform === "win32") {
// Windows
command = `powershell -c "(New-Object Media.SoundPlayer '${tempFile}').PlaySync()"`
} else {
// Linux
command = `aplay "${tempFile}" || mpg123 "${tempFile}" || ffplay -nodisp -autoexit "${tempFile}"`
}
await execAsync(command)
} finally {
// Clean up temp file
await fs.unlink(tempFile).catch(() => {})
}
}
}

View file

@ -0,0 +1,158 @@
import { TtsProviderInterface, TtsVoice } from "../types"
import { ContextProxy } from "../../../core/config/ContextProxy"
export class DefaultTtsProvider implements TtsProviderInterface {
private say: any
constructor(private contextProxy: ContextProxy) {
// Lazy load the 'say' module
try {
this.say = require("say")
} catch (error) {
console.warn("Failed to load say module for default TTS:", error)
this.say = null
}
}
async getVoices(): Promise<TtsVoice[]> {
// Default OS TTS doesn't provide a reliable way to list voices
// Return a generic voice option
return [
{
id: "default",
name: "System Default Voice",
languageCode: "en-US",
gender: "neutral",
premium: false,
},
]
}
async synthesizeSpeech(text: string, voiceId: string, speed: number): Promise<Buffer> {
if (!this.say) {
throw new Error("Default TTS is not available - say module not found")
}
// The default TTS doesn't return audio data, it plays directly
// Return an empty buffer and handle playback in playAudio method
return Buffer.from("")
}
calculateCost(text: string): number {
// Default OS TTS is free
return 0
}
isConfigured(): boolean {
// Default TTS is always configured if the say module is available
return !!this.say
}
/**
* Check if the user has exceeded their free tier for the current month
* Default TTS is always free, so this always returns true
*/
async isWithinFreeTier(charactersUsed: number): Promise<boolean> {
return true
}
/**
* Play text directly using the system's TTS engine
* Note: For default TTS, we play the text directly rather than audio buffer
*/
async playAudio(audioBuffer: Buffer, text?: string, speed: number = 1.0): Promise<void> {
if (!this.say) {
throw new Error("Default TTS is not available - say module not found")
}
if (!text) {
throw new Error("Text is required for default TTS playback")
}
return new Promise((resolve, reject) => {
// Use platform-specific voice if configured
const platform = process.platform
let voice: string | undefined
// Get user's preferred voice for the platform if available
try {
if (platform === "darwin") {
// macOS voices
const macVoice = this.contextProxy.getValue("ttsMacVoice" as any)
if (macVoice && macVoice !== "default") {
voice = macVoice
}
} else if (platform === "win32") {
// Windows voices
const winVoice = this.contextProxy.getValue("ttsWindowsVoice" as any)
if (winVoice && winVoice !== "default") {
voice = winVoice
}
} else {
// Linux - usually uses espeak or festival
const linuxVoice = this.contextProxy.getValue("ttsLinuxVoice" as any)
if (linuxVoice && linuxVoice !== "default") {
voice = linuxVoice
}
}
} catch (error) {
// Ignore voice preference errors and use system default
console.debug("Could not get voice preference:", error)
}
// Convert speed to say module's format (words per minute)
// Default is around 175 WPM, so adjust based on the speed multiplier
const wpm = Math.round(175 * speed)
this.say.speak(text, voice, wpm, (err: any) => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
}
/**
* Stop any ongoing TTS playback
*/
stopPlayback(): void {
if (this.say && this.say.stop) {
this.say.stop()
}
}
/**
* Export text to an audio file (not supported by default TTS)
*/
async exportToFile(text: string, outputPath: string, voiceId: string, speed: number): Promise<void> {
if (!this.say) {
throw new Error("Default TTS is not available - say module not found")
}
return new Promise((resolve, reject) => {
// Use platform-specific export if available
const platform = process.platform
if (platform === "darwin") {
// macOS can export using the say command
const { exec } = require("child_process")
const voice = voiceId !== "default" ? voiceId : undefined
const voiceArg = voice ? ` -v "${voice}"` : ""
const rateArg = ` -r ${Math.round(175 * speed)}`
exec(`say "${text}"${voiceArg}${rateArg} -o "${outputPath}"`, (err: any) => {
if (err) {
reject(new Error(`Failed to export TTS: ${err.message}`))
} else {
resolve()
}
})
} else {
// Other platforms don't support direct export with say module
reject(new Error("Audio export is not supported on this platform with default TTS"))
}
})
}
}

View file

@ -0,0 +1,126 @@
import axios from "axios"
import { TtsProviderInterface, TtsVoice, TTS_PRICING } from "../types"
import { ContextProxy } from "../../../core/config/ContextProxy"
export class GoogleCloudTtsProvider implements TtsProviderInterface {
private apiKey: string | undefined
private baseUrl = "https://texttospeech.googleapis.com/v1"
constructor(private contextProxy: ContextProxy) {
this.apiKey = this.contextProxy.getValue("googleCloudTtsApiKey" as any)
}
async getVoices(): Promise<TtsVoice[]> {
if (!this.isConfigured()) {
return []
}
try {
const response = await axios.get(`${this.baseUrl}/voices`, {
params: { key: this.apiKey },
})
return response.data.voices.map((voice: any) => ({
id: voice.name,
name: `${voice.name} (${voice.ssmlGender})`,
languageCode: voice.languageCodes[0],
gender: voice.ssmlGender,
premium:
voice.name.includes("Wavenet") || voice.name.includes("Neural2") || voice.name.includes("Studio"),
}))
} catch (error) {
console.error("Failed to fetch Google Cloud TTS voices:", error)
return []
}
}
async synthesizeSpeech(text: string, voiceId: string, speed: number): Promise<Buffer> {
if (!this.isConfigured()) {
throw new Error("Google Cloud TTS is not configured")
}
try {
const response = await axios.post(
`${this.baseUrl}/text:synthesize`,
{
input: { text },
voice: {
name: voiceId,
languageCode: voiceId.split("-").slice(0, 2).join("-"),
},
audioConfig: {
audioEncoding: "MP3",
speakingRate: speed,
},
},
{
params: { key: this.apiKey },
headers: { "Content-Type": "application/json" },
},
)
// The response contains base64-encoded audio
const audioBuffer = Buffer.from(response.data.audioContent, "base64")
return audioBuffer
} catch (error) {
console.error("Failed to synthesize speech with Google Cloud TTS:", error)
throw error
}
}
calculateCost(text: string): number {
const characterCount = text.length
// Assume standard voices by default (can be improved by checking voice type)
const pricePerMillion = TTS_PRICING.google.standard
return (characterCount / 1_000_000) * pricePerMillion
}
isConfigured(): boolean {
return !!this.apiKey
}
/**
* Check if the user has exceeded their free tier for the current month
*/
async isWithinFreeTier(charactersUsed: number): Promise<boolean> {
return charactersUsed < TTS_PRICING.google.freeMonthlyCharacters
}
/**
* Play audio buffer using the system's audio player
*/
async playAudio(audioBuffer: Buffer): Promise<void> {
// Save to temporary file and play using system command
const fs = require("fs").promises
const path = require("path")
const { exec } = require("child_process")
const { promisify } = require("util")
const execAsync = promisify(exec)
const os = require("os")
const tempFile = path.join(os.tmpdir(), `tts-${Date.now()}.mp3`)
await fs.writeFile(tempFile, audioBuffer)
try {
// Use different commands based on platform
const platform = process.platform
let command: string
if (platform === "darwin") {
// macOS
command = `afplay "${tempFile}"`
} else if (platform === "win32") {
// Windows
command = `powershell -c "(New-Object Media.SoundPlayer '${tempFile}').PlaySync()"`
} else {
// Linux
command = `aplay "${tempFile}" || mpg123 "${tempFile}" || ffplay -nodisp -autoexit "${tempFile}"`
}
await execAsync(command)
} finally {
// Clean up temp file
await fs.unlink(tempFile).catch(() => {})
}
}
}

67
src/utils/tts/types.ts Normal file
View file

@ -0,0 +1,67 @@
export type TtsProvider = "default" | "google" | "azure"
export interface TtsVoice {
id: string
name: string
languageCode: string
gender?: string
premium?: boolean
}
export interface TtsProviderInterface {
/**
* Get available voices for this provider
*/
getVoices(): Promise<TtsVoice[]>
/**
* Synthesize speech from text
* @param text The text to synthesize
* @param voiceId The voice ID to use
* @param speed The speech speed (0.5 to 2.0)
* @returns The synthesized audio as a buffer
*/
synthesizeSpeech(text: string, voiceId: string, speed: number): Promise<Buffer>
/**
* Calculate the cost for synthesizing the given text
* @param text The text to synthesize
* @returns The cost in USD
*/
calculateCost(text: string): number
/**
* Check if the provider is properly configured
*/
isConfigured(): boolean
/**
* Check if the user has exceeded their free tier for the current month
* @param charactersUsed The number of characters used this month
* @returns True if within free tier, false otherwise
*/
isWithinFreeTier(charactersUsed: number): Promise<boolean>
}
export interface TtsUsageMetrics {
charactersUsed: number
cost: number
timestamp: Date
provider: TtsProvider
}
// Pricing constants (per million characters)
export const TTS_PRICING = {
google: {
standard: 4.0, // $4.00 per 1M characters for standard voices
wavenet: 16.0, // $16.00 per 1M characters for WaveNet voices
neural2: 16.0, // $16.00 per 1M characters for Neural2 voices
studio: 160.0, // $160.00 per 1M characters for Studio voices
freeMonthlyCharacters: 4_000_000, // 4M free characters per month (standard voices)
},
azure: {
standard: 4.0, // $4.00 per 1M characters for standard voices
neural: 15.0, // $15.00 per 1M characters for neural voices
freeMonthlyCharacters: 500_000, // 0.5M free characters per month
},
}