mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-07 08:26:51 +00:00
feat: Add Google Cloud and Azure TTS integration
- Add TTS provider interface and types with pricing constants - Implement Google Cloud TTS provider with voice selection and cost tracking - Implement Azure TTS provider with SSML support and region configuration - Create default OS TTS provider wrapping existing say module - Add provider factory for managing multiple TTS providers - Update TTS utility to support provider switching and usage tracking - Add monthly usage tracking for free tier management - Update webview message handler to support TTS provider operations - Add TTS-related settings to global settings schema
This commit is contained in:
parent
54745fc234
commit
82a3e7b3a7
8 changed files with 969 additions and 28 deletions
|
|
@ -131,7 +131,18 @@ export const globalSettingsSchema = z.object({
|
|||
.optional(),
|
||||
|
||||
ttsEnabled: z.boolean().optional(),
|
||||
ttsProvider: z.enum(["default", "google-cloud", "azure"]).optional(),
|
||||
ttsSpeed: z.number().optional(),
|
||||
ttsSelectedVoice: z.string().optional(),
|
||||
ttsGoogleVoice: z.string().optional(),
|
||||
ttsAzureVoice: z.string().optional(),
|
||||
ttsMonthlyUsage: z
|
||||
.object({
|
||||
google: z.number().optional(),
|
||||
azure: z.number().optional(),
|
||||
lastResetDate: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
soundEnabled: z.boolean().optional(),
|
||||
soundVolume: z.number().optional(),
|
||||
|
||||
|
|
@ -245,6 +256,9 @@ export const SECRET_STATE_KEYS = [
|
|||
// Global secrets that are part of GlobalSettings (not ProviderSettings)
|
||||
export const GLOBAL_SECRET_KEYS = [
|
||||
"openRouterImageApiKey", // For image generation
|
||||
"googleCloudTtsApiKey", // For Google Cloud TTS
|
||||
"azureTtsApiKey", // For Microsoft Azure TTS
|
||||
"azureTtsRegion", // Azure TTS region/endpoint
|
||||
] as const
|
||||
|
||||
// Type for the actual secret storage keys
|
||||
|
|
|
|||
|
|
@ -43,7 +43,17 @@ import { getTheme } from "../../integrations/theme/getTheme"
|
|||
import { discoverChromeHostUrl, tryChromeHostUrl } from "../../services/browser/browserDiscovery"
|
||||
import { searchWorkspaceFiles } from "../../services/search/file-search"
|
||||
import { fileExistsAtPath } from "../../utils/fs"
|
||||
import { playTts, setTtsEnabled, setTtsSpeed, stopTts } from "../../utils/tts"
|
||||
import {
|
||||
playTts,
|
||||
setTtsEnabled,
|
||||
setTtsSpeed,
|
||||
stopTts,
|
||||
initializeTts,
|
||||
getAvailableVoices,
|
||||
getCurrentProviderName,
|
||||
getTtsUsageStats,
|
||||
isProviderConfigured,
|
||||
} from "../../utils/tts"
|
||||
import { searchCommits } from "../../utils/git"
|
||||
import { exportSettings, importSettingsWithFeedback } from "../config/importExport"
|
||||
import { getOpenAiModels } from "../../api/providers/openai"
|
||||
|
|
@ -427,6 +437,9 @@ export const webviewMessageHandler = async (
|
|||
|
||||
switch (message.type) {
|
||||
case "webviewDidLaunch":
|
||||
// Initialize TTS system with context proxy
|
||||
initializeTts(provider.contextProxy)
|
||||
|
||||
// Load custom modes first
|
||||
const customModes = await provider.customModesManager.getCustomModes()
|
||||
await updateGlobalState("customModes", customModes)
|
||||
|
|
@ -1302,6 +1315,73 @@ export const webviewMessageHandler = async (
|
|||
case "stopTts":
|
||||
stopTts()
|
||||
break
|
||||
case "ttsProvider":
|
||||
// Handle TTS provider selection
|
||||
if (message.text) {
|
||||
await updateGlobalState("ttsProvider", message.text)
|
||||
await provider.postStateToWebview()
|
||||
}
|
||||
break
|
||||
case "ttsGoogleVoice":
|
||||
// Handle Google Cloud TTS voice selection
|
||||
if (message.text) {
|
||||
await updateGlobalState("ttsGoogleVoice", message.text)
|
||||
await provider.postStateToWebview()
|
||||
}
|
||||
break
|
||||
case "ttsAzureVoice":
|
||||
// Handle Azure TTS voice selection
|
||||
if (message.text) {
|
||||
await updateGlobalState("ttsAzureVoice", message.text)
|
||||
await provider.postStateToWebview()
|
||||
}
|
||||
break
|
||||
case "ttsAzureRegion":
|
||||
// Handle Azure region selection
|
||||
if (message.text) {
|
||||
await updateGlobalState("ttsAzureRegion", message.text)
|
||||
await provider.postStateToWebview()
|
||||
}
|
||||
break
|
||||
case "getTtsVoices":
|
||||
// Get available voices for the current provider
|
||||
try {
|
||||
const voices = await getAvailableVoices()
|
||||
await provider.postMessageToWebview({
|
||||
type: "ttsVoices",
|
||||
voices: voices,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error("Failed to get TTS voices:", error)
|
||||
await provider.postMessageToWebview({
|
||||
type: "ttsVoices",
|
||||
voices: [],
|
||||
})
|
||||
}
|
||||
break
|
||||
case "getTtsUsageStats":
|
||||
// Get TTS usage statistics
|
||||
try {
|
||||
const stats = getTtsUsageStats()
|
||||
await provider.postMessageToWebview({
|
||||
type: "ttsUsageStats",
|
||||
stats: stats,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error("Failed to get TTS usage stats:", error)
|
||||
}
|
||||
break
|
||||
case "checkTtsProviderConfigured":
|
||||
// Check if a specific provider is configured
|
||||
if (message.text) {
|
||||
const isConfigured = isProviderConfigured(message.text)
|
||||
await provider.postMessageToWebview({
|
||||
type: "ttsProviderConfigured",
|
||||
provider: message.text,
|
||||
configured: isConfigured,
|
||||
})
|
||||
}
|
||||
break
|
||||
case "diffEnabled":
|
||||
const diffEnabled = message.bool ?? true
|
||||
await updateGlobalState("diffEnabled", diffEnabled)
|
||||
|
|
|
|||
263
src/utils/tts.ts
263
src/utils/tts.ts
|
|
@ -1,3 +1,7 @@
|
|||
import { TtsProviderFactory } from "./tts/provider-factory"
|
||||
import { TtsProviderInterface, TtsVoice } from "./tts/types"
|
||||
import { ContextProxy } from "../core/config/ContextProxy"
|
||||
|
||||
interface Say {
|
||||
speak: (text: string, voice?: string, speed?: number, callback?: (err?: string) => void) => void
|
||||
stop: () => void
|
||||
|
|
@ -14,16 +18,66 @@ type QueueItem = {
|
|||
}
|
||||
|
||||
let isTtsEnabled = false
|
||||
|
||||
export const setTtsEnabled = (enabled: boolean) => (isTtsEnabled = enabled)
|
||||
|
||||
let speed = 1.0
|
||||
|
||||
export const setTtsSpeed = (newSpeed: number) => (speed = newSpeed)
|
||||
|
||||
let sayInstance: Say | undefined = undefined
|
||||
let queue: QueueItem[] = []
|
||||
let currentProvider: TtsProviderInterface | null = null
|
||||
let providerFactory: TtsProviderFactory | null = null
|
||||
let contextProxy: ContextProxy | null = null
|
||||
|
||||
/**
|
||||
* Initialize the TTS system with a context proxy
|
||||
*/
|
||||
export const initializeTts = (proxy: ContextProxy) => {
|
||||
contextProxy = proxy
|
||||
providerFactory = TtsProviderFactory.getInstance(proxy)
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether TTS is enabled
|
||||
*/
|
||||
export const setTtsEnabled = (enabled: boolean) => {
|
||||
isTtsEnabled = enabled
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the TTS speed
|
||||
*/
|
||||
export const setTtsSpeed = (newSpeed: number) => {
|
||||
speed = newSpeed
|
||||
}
|
||||
|
||||
/**
|
||||
* Get available voices for the current provider
|
||||
*/
|
||||
export const getAvailableVoices = async (): Promise<TtsVoice[]> => {
|
||||
if (!providerFactory) {
|
||||
console.error("TTS not initialized. Call initializeTts() first.")
|
||||
return []
|
||||
}
|
||||
|
||||
try {
|
||||
const provider = providerFactory.getCurrentProvider()
|
||||
return await provider.getVoices()
|
||||
} catch (error) {
|
||||
console.error("Failed to get available voices:", error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current TTS provider name
|
||||
*/
|
||||
export const getCurrentProviderName = (): string => {
|
||||
if (!contextProxy) {
|
||||
return "default"
|
||||
}
|
||||
return contextProxy.getValue("ttsProvider" as any) || "default"
|
||||
}
|
||||
|
||||
/**
|
||||
* Play TTS for a message
|
||||
*/
|
||||
export const playTts = async (message: string, options: PlayTtsOptions = {}) => {
|
||||
if (!isTtsEnabled) {
|
||||
return
|
||||
|
|
@ -32,50 +86,205 @@ export const playTts = async (message: string, options: PlayTtsOptions = {}) =>
|
|||
try {
|
||||
queue.push({ message, options })
|
||||
await processQueue()
|
||||
} catch (error) {}
|
||||
} catch (error) {
|
||||
console.error("TTS playback error:", error)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop TTS playback
|
||||
*/
|
||||
export const stopTts = () => {
|
||||
sayInstance?.stop()
|
||||
sayInstance = undefined
|
||||
// Stop the current provider's playback if it's the default provider
|
||||
const providerName = getCurrentProviderName()
|
||||
if (providerName === "default" && sayInstance) {
|
||||
sayInstance.stop()
|
||||
sayInstance = undefined
|
||||
}
|
||||
// For cloud providers, we may need to add audio player cancellation
|
||||
// This would require tracking the audio playback process
|
||||
queue = []
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the TTS queue
|
||||
*/
|
||||
const processQueue = async (): Promise<void> => {
|
||||
if (!isTtsEnabled || sayInstance) {
|
||||
return
|
||||
}
|
||||
|
||||
const item = queue.shift()
|
||||
|
||||
if (!item) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!providerFactory) {
|
||||
console.error("TTS not initialized. Call initializeTts() first.")
|
||||
await processQueue()
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const { message: nextUtterance, options } = item
|
||||
const provider = providerFactory.getCurrentProvider()
|
||||
const providerName = getCurrentProviderName()
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const say: Say = require("say")
|
||||
sayInstance = say
|
||||
options.onStart?.()
|
||||
|
||||
say.speak(nextUtterance, undefined, speed, (err) => {
|
||||
options.onStop?.()
|
||||
|
||||
if (err) {
|
||||
reject(new Error(err))
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
|
||||
sayInstance = undefined
|
||||
})
|
||||
})
|
||||
// Handle different providers
|
||||
if (providerName === "default") {
|
||||
// Use the existing Say-based implementation for default provider
|
||||
await playWithDefaultProvider(nextUtterance, options)
|
||||
} else {
|
||||
// Use cloud providers (Google Cloud or Azure)
|
||||
await playWithCloudProvider(provider, providerName, nextUtterance, options)
|
||||
}
|
||||
|
||||
await processQueue()
|
||||
} catch (error: any) {
|
||||
console.error("TTS processing error:", error)
|
||||
sayInstance = undefined
|
||||
await processQueue()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Play TTS using the default OS provider
|
||||
*/
|
||||
const playWithDefaultProvider = async (text: string, options: PlayTtsOptions): Promise<void> => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const say: Say = require("say")
|
||||
sayInstance = say
|
||||
options.onStart?.()
|
||||
|
||||
say.speak(text, undefined, speed, (err) => {
|
||||
options.onStop?.()
|
||||
|
||||
if (err) {
|
||||
reject(new Error(err))
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
|
||||
sayInstance = undefined
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Play TTS using a cloud provider
|
||||
*/
|
||||
const playWithCloudProvider = async (
|
||||
provider: TtsProviderInterface,
|
||||
providerName: string,
|
||||
text: string,
|
||||
options: PlayTtsOptions,
|
||||
): Promise<void> => {
|
||||
if (!contextProxy || !providerFactory) {
|
||||
throw new Error("TTS not initialized")
|
||||
}
|
||||
|
||||
// Get the selected voice for this provider
|
||||
let voiceId = "default"
|
||||
if (providerName === "google-cloud") {
|
||||
voiceId = contextProxy.getValue("ttsGoogleVoice" as any) || "en-US-Wavenet-D"
|
||||
} else if (providerName === "azure") {
|
||||
voiceId = contextProxy.getValue("ttsAzureVoice" as any) || "en-US-JennyNeural"
|
||||
}
|
||||
|
||||
options.onStart?.()
|
||||
|
||||
try {
|
||||
// Check if within free tier
|
||||
const usage = providerFactory.getMonthlyUsage(providerName)
|
||||
const isWithinFreeTier = await provider.isWithinFreeTier(usage.characters)
|
||||
|
||||
if (!isWithinFreeTier) {
|
||||
console.warn(
|
||||
`TTS: Monthly free tier exceeded for ${providerName}. Current usage: ${usage.characters} characters`,
|
||||
)
|
||||
// Optionally fall back to default provider or show a warning to the user
|
||||
}
|
||||
|
||||
// Calculate cost
|
||||
const cost = provider.calculateCost(text)
|
||||
if (cost > 0) {
|
||||
console.debug(`TTS cost for ${text.length} characters: $${cost.toFixed(6)}`)
|
||||
}
|
||||
|
||||
// Synthesize speech
|
||||
const audioBuffer = await provider.synthesizeSpeech(text, voiceId, speed)
|
||||
|
||||
// Play the audio
|
||||
await playAudioBuffer(audioBuffer)
|
||||
|
||||
// Update usage tracking
|
||||
await providerFactory.updateUsageTracking(text.length)
|
||||
} finally {
|
||||
options.onStop?.()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Play an audio buffer using system commands
|
||||
*/
|
||||
const playAudioBuffer = async (audioBuffer: Buffer): Promise<void> => {
|
||||
const fs = require("fs").promises
|
||||
const path = require("path")
|
||||
const { exec } = require("child_process")
|
||||
const { promisify } = require("util")
|
||||
const execAsync = promisify(exec)
|
||||
const os = require("os")
|
||||
|
||||
const tempFile = path.join(os.tmpdir(), `tts-${Date.now()}.mp3`)
|
||||
await fs.writeFile(tempFile, audioBuffer)
|
||||
|
||||
try {
|
||||
const platform = process.platform
|
||||
let command: string
|
||||
|
||||
if (platform === "darwin") {
|
||||
// macOS
|
||||
command = `afplay "${tempFile}"`
|
||||
} else if (platform === "win32") {
|
||||
// Windows
|
||||
command = `powershell -c "(New-Object Media.SoundPlayer '${tempFile}').PlaySync()"`
|
||||
} else {
|
||||
// Linux - try multiple players
|
||||
command = `aplay "${tempFile}" || mpg123 "${tempFile}" || ffplay -nodisp -autoexit "${tempFile}"`
|
||||
}
|
||||
|
||||
await execAsync(command)
|
||||
} finally {
|
||||
// Clean up temp file
|
||||
await fs.unlink(tempFile).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get TTS usage statistics for the current month
|
||||
*/
|
||||
export const getTtsUsageStats = () => {
|
||||
if (!providerFactory || !contextProxy) {
|
||||
return null
|
||||
}
|
||||
|
||||
const providerName = getCurrentProviderName()
|
||||
const usage = providerFactory.getMonthlyUsage(providerName)
|
||||
const provider = providerFactory.getCurrentProvider()
|
||||
|
||||
return {
|
||||
provider: providerName,
|
||||
monthlyUsage: usage,
|
||||
isWithinFreeTier: provider.isWithinFreeTier(usage.characters),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a specific provider is configured
|
||||
*/
|
||||
export const isProviderConfigured = (providerName: string): boolean => {
|
||||
if (!providerFactory) {
|
||||
return false
|
||||
}
|
||||
return providerFactory.isProviderAvailable(providerName)
|
||||
}
|
||||
|
|
|
|||
145
src/utils/tts/provider-factory.ts
Normal file
145
src/utils/tts/provider-factory.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import { TtsProviderInterface } from "./types"
|
||||
import { DefaultTtsProvider } from "./providers/default-tts"
|
||||
import { GoogleCloudTtsProvider } from "./providers/google-cloud-tts"
|
||||
import { AzureTtsProvider } from "./providers/azure-tts"
|
||||
import { ContextProxy } from "../../core/config/ContextProxy"
|
||||
|
||||
export class TtsProviderFactory {
|
||||
private static instance: TtsProviderFactory
|
||||
private providers: Map<string, TtsProviderInterface> = new Map()
|
||||
private currentProvider: TtsProviderInterface | null = null
|
||||
|
||||
private constructor(private contextProxy: ContextProxy) {
|
||||
this.initializeProviders()
|
||||
}
|
||||
|
||||
static getInstance(contextProxy: ContextProxy): TtsProviderFactory {
|
||||
if (!TtsProviderFactory.instance) {
|
||||
TtsProviderFactory.instance = new TtsProviderFactory(contextProxy)
|
||||
}
|
||||
return TtsProviderFactory.instance
|
||||
}
|
||||
|
||||
private initializeProviders() {
|
||||
// Initialize all available providers
|
||||
this.providers.set("default", new DefaultTtsProvider(this.contextProxy))
|
||||
this.providers.set("google-cloud", new GoogleCloudTtsProvider(this.contextProxy))
|
||||
this.providers.set("azure", new AzureTtsProvider(this.contextProxy))
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current TTS provider based on user settings
|
||||
*/
|
||||
getCurrentProvider(): TtsProviderInterface {
|
||||
const providerName = this.contextProxy.getValue("ttsProvider" as any) || "default"
|
||||
|
||||
const provider = this.providers.get(providerName)
|
||||
if (!provider) {
|
||||
// Fall back to default if the selected provider is not available
|
||||
console.warn(`TTS provider '${providerName}' not found, falling back to default`)
|
||||
return this.providers.get("default")!
|
||||
}
|
||||
|
||||
// Check if the provider is properly configured
|
||||
if (!provider.isConfigured() && providerName !== "default") {
|
||||
console.warn(`TTS provider '${providerName}' is not configured, falling back to default`)
|
||||
return this.providers.get("default")!
|
||||
}
|
||||
|
||||
this.currentProvider = provider
|
||||
return provider
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific provider by name
|
||||
*/
|
||||
getProvider(name: string): TtsProviderInterface | undefined {
|
||||
return this.providers.get(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all available providers
|
||||
*/
|
||||
getAllProviders(): Map<string, TtsProviderInterface> {
|
||||
return this.providers
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a provider is available and configured
|
||||
*/
|
||||
isProviderAvailable(name: string): boolean {
|
||||
const provider = this.providers.get(name)
|
||||
return provider ? provider.isConfigured() : false
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the usage tracking for the current month
|
||||
*/
|
||||
async updateUsageTracking(charactersUsed: number): Promise<void> {
|
||||
const currentMonth = new Date().toISOString().slice(0, 7) // YYYY-MM format
|
||||
const providerName = this.contextProxy.getValue("ttsProvider" as any) || "default"
|
||||
|
||||
// Only track usage for cloud providers
|
||||
if (providerName === "google-cloud" || providerName === "azure") {
|
||||
const usageKey =
|
||||
`tts${providerName.charAt(0).toUpperCase() + providerName.slice(1).replace("-", "")}MonthlyUsage` as any
|
||||
const currentUsage = this.contextProxy.getValue(usageKey) || { month: currentMonth, characters: 0 }
|
||||
|
||||
// Reset usage if it's a new month
|
||||
if (currentUsage.month !== currentMonth) {
|
||||
currentUsage.month = currentMonth
|
||||
currentUsage.characters = 0
|
||||
}
|
||||
|
||||
// Update the character count
|
||||
currentUsage.characters += charactersUsed
|
||||
|
||||
// Save the updated usage
|
||||
await this.contextProxy.setValue(usageKey, currentUsage)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current month's usage for a provider
|
||||
*/
|
||||
getMonthlyUsage(providerName: string): { month: string; characters: number } {
|
||||
const currentMonth = new Date().toISOString().slice(0, 7)
|
||||
|
||||
if (providerName === "google-cloud" || providerName === "azure") {
|
||||
const usageKey =
|
||||
`tts${providerName.charAt(0).toUpperCase() + providerName.slice(1).replace("-", "")}MonthlyUsage` as any
|
||||
const usage = this.contextProxy.getValue(usageKey) || { month: currentMonth, characters: 0 }
|
||||
|
||||
// Reset if it's a new month
|
||||
if (usage.month !== currentMonth) {
|
||||
return { month: currentMonth, characters: 0 }
|
||||
}
|
||||
|
||||
return usage
|
||||
}
|
||||
|
||||
return { month: currentMonth, characters: 0 }
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the user is within the free tier for the current provider
|
||||
*/
|
||||
async isWithinFreeTier(): Promise<boolean> {
|
||||
const providerName = this.contextProxy.getValue("ttsProvider" as any) || "default"
|
||||
const provider = this.getCurrentProvider()
|
||||
|
||||
if (providerName === "default") {
|
||||
return true // Default TTS is always free
|
||||
}
|
||||
|
||||
const usage = this.getMonthlyUsage(providerName)
|
||||
return provider.isWithinFreeTier(usage.characters)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the instance (useful for testing)
|
||||
*/
|
||||
static reset(): void {
|
||||
TtsProviderFactory.instance = null as any
|
||||
}
|
||||
}
|
||||
142
src/utils/tts/providers/azure-tts.ts
Normal file
142
src/utils/tts/providers/azure-tts.ts
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
import axios from "axios"
|
||||
import { TtsProviderInterface, TtsVoice, TTS_PRICING } from "../types"
|
||||
import { ContextProxy } from "../../../core/config/ContextProxy"
|
||||
|
||||
export class AzureTtsProvider implements TtsProviderInterface {
|
||||
private apiKey: string | undefined
|
||||
private region: string | undefined
|
||||
private baseUrl: string
|
||||
|
||||
constructor(private contextProxy: ContextProxy) {
|
||||
this.apiKey = this.contextProxy.getValue("azureTtsApiKey" as any)
|
||||
this.region = this.contextProxy.getValue("azureTtsRegion" as any) || "eastus"
|
||||
this.baseUrl = `https://${this.region}.tts.speech.microsoft.com/cognitiveservices`
|
||||
}
|
||||
|
||||
async getVoices(): Promise<TtsVoice[]> {
|
||||
if (!this.isConfigured()) {
|
||||
return []
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await axios.get(`${this.baseUrl}/voices/list`, {
|
||||
headers: {
|
||||
"Ocp-Apim-Subscription-Key": this.apiKey!,
|
||||
},
|
||||
})
|
||||
|
||||
return response.data.map((voice: any) => ({
|
||||
id: voice.ShortName,
|
||||
name: `${voice.DisplayName} (${voice.LocalName})`,
|
||||
languageCode: voice.Locale,
|
||||
gender: voice.Gender,
|
||||
premium: voice.VoiceType === "Neural",
|
||||
}))
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch Azure TTS voices:", error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
async synthesizeSpeech(text: string, voiceId: string, speed: number): Promise<Buffer> {
|
||||
if (!this.isConfigured()) {
|
||||
throw new Error("Azure TTS is not configured")
|
||||
}
|
||||
|
||||
try {
|
||||
// Convert speed to Azure's rate format (e.g., "+20%" or "-10%")
|
||||
const rate = speed === 1 ? "default" : `${Math.round((speed - 1) * 100)}%`
|
||||
|
||||
const ssml = `
|
||||
<speak version='1.0' xml:lang='en-US'>
|
||||
<voice xml:lang='en-US' name='${voiceId}'>
|
||||
<prosody rate='${rate}'>
|
||||
${this.escapeXml(text)}
|
||||
</prosody>
|
||||
</voice>
|
||||
</speak>`
|
||||
|
||||
const response = await axios.post(`${this.baseUrl}/v1`, ssml, {
|
||||
headers: {
|
||||
"Ocp-Apim-Subscription-Key": this.apiKey!,
|
||||
"Content-Type": "application/ssml+xml",
|
||||
"X-Microsoft-OutputFormat": "audio-16khz-128kbitrate-mono-mp3",
|
||||
},
|
||||
responseType: "arraybuffer",
|
||||
})
|
||||
|
||||
return Buffer.from(response.data)
|
||||
} catch (error) {
|
||||
console.error("Failed to synthesize speech with Azure TTS:", error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
calculateCost(text: string): number {
|
||||
const characterCount = text.length
|
||||
// Assume neural voices by default (can be improved by checking voice type)
|
||||
const pricePerMillion = TTS_PRICING.azure.neural
|
||||
return (characterCount / 1_000_000) * pricePerMillion
|
||||
}
|
||||
|
||||
isConfigured(): boolean {
|
||||
return !!this.apiKey
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the user has exceeded their free tier for the current month
|
||||
*/
|
||||
async isWithinFreeTier(charactersUsed: number): Promise<boolean> {
|
||||
return charactersUsed < TTS_PRICING.azure.freeMonthlyCharacters
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape XML special characters for SSML
|
||||
*/
|
||||
private escapeXml(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'")
|
||||
}
|
||||
|
||||
/**
|
||||
* Play audio buffer using the system's audio player
|
||||
*/
|
||||
async playAudio(audioBuffer: Buffer): Promise<void> {
|
||||
// Save to temporary file and play using system command
|
||||
const fs = require("fs").promises
|
||||
const path = require("path")
|
||||
const { exec } = require("child_process")
|
||||
const { promisify } = require("util")
|
||||
const execAsync = promisify(exec)
|
||||
const os = require("os")
|
||||
|
||||
const tempFile = path.join(os.tmpdir(), `tts-${Date.now()}.mp3`)
|
||||
await fs.writeFile(tempFile, audioBuffer)
|
||||
|
||||
try {
|
||||
// Use different commands based on platform
|
||||
const platform = process.platform
|
||||
let command: string
|
||||
|
||||
if (platform === "darwin") {
|
||||
// macOS
|
||||
command = `afplay "${tempFile}"`
|
||||
} else if (platform === "win32") {
|
||||
// Windows
|
||||
command = `powershell -c "(New-Object Media.SoundPlayer '${tempFile}').PlaySync()"`
|
||||
} else {
|
||||
// Linux
|
||||
command = `aplay "${tempFile}" || mpg123 "${tempFile}" || ffplay -nodisp -autoexit "${tempFile}"`
|
||||
}
|
||||
|
||||
await execAsync(command)
|
||||
} finally {
|
||||
// Clean up temp file
|
||||
await fs.unlink(tempFile).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
158
src/utils/tts/providers/default-tts.ts
Normal file
158
src/utils/tts/providers/default-tts.ts
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
import { TtsProviderInterface, TtsVoice } from "../types"
|
||||
import { ContextProxy } from "../../../core/config/ContextProxy"
|
||||
|
||||
export class DefaultTtsProvider implements TtsProviderInterface {
|
||||
private say: any
|
||||
|
||||
constructor(private contextProxy: ContextProxy) {
|
||||
// Lazy load the 'say' module
|
||||
try {
|
||||
this.say = require("say")
|
||||
} catch (error) {
|
||||
console.warn("Failed to load say module for default TTS:", error)
|
||||
this.say = null
|
||||
}
|
||||
}
|
||||
|
||||
async getVoices(): Promise<TtsVoice[]> {
|
||||
// Default OS TTS doesn't provide a reliable way to list voices
|
||||
// Return a generic voice option
|
||||
return [
|
||||
{
|
||||
id: "default",
|
||||
name: "System Default Voice",
|
||||
languageCode: "en-US",
|
||||
gender: "neutral",
|
||||
premium: false,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
async synthesizeSpeech(text: string, voiceId: string, speed: number): Promise<Buffer> {
|
||||
if (!this.say) {
|
||||
throw new Error("Default TTS is not available - say module not found")
|
||||
}
|
||||
|
||||
// The default TTS doesn't return audio data, it plays directly
|
||||
// Return an empty buffer and handle playback in playAudio method
|
||||
return Buffer.from("")
|
||||
}
|
||||
|
||||
calculateCost(text: string): number {
|
||||
// Default OS TTS is free
|
||||
return 0
|
||||
}
|
||||
|
||||
isConfigured(): boolean {
|
||||
// Default TTS is always configured if the say module is available
|
||||
return !!this.say
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the user has exceeded their free tier for the current month
|
||||
* Default TTS is always free, so this always returns true
|
||||
*/
|
||||
async isWithinFreeTier(charactersUsed: number): Promise<boolean> {
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Play text directly using the system's TTS engine
|
||||
* Note: For default TTS, we play the text directly rather than audio buffer
|
||||
*/
|
||||
async playAudio(audioBuffer: Buffer, text?: string, speed: number = 1.0): Promise<void> {
|
||||
if (!this.say) {
|
||||
throw new Error("Default TTS is not available - say module not found")
|
||||
}
|
||||
|
||||
if (!text) {
|
||||
throw new Error("Text is required for default TTS playback")
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
// Use platform-specific voice if configured
|
||||
const platform = process.platform
|
||||
let voice: string | undefined
|
||||
|
||||
// Get user's preferred voice for the platform if available
|
||||
try {
|
||||
if (platform === "darwin") {
|
||||
// macOS voices
|
||||
const macVoice = this.contextProxy.getValue("ttsMacVoice" as any)
|
||||
if (macVoice && macVoice !== "default") {
|
||||
voice = macVoice
|
||||
}
|
||||
} else if (platform === "win32") {
|
||||
// Windows voices
|
||||
const winVoice = this.contextProxy.getValue("ttsWindowsVoice" as any)
|
||||
if (winVoice && winVoice !== "default") {
|
||||
voice = winVoice
|
||||
}
|
||||
} else {
|
||||
// Linux - usually uses espeak or festival
|
||||
const linuxVoice = this.contextProxy.getValue("ttsLinuxVoice" as any)
|
||||
if (linuxVoice && linuxVoice !== "default") {
|
||||
voice = linuxVoice
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore voice preference errors and use system default
|
||||
console.debug("Could not get voice preference:", error)
|
||||
}
|
||||
|
||||
// Convert speed to say module's format (words per minute)
|
||||
// Default is around 175 WPM, so adjust based on the speed multiplier
|
||||
const wpm = Math.round(175 * speed)
|
||||
|
||||
this.say.speak(text, voice, wpm, (err: any) => {
|
||||
if (err) {
|
||||
reject(err)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop any ongoing TTS playback
|
||||
*/
|
||||
stopPlayback(): void {
|
||||
if (this.say && this.say.stop) {
|
||||
this.say.stop()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export text to an audio file (not supported by default TTS)
|
||||
*/
|
||||
async exportToFile(text: string, outputPath: string, voiceId: string, speed: number): Promise<void> {
|
||||
if (!this.say) {
|
||||
throw new Error("Default TTS is not available - say module not found")
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
// Use platform-specific export if available
|
||||
const platform = process.platform
|
||||
|
||||
if (platform === "darwin") {
|
||||
// macOS can export using the say command
|
||||
const { exec } = require("child_process")
|
||||
const voice = voiceId !== "default" ? voiceId : undefined
|
||||
const voiceArg = voice ? ` -v "${voice}"` : ""
|
||||
const rateArg = ` -r ${Math.round(175 * speed)}`
|
||||
|
||||
exec(`say "${text}"${voiceArg}${rateArg} -o "${outputPath}"`, (err: any) => {
|
||||
if (err) {
|
||||
reject(new Error(`Failed to export TTS: ${err.message}`))
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
} else {
|
||||
// Other platforms don't support direct export with say module
|
||||
reject(new Error("Audio export is not supported on this platform with default TTS"))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
126
src/utils/tts/providers/google-cloud-tts.ts
Normal file
126
src/utils/tts/providers/google-cloud-tts.ts
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
import axios from "axios"
|
||||
import { TtsProviderInterface, TtsVoice, TTS_PRICING } from "../types"
|
||||
import { ContextProxy } from "../../../core/config/ContextProxy"
|
||||
|
||||
export class GoogleCloudTtsProvider implements TtsProviderInterface {
|
||||
private apiKey: string | undefined
|
||||
private baseUrl = "https://texttospeech.googleapis.com/v1"
|
||||
|
||||
constructor(private contextProxy: ContextProxy) {
|
||||
this.apiKey = this.contextProxy.getValue("googleCloudTtsApiKey" as any)
|
||||
}
|
||||
|
||||
async getVoices(): Promise<TtsVoice[]> {
|
||||
if (!this.isConfigured()) {
|
||||
return []
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await axios.get(`${this.baseUrl}/voices`, {
|
||||
params: { key: this.apiKey },
|
||||
})
|
||||
|
||||
return response.data.voices.map((voice: any) => ({
|
||||
id: voice.name,
|
||||
name: `${voice.name} (${voice.ssmlGender})`,
|
||||
languageCode: voice.languageCodes[0],
|
||||
gender: voice.ssmlGender,
|
||||
premium:
|
||||
voice.name.includes("Wavenet") || voice.name.includes("Neural2") || voice.name.includes("Studio"),
|
||||
}))
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch Google Cloud TTS voices:", error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
async synthesizeSpeech(text: string, voiceId: string, speed: number): Promise<Buffer> {
|
||||
if (!this.isConfigured()) {
|
||||
throw new Error("Google Cloud TTS is not configured")
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await axios.post(
|
||||
`${this.baseUrl}/text:synthesize`,
|
||||
{
|
||||
input: { text },
|
||||
voice: {
|
||||
name: voiceId,
|
||||
languageCode: voiceId.split("-").slice(0, 2).join("-"),
|
||||
},
|
||||
audioConfig: {
|
||||
audioEncoding: "MP3",
|
||||
speakingRate: speed,
|
||||
},
|
||||
},
|
||||
{
|
||||
params: { key: this.apiKey },
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
)
|
||||
|
||||
// The response contains base64-encoded audio
|
||||
const audioBuffer = Buffer.from(response.data.audioContent, "base64")
|
||||
return audioBuffer
|
||||
} catch (error) {
|
||||
console.error("Failed to synthesize speech with Google Cloud TTS:", error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
calculateCost(text: string): number {
|
||||
const characterCount = text.length
|
||||
// Assume standard voices by default (can be improved by checking voice type)
|
||||
const pricePerMillion = TTS_PRICING.google.standard
|
||||
return (characterCount / 1_000_000) * pricePerMillion
|
||||
}
|
||||
|
||||
isConfigured(): boolean {
|
||||
return !!this.apiKey
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the user has exceeded their free tier for the current month
|
||||
*/
|
||||
async isWithinFreeTier(charactersUsed: number): Promise<boolean> {
|
||||
return charactersUsed < TTS_PRICING.google.freeMonthlyCharacters
|
||||
}
|
||||
|
||||
/**
|
||||
* Play audio buffer using the system's audio player
|
||||
*/
|
||||
async playAudio(audioBuffer: Buffer): Promise<void> {
|
||||
// Save to temporary file and play using system command
|
||||
const fs = require("fs").promises
|
||||
const path = require("path")
|
||||
const { exec } = require("child_process")
|
||||
const { promisify } = require("util")
|
||||
const execAsync = promisify(exec)
|
||||
const os = require("os")
|
||||
|
||||
const tempFile = path.join(os.tmpdir(), `tts-${Date.now()}.mp3`)
|
||||
await fs.writeFile(tempFile, audioBuffer)
|
||||
|
||||
try {
|
||||
// Use different commands based on platform
|
||||
const platform = process.platform
|
||||
let command: string
|
||||
|
||||
if (platform === "darwin") {
|
||||
// macOS
|
||||
command = `afplay "${tempFile}"`
|
||||
} else if (platform === "win32") {
|
||||
// Windows
|
||||
command = `powershell -c "(New-Object Media.SoundPlayer '${tempFile}').PlaySync()"`
|
||||
} else {
|
||||
// Linux
|
||||
command = `aplay "${tempFile}" || mpg123 "${tempFile}" || ffplay -nodisp -autoexit "${tempFile}"`
|
||||
}
|
||||
|
||||
await execAsync(command)
|
||||
} finally {
|
||||
// Clean up temp file
|
||||
await fs.unlink(tempFile).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
67
src/utils/tts/types.ts
Normal file
67
src/utils/tts/types.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
export type TtsProvider = "default" | "google" | "azure"
|
||||
|
||||
export interface TtsVoice {
|
||||
id: string
|
||||
name: string
|
||||
languageCode: string
|
||||
gender?: string
|
||||
premium?: boolean
|
||||
}
|
||||
|
||||
export interface TtsProviderInterface {
|
||||
/**
|
||||
* Get available voices for this provider
|
||||
*/
|
||||
getVoices(): Promise<TtsVoice[]>
|
||||
|
||||
/**
|
||||
* Synthesize speech from text
|
||||
* @param text The text to synthesize
|
||||
* @param voiceId The voice ID to use
|
||||
* @param speed The speech speed (0.5 to 2.0)
|
||||
* @returns The synthesized audio as a buffer
|
||||
*/
|
||||
synthesizeSpeech(text: string, voiceId: string, speed: number): Promise<Buffer>
|
||||
|
||||
/**
|
||||
* Calculate the cost for synthesizing the given text
|
||||
* @param text The text to synthesize
|
||||
* @returns The cost in USD
|
||||
*/
|
||||
calculateCost(text: string): number
|
||||
|
||||
/**
|
||||
* Check if the provider is properly configured
|
||||
*/
|
||||
isConfigured(): boolean
|
||||
|
||||
/**
|
||||
* Check if the user has exceeded their free tier for the current month
|
||||
* @param charactersUsed The number of characters used this month
|
||||
* @returns True if within free tier, false otherwise
|
||||
*/
|
||||
isWithinFreeTier(charactersUsed: number): Promise<boolean>
|
||||
}
|
||||
|
||||
export interface TtsUsageMetrics {
|
||||
charactersUsed: number
|
||||
cost: number
|
||||
timestamp: Date
|
||||
provider: TtsProvider
|
||||
}
|
||||
|
||||
// Pricing constants (per million characters)
|
||||
export const TTS_PRICING = {
|
||||
google: {
|
||||
standard: 4.0, // $4.00 per 1M characters for standard voices
|
||||
wavenet: 16.0, // $16.00 per 1M characters for WaveNet voices
|
||||
neural2: 16.0, // $16.00 per 1M characters for Neural2 voices
|
||||
studio: 160.0, // $160.00 per 1M characters for Studio voices
|
||||
freeMonthlyCharacters: 4_000_000, // 4M free characters per month (standard voices)
|
||||
},
|
||||
azure: {
|
||||
standard: 4.0, // $4.00 per 1M characters for standard voices
|
||||
neural: 15.0, // $15.00 per 1M characters for neural voices
|
||||
freeMonthlyCharacters: 500_000, // 0.5M free characters per month
|
||||
},
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue