/** * Sherpa TTS Service - Dynamic model loading * Uses react-native-sherpa-onnx-offline-tts with Piper VITS models * Models downloaded from Hugging Face on first use */ import { NativeEventEmitter } from 'react-native'; import TTSManager from 'react-native-sherpa-onnx-offline-tts'; import RNFS from '@dr.pogodin/react-native-fs'; import { debugLogger } from '@/services/DebugLogger'; // Only Orayan (Ryan) voice - downloaded from Hugging Face const ORAYAN_VOICE = { id: 'ryan-medium', name: 'Ryan (Orayan)', description: 'Male, clear voice', hfRepo: 'csukuangfj/vits-piper-en_US-ryan-medium', modelDir: 'vits-piper-en_US-ryan-medium', onnxFile: 'en_US-ryan-medium.onnx', }; interface SherpaTTSState { initialized: boolean; initializing: boolean; speaking: boolean; error: string | null; } // Check if native module is available const NATIVE_MODULE_AVAILABLE = !!TTSManager; if (NATIVE_MODULE_AVAILABLE) { debugLogger.info('TTS', 'TTSManager native module loaded successfully'); } else { debugLogger.error('TTS', 'TTSManager native module NOT available - prebuild required'); } let ttsManagerEmitter: NativeEventEmitter | null = null; if (NATIVE_MODULE_AVAILABLE) { ttsManagerEmitter = new NativeEventEmitter(TTSManager); debugLogger.info('TTS', 'TTS event emitter initialized'); } let currentState: SherpaTTSState = { initialized: false, initializing: false, speaking: false, error: null, }; // State listeners const stateListeners: ((state: SherpaTTSState) => void)[] = []; function updateState(updates: Partial) { currentState = { ...currentState, ...updates }; notifyListeners(); } function notifyListeners() { stateListeners.forEach(listener => listener({ ...currentState })); } export function addStateListener(listener: (state: SherpaTTSState) => void) { stateListeners.push(listener); listener({ ...currentState }); return () => { const index = stateListeners.indexOf(listener); if (index >= 0) stateListeners.splice(index, 1); }; } export function getState(): SherpaTTSState { return { ...currentState }; } /** * Download TTS model from Hugging Face */ async function downloadModelFromHuggingFace(): Promise { const extractPath = `${RNFS.DocumentDirectoryPath}/voices`; const modelDir = `${extractPath}/${ORAYAN_VOICE.modelDir}`; const modelPath = `${modelDir}/${ORAYAN_VOICE.onnxFile}`; // Check if already downloaded const exists = await RNFS.exists(modelPath); if (exists) { debugLogger.info('TTS', `Model already downloaded at: ${modelPath}`); return true; } debugLogger.info('TTS', `Downloading model from Hugging Face: ${ORAYAN_VOICE.hfRepo}`); updateState({ initializing: true, error: 'Downloading voice model...' }); try { // Create directories await RNFS.mkdir(modelDir, { intermediateDirectories: true }); // Download model files from Hugging Face const baseUrl = `https://huggingface.co/${ORAYAN_VOICE.hfRepo}/resolve/main`; const filesToDownload = [ { url: `${baseUrl}/${ORAYAN_VOICE.onnxFile}`, path: modelPath }, { url: `${baseUrl}/tokens.txt`, path: `${modelDir}/tokens.txt` }, { url: `${baseUrl}/espeak-ng-data.tar.bz2`, path: `${modelDir}/espeak-ng-data.tar.bz2` }, ]; // Download each file for (const file of filesToDownload) { debugLogger.log('TTS', `Downloading: ${file.url}`); const downloadResult = await RNFS.downloadFile({ fromUrl: file.url, toFile: file.path, }).promise; if (downloadResult.statusCode !== 200) { throw new Error(`Failed to download ${file.url}: ${downloadResult.statusCode}`); } } // Extract espeak-ng-data debugLogger.log('TTS', 'Extracting espeak-ng-data...'); // Note: Extraction would need native module or untar library // For now, assume it's extracted manually or via separate process debugLogger.info('TTS', '✅ Model downloaded successfully'); return true; } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Download failed'; debugLogger.error('TTS', `Model download failed: ${errorMessage}`, error); updateState({ error: errorMessage, initializing: false }); return false; } } /** * Initialize Sherpa TTS with Orayan voice */ export async function initializeSherpaTTS(): Promise { if (!NATIVE_MODULE_AVAILABLE) { debugLogger.error('TTS', 'Cannot initialize - native module not available'); updateState({ initialized: false, error: 'Native module not available - run npx expo prebuild and rebuild' }); return false; } if (currentState.initializing) { debugLogger.warn('TTS', 'Already initializing - skipping duplicate call'); return false; } debugLogger.info('TTS', `Starting initialization with voice: ${ORAYAN_VOICE.name}`); updateState({ initializing: true, error: null }); try { // Download model if needed const downloaded = await downloadModelFromHuggingFace(); if (!downloaded) { throw new Error('Model download failed'); } // Build paths to model files const extractPath = `${RNFS.DocumentDirectoryPath}/voices`; const modelDir = `${extractPath}/${ORAYAN_VOICE.modelDir}`; const modelPath = `${modelDir}/${ORAYAN_VOICE.onnxFile}`; const tokensPath = `${modelDir}/tokens.txt`; const dataDirPath = `${modelDir}/espeak-ng-data`; debugLogger.log('TTS', 'Model paths:', { model: modelPath, tokens: tokensPath, dataDir: dataDirPath }); // Create config JSON for native module const configJSON = JSON.stringify({ modelPath, tokensPath, dataDirPath, }); debugLogger.log('TTS', `Calling TTSManager.initialize() with config JSON`); // Initialize native TTS await TTSManager.initialize(configJSON); updateState({ initialized: true, initializing: false, error: null }); debugLogger.info('TTS', '✅ Initialization successful'); return true; } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; debugLogger.error('TTS', `Initialization failed: ${errorMessage}`, error); updateState({ initialized: false, initializing: false, error: errorMessage }); return false; } } /** * Speak text using Sherpa TTS */ export async function speak( text: string, options?: { speed?: number; speakerId?: number; onStart?: () => void; onDone?: () => void; onError?: (error: Error) => void; } ): Promise { debugLogger.log('TTS', `speak() called with text: "${text.substring(0, 50)}${text.length > 50 ? '...' : ''}"`); if (!NATIVE_MODULE_AVAILABLE || !currentState.initialized) { debugLogger.error('TTS', 'Cannot speak - TTS not initialized or module unavailable'); options?.onError?.(new Error('Sherpa TTS not initialized')); return; } if (!text || text.trim().length === 0) { debugLogger.warn('TTS', 'Empty text provided, skipping speech'); return; } const speed = options?.speed ?? 1.0; const speakerId = options?.speakerId ?? 0; debugLogger.log('TTS', `Speech parameters: speed=${speed}, speakerId=${speakerId}`); try { updateState({ speaking: true }); debugLogger.info('TTS', 'State updated to speaking=true, calling onStart callback'); options?.onStart?.(); debugLogger.log('TTS', `Calling TTSManager.generateAndPlay("${text}", ${speakerId}, ${speed})`); await TTSManager.generateAndPlay(text, speakerId, speed); debugLogger.info('TTS', '✅ Speech playback completed successfully'); updateState({ speaking: false }); options?.onDone?.(); } catch (error) { const err = error instanceof Error ? error : new Error('TTS playback failed'); debugLogger.error('TTS', `💥 Speech playback error: ${err.message}`, error); updateState({ speaking: false }); options?.onError?.(err); } } /** * Stop current speech playback */ export async function stop(): Promise { debugLogger.info('TTS', 'stop() called'); if (NATIVE_MODULE_AVAILABLE && currentState.initialized) { try { debugLogger.log('TTS', 'Calling TTSManager.deinitialize() to stop playback'); TTSManager.deinitialize(); updateState({ speaking: false }); // Re-initialize after stop to be ready for next speech debugLogger.log('TTS', 'Scheduling re-initialization in 100ms'); setTimeout(() => { initializeSherpaTTS(); }, 100); debugLogger.info('TTS', 'Playback stopped successfully'); } catch (error) { debugLogger.error('TTS', 'Failed to stop playback', error); } } else { debugLogger.warn('TTS', 'Cannot stop - module not available or not initialized'); } } /** * Deinitialize and free resources */ export function deinitialize(): void { debugLogger.info('TTS', 'deinitialize() called'); if (NATIVE_MODULE_AVAILABLE) { try { debugLogger.log('TTS', 'Calling TTSManager.deinitialize() to free resources'); TTSManager.deinitialize(); debugLogger.info('TTS', 'TTS resources freed successfully'); } catch (error) { debugLogger.error('TTS', 'Failed to deinitialize', error); } } updateState({ initialized: false, speaking: false, error: null }); } /** * Check if Sherpa TTS is available (native module loaded) */ export function isAvailable(): boolean { return NATIVE_MODULE_AVAILABLE && currentState.initialized; } /** * Check if currently speaking */ export async function isSpeaking(): Promise { return currentState.speaking; } // Voice selection removed - only Lessac voice is available /** * Add listener for volume updates during playback */ export function addVolumeListener(callback: (volume: number) => void): (() => void) | null { if (!ttsManagerEmitter) return null; const subscription = ttsManagerEmitter.addListener('VolumeUpdate', (event) => { callback(event.volume); }); return () => subscription.remove(); } export default { initialize: initializeSherpaTTS, speak, stop, deinitialize, isAvailable, isSpeaking, addStateListener, getState, addVolumeListener, };