Fix iOS audio and transcript streaming

- Add AVAudioSession configuration via @livekit/react-native
- Configure playAndRecord, defaultToSpeaker, voiceChat for iOS
- Fix transcript spam: update existing message until isFinal
- Remove unused Sherpa TTS service
- Add simulator build profile to eas.json
This commit is contained in:
Sergei 2026-01-16 13:56:29 -08:00
parent c1380b55dd
commit a2eb4e6882
7 changed files with 793 additions and 11754 deletions

View File

@ -23,7 +23,10 @@ import {
Keyboard, Keyboard,
Animated, Animated,
Easing, Easing,
ScrollView,
Share,
} from 'react-native'; } from 'react-native';
import * as Clipboard from 'expo-clipboard';
import { Ionicons, Feather } from '@expo/vector-icons'; import { Ionicons, Feather } from '@expo/vector-icons';
import { SafeAreaView } from 'react-native-safe-area-context'; import { SafeAreaView } from 'react-native-safe-area-context';
import * as SecureStore from 'expo-secure-store'; import * as SecureStore from 'expo-secure-store';
@ -33,6 +36,7 @@ import {
useUltravox, useUltravox,
UltravoxSessionStatus, UltravoxSessionStatus,
} from 'ultravox-react-native'; } from 'ultravox-react-native';
import { AudioSession } from '@livekit/react-native';
import { api } from '@/services/api'; import { api } from '@/services/api';
import { useBeneficiary } from '@/contexts/BeneficiaryContext'; import { useBeneficiary } from '@/contexts/BeneficiaryContext';
import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme'; import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme';
@ -47,6 +51,13 @@ const API_URL = 'https://eluxnetworks.net/function/well-api/api';
type VoiceCallState = 'idle' | 'connecting' | 'active' | 'ending'; type VoiceCallState = 'idle' | 'connecting' | 'active' | 'ending';
// Log entry type
interface LogEntry {
time: string;
type: 'info' | 'error' | 'status' | 'api';
message: string;
}
export default function ChatScreen() { export default function ChatScreen() {
const router = useRouter(); const router = useRouter();
const { currentBeneficiary, setCurrentBeneficiary } = useBeneficiary(); const { currentBeneficiary, setCurrentBeneficiary } = useBeneficiary();
@ -66,8 +77,44 @@ export default function ChatScreen() {
// Voice call state (Ultravox) // Voice call state (Ultravox)
const [voiceCallState, setVoiceCallState] = useState<VoiceCallState>('idle'); const [voiceCallState, setVoiceCallState] = useState<VoiceCallState>('idle');
const voiceCallStateRef = useRef<VoiceCallState>('idle'); // Ref to avoid useFocusEffect deps
const [isMuted, setIsMuted] = useState(false); const [isMuted, setIsMuted] = useState(false);
// Debug logs state
const [logs, setLogs] = useState<LogEntry[]>([]);
const [showLogs, setShowLogs] = useState(true);
const logsScrollRef = useRef<ScrollView>(null);
// Add log helper
const addLog = useCallback((type: LogEntry['type'], message: string) => {
const time = new Date().toLocaleTimeString('en-US', { hour12: false });
setLogs(prev => [...prev.slice(-50), { time, type, message }]); // Keep last 50 logs
console.log(`[Chat ${type}] ${message}`);
}, []);
// Copy logs to clipboard
const copyLogs = useCallback(async () => {
const logsText = logs.map(l => `[${l.time}] [${l.type.toUpperCase()}] ${l.message}`).join('\n');
await Clipboard.setStringAsync(logsText);
addLog('info', 'Logs copied to clipboard!');
}, [logs, addLog]);
// Share logs
const shareLogs = useCallback(async () => {
const logsText = logs.map(l => `[${l.time}] [${l.type.toUpperCase()}] ${l.message}`).join('\n');
try {
await Share.share({ message: logsText, title: 'WellNuo Voice Logs' });
} catch (err) {
addLog('error', `Share failed: ${err}`);
}
}, [logs, addLog]);
// Clear logs
const clearLogs = useCallback(() => {
setLogs([]);
addLog('info', 'Logs cleared');
}, [addLog]);
// Animations // Animations
const pulseAnim = useRef(new Animated.Value(1)).current; const pulseAnim = useRef(new Animated.Value(1)).current;
const rotateAnim = useRef(new Animated.Value(0)).current; const rotateAnim = useRef(new Animated.Value(0)).current;
@ -100,7 +147,7 @@ export default function ChatScreen() {
const { transcripts, joinCall, leaveCall, session } = useUltravox({ const { transcripts, joinCall, leaveCall, session } = useUltravox({
tools: toolImplementations, tools: toolImplementations,
onStatusChange: (event) => { onStatusChange: (event) => {
console.log('[Chat] Ultravox status:', event.status); addLog('status', `Ultravox status: ${event.status}`);
switch (event.status) { switch (event.status) {
case UltravoxSessionStatus.IDLE: case UltravoxSessionStatus.IDLE:
@ -111,8 +158,15 @@ export default function ChatScreen() {
setVoiceCallState('connecting'); setVoiceCallState('connecting');
break; break;
case UltravoxSessionStatus.LISTENING: case UltravoxSessionStatus.LISTENING:
addLog('info', '🎤 LISTENING - microphone should be active');
setVoiceCallState('active');
break;
case UltravoxSessionStatus.THINKING: case UltravoxSessionStatus.THINKING:
addLog('info', '🤔 THINKING - processing audio');
setVoiceCallState('active');
break;
case UltravoxSessionStatus.SPEAKING: case UltravoxSessionStatus.SPEAKING:
addLog('info', '🔊 SPEAKING - audio output should play');
setVoiceCallState('active'); setVoiceCallState('active');
break; break;
case UltravoxSessionStatus.DISCONNECTING: case UltravoxSessionStatus.DISCONNECTING:
@ -122,28 +176,67 @@ export default function ChatScreen() {
}, },
}); });
// Add voice transcripts to chat history // Log on mount
useEffect(() => { useEffect(() => {
if (transcripts.length > 0) { addLog('info', 'Chat screen mounted');
const lastTranscript = transcripts[transcripts.length - 1]; addLog('info', `Beneficiary: ${currentBeneficiary?.name || 'none'}`);
}, []);
// Check if this transcript is already in messages (by content match) // Track current streaming message ID for each speaker
const isDuplicate = messages.some( const streamingMessageIdRef = useRef<{ agent: string | null; user: string | null }>({
m => m.content === lastTranscript.text && agent: null,
m.role === (lastTranscript.speaker === 'agent' ? 'assistant' : 'user') user: null,
); });
const lastTranscriptIndexRef = useRef<number>(-1);
// Add voice transcripts to chat history - update existing message until final
useEffect(() => {
if (transcripts.length === 0) return;
// Process only new transcripts
for (let i = lastTranscriptIndexRef.current + 1; i < transcripts.length; i++) {
const transcript = transcripts[i];
if (!transcript.text.trim()) continue;
const role = transcript.speaker === 'agent' ? 'assistant' : 'user';
const speakerKey = transcript.speaker === 'agent' ? 'agent' : 'user';
const isFinal = transcript.isFinal;
if (streamingMessageIdRef.current[speakerKey] && !isFinal) {
// Update existing streaming message
setMessages(prev => prev.map(m =>
m.id === streamingMessageIdRef.current[speakerKey]
? { ...m, content: transcript.text }
: m
));
} else if (!streamingMessageIdRef.current[speakerKey]) {
// Create new message for this speaker
const newId = `voice-${speakerKey}-${Date.now()}`;
streamingMessageIdRef.current[speakerKey] = newId;
if (!isDuplicate && lastTranscript.text.trim()) {
const newMessage: Message = { const newMessage: Message = {
id: `voice-${Date.now()}`, id: newId,
role: lastTranscript.speaker === 'agent' ? 'assistant' : 'user', role,
content: lastTranscript.text, content: transcript.text,
timestamp: new Date(), timestamp: new Date(),
isVoice: true, isVoice: true,
}; };
setMessages(prev => [...prev, newMessage]); setMessages(prev => [...prev, newMessage]);
} }
// If final, clear the streaming ID so next utterance creates new message
if (isFinal) {
// Final update to ensure we have the complete text
setMessages(prev => prev.map(m =>
m.id === streamingMessageIdRef.current[speakerKey]
? { ...m, content: transcript.text }
: m
));
streamingMessageIdRef.current[speakerKey] = null;
}
} }
lastTranscriptIndexRef.current = transcripts.length - 1;
}, [transcripts]); }, [transcripts]);
// Pulse animation when voice call is active // Pulse animation when voice call is active
@ -192,6 +285,7 @@ export default function ChatScreen() {
// Start voice call with Ultravox // Start voice call with Ultravox
const startVoiceCall = useCallback(async () => { const startVoiceCall = useCallback(async () => {
addLog('info', 'Starting voice call...');
setVoiceCallState('connecting'); setVoiceCallState('connecting');
Keyboard.dismiss(); Keyboard.dismiss();
@ -206,20 +300,46 @@ export default function ChatScreen() {
setMessages(prev => [...prev, systemMsg]); setMessages(prev => [...prev, systemMsg]);
const systemPrompt = getSystemPrompt(); const systemPrompt = getSystemPrompt();
addLog('api', `System prompt length: ${systemPrompt.length} chars`);
try { try {
// Configure iOS audio session for voice calls
if (Platform.OS === 'ios') {
addLog('info', 'Configuring iOS audio session...');
await AudioSession.setAppleAudioConfiguration({
audioCategory: 'playAndRecord',
audioCategoryOptions: ['allowBluetooth', 'defaultToSpeaker', 'mixWithOthers'],
audioMode: 'voiceChat',
});
await AudioSession.startAudioSession();
addLog('info', 'iOS audio session configured');
}
addLog('api', 'Calling createCall API...');
const result = await createCall({ const result = await createCall({
systemPrompt, systemPrompt,
firstSpeaker: 'FIRST_SPEAKER_AGENT', firstSpeaker: 'FIRST_SPEAKER_AGENT',
}); });
if (!result.success) { if (!result.success) {
addLog('error', `createCall failed: ${result.error}`);
throw new Error(result.error); throw new Error(result.error);
} }
console.log('[Chat] Call created, joining...'); addLog('api', `Call created! joinUrl: ${result.data.joinUrl?.substring(0, 50)}...`);
addLog('info', 'Joining call via Ultravox...');
await joinCall(result.data.joinUrl); await joinCall(result.data.joinUrl);
addLog('info', 'joinCall completed successfully');
// Log session info for audio debugging
setTimeout(() => {
if (session) {
addLog('info', `Session active: ${!!session}`);
addLog('info', `Session status: ${session.status}`);
}
}, 1000);
// Update system message // Update system message
setMessages(prev => prev.map(m => setMessages(prev => prev.map(m =>
m.id === systemMsg.id m.id === systemMsg.id
@ -227,17 +347,18 @@ export default function ChatScreen() {
: m : m
)); ));
} catch (err) { } catch (err) {
console.error('[Chat] Failed to start voice call:', err); const errorMsg = err instanceof Error ? err.message : String(err);
addLog('error', `Voice call failed: ${errorMsg}`);
setVoiceCallState('idle'); setVoiceCallState('idle');
// Update with error // Update with error
setMessages(prev => prev.map(m => setMessages(prev => prev.map(m =>
m.id === systemMsg.id m.id === systemMsg.id
? { ...m, content: `❌ Failed to connect: ${err instanceof Error ? err.message : 'Unknown error'}` } ? { ...m, content: `❌ Failed to connect: ${errorMsg}` }
: m : m
)); ));
} }
}, [joinCall]); }, [joinCall, addLog]);
// End voice call // End voice call
const endVoiceCall = useCallback(async () => { const endVoiceCall = useCallback(async () => {
@ -247,6 +368,16 @@ export default function ChatScreen() {
} catch (err) { } catch (err) {
console.error('[Chat] Error leaving call:', err); console.error('[Chat] Error leaving call:', err);
} }
// Stop iOS audio session
if (Platform.OS === 'ios') {
try {
await AudioSession.stopAudioSession();
} catch (err) {
console.error('[Chat] Error stopping audio session:', err);
}
}
setVoiceCallState('idle'); setVoiceCallState('idle');
// Add end message // Add end message
@ -273,17 +404,31 @@ export default function ChatScreen() {
} }
}, [session, isMuted]); }, [session, isMuted]);
// End call when leaving screen // Sync voiceCallState with ref (to avoid useFocusEffect deps causing re-renders)
useEffect(() => {
voiceCallStateRef.current = voiceCallState;
}, [voiceCallState]);
// Store leaveCall in ref to avoid dependency issues
const leaveCallRef = useRef(leaveCall);
useEffect(() => {
leaveCallRef.current = leaveCall;
}, [leaveCall]);
// End call when screen loses focus - NO dependencies to prevent callback recreation
useFocusEffect( useFocusEffect(
useCallback(() => { useCallback(() => {
// Focus callback - nothing to do here
return () => { return () => {
if (voiceCallState === 'active' || voiceCallState === 'connecting') { // Cleanup on unfocus - use refs to get current values
const currentState = voiceCallStateRef.current;
if (currentState === 'active' || currentState === 'connecting') {
console.log('[Chat] Screen unfocused, ending voice call'); console.log('[Chat] Screen unfocused, ending voice call');
leaveCall().catch(console.error); leaveCallRef.current().catch(console.error);
setVoiceCallState('idle'); // Note: Don't setVoiceCallState here - let the status change effect handle it
} }
}; };
}, [voiceCallState, leaveCall]) }, []) // Empty deps - callback never recreates
); );
// Load beneficiaries // Load beneficiaries
@ -640,6 +785,60 @@ export default function ChatScreen() {
</TouchableOpacity> </TouchableOpacity>
</View> </View>
</KeyboardAvoidingView> </KeyboardAvoidingView>
{/* Debug Logs Panel */}
{showLogs && (
<View style={styles.logsContainer}>
<View style={styles.logsHeader}>
<Text style={styles.logsTitle}>Debug Logs ({logs.length})</Text>
<View style={styles.logsButtons}>
<TouchableOpacity onPress={copyLogs} style={styles.logButton}>
<Ionicons name="copy-outline" size={18} color={AppColors.primary} />
</TouchableOpacity>
<TouchableOpacity onPress={shareLogs} style={styles.logButton}>
<Ionicons name="share-outline" size={18} color={AppColors.primary} />
</TouchableOpacity>
<TouchableOpacity onPress={clearLogs} style={styles.logButton}>
<Ionicons name="trash-outline" size={18} color={AppColors.error} />
</TouchableOpacity>
<TouchableOpacity onPress={() => setShowLogs(false)} style={styles.logButton}>
<Ionicons name="chevron-down" size={18} color={AppColors.textMuted} />
</TouchableOpacity>
</View>
</View>
<ScrollView
ref={logsScrollRef}
style={styles.logsScrollView}
onContentSizeChange={() => logsScrollRef.current?.scrollToEnd({ animated: true })}
>
{logs.length === 0 ? (
<Text style={styles.logEmpty}>No logs yet. Tap voice call button to start.</Text>
) : (
logs.map((log, index) => (
<Text
key={index}
style={[
styles.logLine,
log.type === 'error' && styles.logError,
log.type === 'api' && styles.logApi,
log.type === 'status' && styles.logStatus,
]}
>
[{log.time}] [{log.type.toUpperCase()}] {log.message}
</Text>
))
)}
</ScrollView>
</View>
)}
{/* Show logs toggle when hidden */}
{!showLogs && (
<TouchableOpacity style={styles.showLogsButton} onPress={() => setShowLogs(true)}>
<Ionicons name="code-slash" size={16} color={AppColors.white} />
<Text style={styles.showLogsText}>Logs ({logs.length})</Text>
</TouchableOpacity>
)}
</SafeAreaView> </SafeAreaView>
); );
} }
@ -950,4 +1149,82 @@ const styles = StyleSheet.create({
fontWeight: '500', fontWeight: '500',
color: AppColors.textPrimary, color: AppColors.textPrimary,
}, },
// Debug Logs styles
logsContainer: {
position: 'absolute',
bottom: 0,
left: 0,
right: 0,
backgroundColor: '#1a1a2e',
borderTopLeftRadius: BorderRadius.lg,
borderTopRightRadius: BorderRadius.lg,
maxHeight: 200,
zIndex: 100,
},
logsHeader: {
flexDirection: 'row',
justifyContent: 'space-between',
alignItems: 'center',
paddingHorizontal: Spacing.md,
paddingVertical: Spacing.sm,
borderBottomWidth: 1,
borderBottomColor: '#2d2d44',
},
logsTitle: {
fontSize: FontSizes.sm,
fontWeight: '600',
color: '#fff',
},
logsButtons: {
flexDirection: 'row',
gap: Spacing.sm,
},
logButton: {
padding: Spacing.xs,
},
logsScrollView: {
maxHeight: 150,
paddingHorizontal: Spacing.sm,
paddingVertical: Spacing.xs,
},
logEmpty: {
color: '#888',
fontSize: FontSizes.sm,
fontStyle: 'italic',
padding: Spacing.md,
textAlign: 'center',
},
logLine: {
fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace',
fontSize: 11,
color: '#ccc',
paddingVertical: 2,
},
logError: {
color: '#ff6b6b',
},
logApi: {
color: '#4ecdc4',
},
logStatus: {
color: '#ffe66d',
},
showLogsButton: {
position: 'absolute',
bottom: 100,
right: Spacing.md,
backgroundColor: '#1a1a2e',
flexDirection: 'row',
alignItems: 'center',
paddingHorizontal: Spacing.md,
paddingVertical: Spacing.sm,
borderRadius: BorderRadius.full,
gap: Spacing.xs,
zIndex: 50,
},
showLogsText: {
color: '#fff',
fontSize: FontSizes.sm,
fontWeight: '500',
},
}); });

View File

@ -13,26 +13,21 @@ import { SafeAreaView } from 'react-native-safe-area-context';
import { Ionicons } from '@expo/vector-icons'; import { Ionicons } from '@expo/vector-icons';
import { debugLogger, type LogEntry } from '@/services/DebugLogger'; import { debugLogger, type LogEntry } from '@/services/DebugLogger';
import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme'; import { AppColors, BorderRadius, FontSizes, Spacing } from '@/constants/theme';
import sherpaTTS from '@/services/sherpaTTS'; import * as Speech from 'expo-speech';
export default function DebugScreen() { export default function DebugScreen() {
const [logs, setLogs] = useState<LogEntry[]>([]); const [logs, setLogs] = useState<LogEntry[]>([]);
const [filter, setFilter] = useState<string>(''); const [filter, setFilter] = useState<string>('');
const [selectedCategory, setSelectedCategory] = useState<string>('All'); const [selectedCategory, setSelectedCategory] = useState<string>('All');
const [ttsState, setTtsState] = useState<ReturnType<typeof sherpaTTS.getState>>(sherpaTTS.getState()); const [ttsState, setTtsState] = useState({ initialized: true, initializing: false, error: null as string | null });
const flatListRef = useRef<FlatList>(null); const flatListRef = useRef<FlatList>(null);
// Initialize TTS and subscribe to state changes // Initialize TTS (expo-speech is always available)
useEffect(() => { useEffect(() => {
// Subscribe to TTS state changes debugLogger.info('TTS', 'Using Expo Speech (always ready)');
const unsubscribeTTS = sherpaTTS.addStateListener(setTtsState); return () => {
Speech.stop();
// Start initialization };
sherpaTTS.initialize().catch(e =>
debugLogger.error('TTS', `Init failed: ${e}`)
);
return unsubscribeTTS;
}, []); }, []);
// Subscribe to log updates // Subscribe to log updates
@ -79,15 +74,10 @@ export default function DebugScreen() {
} }
}; };
// Test TTS - check if ready before speaking // Test TTS
const handleTestTTS = () => { const handleTestTTS = () => {
if (!ttsState.initialized) {
debugLogger.warn('TTS', 'Cannot test - TTS not ready yet');
return;
}
debugLogger.info('TTS', 'Testing voice...'); debugLogger.info('TTS', 'Testing voice...');
sherpaTTS.speak('Hello, this is a test message', { Speech.speak('Hello, this is a test message', {
onDone: () => debugLogger.info('TTS', 'Voice test complete'), onDone: () => debugLogger.info('TTS', 'Voice test complete'),
onError: (e) => debugLogger.error('TTS', `Voice test failed: ${e}`) onError: (e) => debugLogger.error('TTS', `Voice test failed: ${e}`)
}); });

View File

@ -8,6 +8,13 @@
"developmentClient": true, "developmentClient": true,
"distribution": "internal" "distribution": "internal"
}, },
"development-simulator": {
"developmentClient": true,
"distribution": "internal",
"ios": {
"simulator": true
}
},
"preview": { "preview": {
"distribution": "internal" "distribution": "internal"
}, },

11838
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -11,12 +11,15 @@
"lint": "expo lint" "lint": "expo lint"
}, },
"dependencies": { "dependencies": {
"@config-plugins/react-native-webrtc": "^13.0.0",
"@dr.pogodin/react-native-fs": "^2.36.2", "@dr.pogodin/react-native-fs": "^2.36.2",
"@expo/vector-icons": "^15.0.3", "@expo/vector-icons": "^15.0.3",
"@livekit/react-native-expo-plugin": "^1.0.1",
"@react-navigation/bottom-tabs": "^7.4.0", "@react-navigation/bottom-tabs": "^7.4.0",
"@react-navigation/elements": "^2.6.3", "@react-navigation/elements": "^2.6.3",
"@react-navigation/native": "^7.1.8", "@react-navigation/native": "^7.1.8",
"expo": "~54.0.29", "expo": "~54.0.29",
"expo-clipboard": "~8.0.8",
"expo-constants": "~18.0.12", "expo-constants": "~18.0.12",
"expo-file-system": "~19.0.21", "expo-file-system": "~19.0.21",
"expo-font": "~14.0.10", "expo-font": "~14.0.10",
@ -39,10 +42,10 @@
"react-native-reanimated": "~4.1.1", "react-native-reanimated": "~4.1.1",
"react-native-safe-area-context": "~5.6.0", "react-native-safe-area-context": "~5.6.0",
"react-native-screens": "~4.16.0", "react-native-screens": "~4.16.0",
"react-native-sherpa-onnx-offline-tts": "^0.2.6",
"react-native-web": "~0.21.0", "react-native-web": "~0.21.0",
"react-native-webview": "^13.16.0", "react-native-webview": "^13.16.0",
"react-native-worklets": "0.5.1" "react-native-worklets": "0.5.1",
"ultravox-react-native": "^0.0.1"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "~19.1.0", "@types/react": "~19.1.0",

View File

@ -1,345 +0,0 @@
/**
* Sherpa TTS Service - Dynamic model loading
* Uses react-native-sherpa-onnx-offline-tts with Piper VITS models
* Models downloaded from Hugging Face on first use
*/
import { NativeEventEmitter } from 'react-native';
import TTSManager from 'react-native-sherpa-onnx-offline-tts';
import RNFS from '@dr.pogodin/react-native-fs';
import { debugLogger } from '@/services/DebugLogger';
// Only Orayan (Ryan) voice - downloaded from Hugging Face
const ORAYAN_VOICE = {
id: 'ryan-medium',
name: 'Ryan (Orayan)',
description: 'Male, clear voice',
hfRepo: 'csukuangfj/vits-piper-en_US-ryan-medium',
modelDir: 'vits-piper-en_US-ryan-medium',
onnxFile: 'en_US-ryan-medium.onnx',
};
interface SherpaTTSState {
initialized: boolean;
initializing: boolean;
speaking: boolean;
error: string | null;
}
// Check if native module is available
const NATIVE_MODULE_AVAILABLE = !!TTSManager;
if (NATIVE_MODULE_AVAILABLE) {
debugLogger.info('TTS', 'TTSManager native module loaded successfully');
} else {
debugLogger.error('TTS', 'TTSManager native module NOT available - prebuild required');
}
let ttsManagerEmitter: NativeEventEmitter | null = null;
if (NATIVE_MODULE_AVAILABLE) {
ttsManagerEmitter = new NativeEventEmitter(TTSManager);
debugLogger.info('TTS', 'TTS event emitter initialized');
}
let currentState: SherpaTTSState = {
initialized: false,
initializing: false,
speaking: false,
error: null,
};
// State listeners
const stateListeners: ((state: SherpaTTSState) => void)[] = [];
function updateState(updates: Partial<SherpaTTSState>) {
currentState = { ...currentState, ...updates };
notifyListeners();
}
function notifyListeners() {
stateListeners.forEach(listener => listener({ ...currentState }));
}
export function addStateListener(listener: (state: SherpaTTSState) => void) {
stateListeners.push(listener);
listener({ ...currentState });
return () => {
const index = stateListeners.indexOf(listener);
if (index >= 0) stateListeners.splice(index, 1);
};
}
export function getState(): SherpaTTSState {
return { ...currentState };
}
/**
* Download TTS model from Hugging Face
*/
async function downloadModelFromHuggingFace(): Promise<boolean> {
const extractPath = `${RNFS.DocumentDirectoryPath}/voices`;
const modelDir = `${extractPath}/${ORAYAN_VOICE.modelDir}`;
const modelPath = `${modelDir}/${ORAYAN_VOICE.onnxFile}`;
// Check if already downloaded
const exists = await RNFS.exists(modelPath);
if (exists) {
debugLogger.info('TTS', `Model already downloaded at: ${modelPath}`);
return true;
}
debugLogger.info('TTS', `Downloading model from Hugging Face: ${ORAYAN_VOICE.hfRepo}`);
updateState({ initializing: true, error: 'Downloading voice model...' });
try {
// Create directories
await RNFS.mkdir(modelDir, { intermediateDirectories: true });
// Download model files from Hugging Face
const baseUrl = `https://huggingface.co/${ORAYAN_VOICE.hfRepo}/resolve/main`;
const filesToDownload = [
{ url: `${baseUrl}/${ORAYAN_VOICE.onnxFile}`, path: modelPath },
{ url: `${baseUrl}/tokens.txt`, path: `${modelDir}/tokens.txt` },
{ url: `${baseUrl}/espeak-ng-data.tar.bz2`, path: `${modelDir}/espeak-ng-data.tar.bz2` },
];
// Download each file
for (const file of filesToDownload) {
debugLogger.log('TTS', `Downloading: ${file.url}`);
const downloadResult = await RNFS.downloadFile({
fromUrl: file.url,
toFile: file.path,
}).promise;
if (downloadResult.statusCode !== 200) {
throw new Error(`Failed to download ${file.url}: ${downloadResult.statusCode}`);
}
}
// Extract espeak-ng-data
debugLogger.log('TTS', 'Extracting espeak-ng-data...');
// Note: Extraction would need native module or untar library
// For now, assume it's extracted manually or via separate process
debugLogger.info('TTS', '✅ Model downloaded successfully');
return true;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Download failed';
debugLogger.error('TTS', `Model download failed: ${errorMessage}`, error);
updateState({ error: errorMessage, initializing: false });
return false;
}
}
/**
* Initialize Sherpa TTS with Orayan voice
*/
export async function initializeSherpaTTS(): Promise<boolean> {
if (!NATIVE_MODULE_AVAILABLE) {
debugLogger.error('TTS', 'Cannot initialize - native module not available');
updateState({
initialized: false,
error: 'Native module not available - run npx expo prebuild and rebuild'
});
return false;
}
if (currentState.initializing) {
debugLogger.warn('TTS', 'Already initializing - skipping duplicate call');
return false;
}
debugLogger.info('TTS', `Starting initialization with voice: ${ORAYAN_VOICE.name}`);
updateState({ initializing: true, error: null });
try {
// Download model if needed
const downloaded = await downloadModelFromHuggingFace();
if (!downloaded) {
throw new Error('Model download failed');
}
// Build paths to model files
const extractPath = `${RNFS.DocumentDirectoryPath}/voices`;
const modelDir = `${extractPath}/${ORAYAN_VOICE.modelDir}`;
const modelPath = `${modelDir}/${ORAYAN_VOICE.onnxFile}`;
const tokensPath = `${modelDir}/tokens.txt`;
const dataDirPath = `${modelDir}/espeak-ng-data`;
debugLogger.log('TTS', 'Model paths:', {
model: modelPath,
tokens: tokensPath,
dataDir: dataDirPath
});
// Create config JSON for native module
const configJSON = JSON.stringify({
modelPath,
tokensPath,
dataDirPath,
});
debugLogger.log('TTS', `Calling TTSManager.initialize() with config JSON`);
// Initialize native TTS
await TTSManager.initialize(configJSON);
updateState({
initialized: true,
initializing: false,
error: null
});
debugLogger.info('TTS', '✅ Initialization successful');
return true;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
debugLogger.error('TTS', `Initialization failed: ${errorMessage}`, error);
updateState({
initialized: false,
initializing: false,
error: errorMessage
});
return false;
}
}
/**
* Speak text using Sherpa TTS
*/
export async function speak(
text: string,
options?: {
speed?: number;
speakerId?: number;
onStart?: () => void;
onDone?: () => void;
onError?: (error: Error) => void;
}
): Promise<void> {
debugLogger.log('TTS', `speak() called with text: "${text.substring(0, 50)}${text.length > 50 ? '...' : ''}"`);
if (!NATIVE_MODULE_AVAILABLE || !currentState.initialized) {
debugLogger.error('TTS', 'Cannot speak - TTS not initialized or module unavailable');
options?.onError?.(new Error('Sherpa TTS not initialized'));
return;
}
if (!text || text.trim().length === 0) {
debugLogger.warn('TTS', 'Empty text provided, skipping speech');
return;
}
const speed = options?.speed ?? 1.0;
const speakerId = options?.speakerId ?? 0;
debugLogger.log('TTS', `Speech parameters: speed=${speed}, speakerId=${speakerId}`);
try {
updateState({ speaking: true });
debugLogger.info('TTS', 'State updated to speaking=true, calling onStart callback');
options?.onStart?.();
debugLogger.log('TTS', `Calling TTSManager.generateAndPlay("${text}", ${speakerId}, ${speed})`);
await TTSManager.generateAndPlay(text, speakerId, speed);
debugLogger.info('TTS', '✅ Speech playback completed successfully');
updateState({ speaking: false });
options?.onDone?.();
} catch (error) {
const err = error instanceof Error ? error : new Error('TTS playback failed');
debugLogger.error('TTS', `💥 Speech playback error: ${err.message}`, error);
updateState({ speaking: false });
options?.onError?.(err);
}
}
/**
* Stop current speech playback
*/
export async function stop(): Promise<void> {
debugLogger.info('TTS', 'stop() called');
if (NATIVE_MODULE_AVAILABLE && currentState.initialized) {
try {
debugLogger.log('TTS', 'Calling TTSManager.deinitialize() to stop playback');
TTSManager.deinitialize();
updateState({ speaking: false });
// Re-initialize after stop to be ready for next speech
debugLogger.log('TTS', 'Scheduling re-initialization in 100ms');
setTimeout(() => {
initializeSherpaTTS();
}, 100);
debugLogger.info('TTS', 'Playback stopped successfully');
} catch (error) {
debugLogger.error('TTS', 'Failed to stop playback', error);
}
} else {
debugLogger.warn('TTS', 'Cannot stop - module not available or not initialized');
}
}
/**
* Deinitialize and free resources
*/
export function deinitialize(): void {
debugLogger.info('TTS', 'deinitialize() called');
if (NATIVE_MODULE_AVAILABLE) {
try {
debugLogger.log('TTS', 'Calling TTSManager.deinitialize() to free resources');
TTSManager.deinitialize();
debugLogger.info('TTS', 'TTS resources freed successfully');
} catch (error) {
debugLogger.error('TTS', 'Failed to deinitialize', error);
}
}
updateState({ initialized: false, speaking: false, error: null });
}
/**
* Check if Sherpa TTS is available (native module loaded)
*/
export function isAvailable(): boolean {
return NATIVE_MODULE_AVAILABLE && currentState.initialized;
}
/**
* Check if currently speaking
*/
export async function isSpeaking(): Promise<boolean> {
return currentState.speaking;
}
// Voice selection removed - only Lessac voice is available
/**
* Add listener for volume updates during playback
*/
export function addVolumeListener(callback: (volume: number) => void): (() => void) | null {
if (!ttsManagerEmitter) return null;
const subscription = ttsManagerEmitter.addListener('VolumeUpdate', (event) => {
callback(event.volume);
});
return () => subscription.remove();
}
export default {
initialize: initializeSherpaTTS,
speak,
stop,
deinitialize,
isAvailable,
isSpeaking,
addStateListener,
getState,
addVolumeListener,
};

View File

@ -2,6 +2,7 @@
"extends": "expo/tsconfig.base", "extends": "expo/tsconfig.base",
"compilerOptions": { "compilerOptions": {
"strict": true, "strict": true,
"resolveJsonModule": true,
"paths": { "paths": {
"@/*": [ "@/*": [
"./*" "./*"