Integrate TTS interruption in VoiceFAB when voice detected
- Add onVoiceDetected callback to useSpeechRecognition hook - Triggered on first interim result (voice activity detected) - Uses voiceDetectedRef to ensure callback fires only once per session - Reset flag on session start/end - Connect STT to VoiceContext in _layout.tsx - Use useSpeechRecognition with onVoiceDetected callback - Call interruptIfSpeaking() when voice detected during 'speaking' state - Forward STT results to VoiceContext (setTranscript, sendTranscript) - Start/stop STT based on isListening state - Export interruptIfSpeaking from VoiceContext provider 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
dbf6a8a74a
commit
3c7a48df5b
@ -1,5 +1,5 @@
|
|||||||
import { Tabs } from 'expo-router';
|
import { Tabs } from 'expo-router';
|
||||||
import React, { useCallback } from 'react';
|
import React, { useCallback, useEffect } from 'react';
|
||||||
import { Platform, View } from 'react-native';
|
import { Platform, View } from 'react-native';
|
||||||
import { Feather } from '@expo/vector-icons';
|
import { Feather } from '@expo/vector-icons';
|
||||||
import { useSafeAreaInsets } from 'react-native-safe-area-context';
|
import { useSafeAreaInsets } from 'react-native-safe-area-context';
|
||||||
@ -10,6 +10,7 @@ import { AppColors } from '@/constants/theme';
|
|||||||
import { useColorScheme } from '@/hooks/use-color-scheme';
|
import { useColorScheme } from '@/hooks/use-color-scheme';
|
||||||
import { useVoiceCall } from '@/contexts/VoiceCallContext';
|
import { useVoiceCall } from '@/contexts/VoiceCallContext';
|
||||||
import { useVoice } from '@/contexts/VoiceContext';
|
import { useVoice } from '@/contexts/VoiceContext';
|
||||||
|
import { useSpeechRecognition } from '@/hooks/useSpeechRecognition';
|
||||||
|
|
||||||
export default function TabLayout() {
|
export default function TabLayout() {
|
||||||
const colorScheme = useColorScheme();
|
const colorScheme = useColorScheme();
|
||||||
@ -18,8 +19,57 @@ export default function TabLayout() {
|
|||||||
// VoiceFAB uses VoiceCallContext internally to hide when call is active
|
// VoiceFAB uses VoiceCallContext internally to hide when call is active
|
||||||
useVoiceCall(); // Ensure context is available
|
useVoiceCall(); // Ensure context is available
|
||||||
|
|
||||||
// Voice context for listening mode toggle
|
// Voice context for listening mode toggle and TTS interruption
|
||||||
const { isListening, startSession, stopSession } = useVoice();
|
const {
|
||||||
|
isListening,
|
||||||
|
status,
|
||||||
|
startSession,
|
||||||
|
stopSession,
|
||||||
|
interruptIfSpeaking,
|
||||||
|
setTranscript,
|
||||||
|
setPartialTranscript,
|
||||||
|
sendTranscript,
|
||||||
|
} = useVoice();
|
||||||
|
|
||||||
|
// Callback for voice detection - interrupt TTS when user speaks
|
||||||
|
const handleVoiceDetected = useCallback(() => {
|
||||||
|
// Interrupt TTS when user starts speaking during 'speaking' state
|
||||||
|
if (status === 'speaking') {
|
||||||
|
console.log('[TabLayout] Voice detected during speaking - interrupting TTS');
|
||||||
|
interruptIfSpeaking();
|
||||||
|
}
|
||||||
|
}, [status, interruptIfSpeaking]);
|
||||||
|
|
||||||
|
// Callback for STT results
|
||||||
|
const handleSpeechResult = useCallback((transcript: string, isFinal: boolean) => {
|
||||||
|
if (isFinal) {
|
||||||
|
setTranscript(transcript);
|
||||||
|
// Send to API when final result is received
|
||||||
|
sendTranscript(transcript);
|
||||||
|
} else {
|
||||||
|
setPartialTranscript(transcript);
|
||||||
|
}
|
||||||
|
}, [setTranscript, setPartialTranscript, sendTranscript]);
|
||||||
|
|
||||||
|
// Speech recognition with voice detection callback
|
||||||
|
const {
|
||||||
|
startListening,
|
||||||
|
stopListening,
|
||||||
|
} = useSpeechRecognition({
|
||||||
|
continuous: true,
|
||||||
|
interimResults: true,
|
||||||
|
onVoiceDetected: handleVoiceDetected,
|
||||||
|
onResult: handleSpeechResult,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Start/stop STT when voice session starts/stops
|
||||||
|
useEffect(() => {
|
||||||
|
if (isListening) {
|
||||||
|
startListening();
|
||||||
|
} else {
|
||||||
|
stopListening();
|
||||||
|
}
|
||||||
|
}, [isListening, startListening, stopListening]);
|
||||||
|
|
||||||
// Handle voice FAB press - toggle listening mode
|
// Handle voice FAB press - toggle listening mode
|
||||||
const handleVoiceFABPress = useCallback(() => {
|
const handleVoiceFABPress = useCallback(() => {
|
||||||
|
|||||||
@ -131,6 +131,8 @@ interface VoiceContextValue {
|
|||||||
speak: (text: string) => Promise<void>;
|
speak: (text: string) => Promise<void>;
|
||||||
// Stop TTS
|
// Stop TTS
|
||||||
stopSpeaking: () => void;
|
stopSpeaking: () => void;
|
||||||
|
// Interrupt TTS if speaking (call when user starts talking)
|
||||||
|
interruptIfSpeaking: () => boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const VoiceContext = createContext<VoiceContextValue | undefined>(undefined);
|
const VoiceContext = createContext<VoiceContextValue | undefined>(undefined);
|
||||||
@ -381,6 +383,7 @@ export function VoiceProvider({ children }: { children: ReactNode }) {
|
|||||||
setIsSpeaking,
|
setIsSpeaking,
|
||||||
speak,
|
speak,
|
||||||
stopSpeaking,
|
stopSpeaking,
|
||||||
|
interruptIfSpeaking,
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{children}
|
{children}
|
||||||
|
|||||||
@ -42,6 +42,8 @@ export interface UseSpeechRecognitionOptions {
|
|||||||
onStart?: () => void;
|
onStart?: () => void;
|
||||||
/** Callback when speech recognition ends */
|
/** Callback when speech recognition ends */
|
||||||
onEnd?: () => void;
|
onEnd?: () => void;
|
||||||
|
/** Callback when voice activity is detected (first interim result) - useful for interrupting TTS */
|
||||||
|
onVoiceDetected?: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface UseSpeechRecognitionReturn {
|
export interface UseSpeechRecognitionReturn {
|
||||||
@ -77,6 +79,7 @@ export function useSpeechRecognition(
|
|||||||
onError,
|
onError,
|
||||||
onStart,
|
onStart,
|
||||||
onEnd,
|
onEnd,
|
||||||
|
onVoiceDetected,
|
||||||
} = options;
|
} = options;
|
||||||
|
|
||||||
const [isListening, setIsListening] = useState(false);
|
const [isListening, setIsListening] = useState(false);
|
||||||
@ -87,6 +90,8 @@ export function useSpeechRecognition(
|
|||||||
|
|
||||||
// Track if we're in the middle of starting to prevent double-starts
|
// Track if we're in the middle of starting to prevent double-starts
|
||||||
const isStartingRef = useRef(false);
|
const isStartingRef = useRef(false);
|
||||||
|
// Track if voice has been detected in current session (for onVoiceDetected callback)
|
||||||
|
const voiceDetectedRef = useRef(false);
|
||||||
|
|
||||||
// Check availability on mount
|
// Check availability on mount
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@ -111,6 +116,7 @@ export function useSpeechRecognition(
|
|||||||
setIsListening(true);
|
setIsListening(true);
|
||||||
setError(null);
|
setError(null);
|
||||||
isStartingRef.current = false;
|
isStartingRef.current = false;
|
||||||
|
voiceDetectedRef.current = false; // Reset voice detection flag for new session
|
||||||
onStart?.();
|
onStart?.();
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -120,6 +126,7 @@ export function useSpeechRecognition(
|
|||||||
setIsListening(false);
|
setIsListening(false);
|
||||||
setPartialTranscript('');
|
setPartialTranscript('');
|
||||||
isStartingRef.current = false;
|
isStartingRef.current = false;
|
||||||
|
voiceDetectedRef.current = false; // Reset for next session
|
||||||
onEnd?.();
|
onEnd?.();
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -133,6 +140,13 @@ export function useSpeechRecognition(
|
|||||||
|
|
||||||
console.log('[SpeechRecognition] Result:', transcript.slice(0, 50), 'final:', isFinal);
|
console.log('[SpeechRecognition] Result:', transcript.slice(0, 50), 'final:', isFinal);
|
||||||
|
|
||||||
|
// Trigger onVoiceDetected on first result (voice activity detected)
|
||||||
|
if (!voiceDetectedRef.current && transcript.length > 0) {
|
||||||
|
voiceDetectedRef.current = true;
|
||||||
|
console.log('[SpeechRecognition] Voice activity detected');
|
||||||
|
onVoiceDetected?.();
|
||||||
|
}
|
||||||
|
|
||||||
if (isFinal) {
|
if (isFinal) {
|
||||||
setRecognizedText(transcript);
|
setRecognizedText(transcript);
|
||||||
setPartialTranscript('');
|
setPartialTranscript('');
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user