From 2c2afd5876676b81930d049d528219023ee6f6d0 Mon Sep 17 00:00:00 2001 From: mukherja04 Date: Wed, 19 Nov 2025 16:58:48 -0600 Subject: [PATCH 1/5] Added mic input not detected message --- src/App.tsx | 37 +++++++++++++ .../api/scribearServer/scribearRecognizer.tsx | 54 +++++++++++++++++++ .../api/whisper/whisperRecognizer.tsx | 53 ++++++++++++++++++ .../redux/reducers/controlReducers.tsx | 3 ++ .../redux/types/controlStatus.tsx | 2 + src/store.tsx | 2 + 6 files changed, 151 insertions(+) diff --git a/src/App.tsx b/src/App.tsx index 2a1be4e7..8773a415 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -16,6 +16,8 @@ function App() { const scribearStatus = useSelector((state: RootState) => state.APIStatusReducer?.scribearServerStatus as number); const scribearMessage = useSelector((state: RootState) => (state.APIStatusReducer as any)?.scribearServerMessage as string | undefined); + const micNoAudio = useSelector((state: RootState) => (state.ControlReducer as any)?.micNoAudio as boolean | undefined); + const listening = useSelector((state: RootState) => (state.ControlReducer as any)?.listening as boolean | undefined); const [snackbarOpen, setSnackbarOpen] = useState(false); const [snackbarMsg, setSnackbarMsg] = useState(''); @@ -39,6 +41,41 @@ function App() { } }, [scribearStatus]); + useEffect(() => { + // show mic inactivity when mic is on but no audio chunks are received + if (listening && micNoAudio) { + setSnackbarMsg('Microphone is active but no audio detected'); + setSnackbarSeverity('warning'); + setSnackbarOpen(true); + } + + // When listening turns ON, start a one-shot timer that expects at least one ondataavailable + // call within thresholdMs. This avoids firing on normal silent pauses after audio has been + // received previously. We only trigger inactivity if no blob arrives at all after enabling mic. + const thresholdMs = 3000; + try { + if (listening) { + try { (window as any).__hasReceivedAudio = false; } catch (e) {} + if ((window as any).__initialAudioTimer) { try { clearTimeout((window as any).__initialAudioTimer); } catch (e) {} } + (window as any).__initialAudioTimer = setTimeout(() => { + try { + const has = (window as any).__hasReceivedAudio === true; + if (!has) { + try { (window as any).store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); } catch (e) {} + } + } catch (e) {} + }, thresholdMs); + } else { + // listening turned off: clear initial timer and ensure flag reset + try { if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} + try { (window as any).store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); } catch (e) {} + } + } catch (e) { + console.warn('Failed to start initial mic monitor', e); + } + // no cleanup needed here because we clear/set timer when listening toggles + }, [listening, micNoAudio]); + const handleClose = (_event?: React.SyntheticEvent | Event, reason?: string) => { if (reason === 'clickaway') return; setSnackbarOpen(false); diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index 931293ce..971d223a 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -31,6 +31,10 @@ export class ScribearRecognizer implements Recognizer { private language: string private recorder?: RecordRTC; private kSampleRate = 16000; + // last time we received an audio chunk (ms since epoch) + private lastAudioTimestamp: number | null = null; + // interval id for inactivity checks + private inactivityInterval: any = null; urlParams = new URLSearchParams(window.location.search); mode = this.urlParams.get('mode'); @@ -58,6 +62,18 @@ export class ScribearRecognizer implements Recognizer { desiredSampRate: this.kSampleRate, timeSlice: 50, ondataavailable: async (blob: Blob) => { + // update last audio timestamp and mark that we've received at least one audio chunk + this.lastAudioTimestamp = Date.now(); + try { (window as any).__lastAudioTimestamp = this.lastAudioTimestamp; } catch (e) {} + try { (window as any).__hasReceivedAudio = true; if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} + try { + const controlState = (store.getState() as any).ControlReducer; + if (controlState?.micNoAudio === true) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Failed to clear mic inactivity', e); + } this.socket?.send(blob); }, recorderType: StereoAudioRecorder, @@ -65,6 +81,34 @@ export class ScribearRecognizer implements Recognizer { }); this.recorder.startRecording(); + + // start inactivity monitor: if mic is on but we haven't received audio for threshold -> set micNoAudio + const thresholdMs = 3000; // consider no audio if no chunks in 3s + if (this.inactivityInterval == null) { + this.inactivityInterval = setInterval(() => { + try { + const state: any = store.getState(); + const listening = state.ControlReducer?.listening === true; + const micNoAudio = state.ControlReducer?.micNoAudio === true; + if (listening) { + if (!this.lastAudioTimestamp || (Date.now() - this.lastAudioTimestamp > thresholdMs)) { + if (!micNoAudio) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); + } + } else { + if (micNoAudio) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } + } else { + // not listening: ensure flag is cleared + if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Error in mic inactivity interval', e); + } + }, 1000); + } } /** @@ -179,6 +223,16 @@ export class ScribearRecognizer implements Recognizer { if (!this.socket) { return; } this.socket.close(); this.socket = null; + // clear inactivity interval and reset mic inactivity flag + if (this.inactivityInterval) { + clearInterval(this.inactivityInterval); + this.inactivityInterval = null; + } + try { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } catch (e) { + console.warn('Failed to clear mic inactivity on stop', e); + } } /** diff --git a/src/components/api/whisper/whisperRecognizer.tsx b/src/components/api/whisper/whisperRecognizer.tsx index ad61e21f..c33a57d2 100644 --- a/src/components/api/whisper/whisperRecognizer.tsx +++ b/src/components/api/whisper/whisperRecognizer.tsx @@ -48,6 +48,9 @@ export class WhisperRecognizer implements Recognizer { private num_threads: number; private transcribed_callback: ((newFinalBlocks: Array, newInProgressBlock: TranscriptBlock) => void) | null = null; + // mic activity tracking + private lastAudioTimestamp: number | null = null; + private inactivityInterval: any = null; /** * Creates an Whisper recognizer instance that listens to the default microphone @@ -135,6 +138,22 @@ export class WhisperRecognizer implements Recognizer { pcm_data = Float32Concat(last_suffix, pcm_data); last_suffix = pcm_data.slice(-(pcm_data.length % 128)) + // update last audio timestamp and mark that we've received at least one audio chunk + this.lastAudioTimestamp = Date.now(); + try { (window as any).__lastAudioTimestamp = this.lastAudioTimestamp; } catch (e) {} + try { (window as any).__hasReceivedAudio = true; if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} + try { + // clear micNoAudio if previously set + // avoid importing store at top; use global store via require to prevent circular import issues + const { store } = require('../../../store'); + const controlState = (store.getState() as any).ControlReducer; + if (controlState?.micNoAudio === true) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Failed to clear mic inactivity (whisper)', e); + } + // Feed process_recorder_message audio in 128 sample chunks for (let i = 0; i < pcm_data.length - 127; i+= 128) { const audio_chunk = pcm_data.subarray(i, i + 128) @@ -149,6 +168,29 @@ export class WhisperRecognizer implements Recognizer { this.recorder.startRecording(); console.log("Whisper: Done setting up audio context"); + // start inactivity monitor + const thresholdMs = 3000; + if (this.inactivityInterval == null) { + const { store } = require('../../../store'); + this.inactivityInterval = setInterval(() => { + try { + const state: any = store.getState(); + const listening = state.ControlReducer?.listening === true; + const micNoAudio = state.ControlReducer?.micNoAudio === true; + if (listening) { + if (!this.lastAudioTimestamp || (Date.now() - this.lastAudioTimestamp > thresholdMs)) { + if (!micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); + } else { + if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } else { + if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Error in whisper mic inactivity interval', e); + } + }, 1000); + } } private async load_model(model: string) { @@ -257,6 +299,17 @@ export class WhisperRecognizer implements Recognizer { this.whisper.set_status("paused"); this.context.suspend(); this.recorder?.stopRecording(); + // clear inactivity interval and reset mic inactivity flag + if (this.inactivityInterval) { + clearInterval(this.inactivityInterval); + this.inactivityInterval = null; + } + try { + const { store } = require('../../../store'); + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } catch (e) { + console.warn('Failed to clear mic inactivity on whisper stop', e); + } } /** diff --git a/src/react-redux&middleware/redux/reducers/controlReducers.tsx b/src/react-redux&middleware/redux/reducers/controlReducers.tsx index f21c2802..847b52b8 100644 --- a/src/react-redux&middleware/redux/reducers/controlReducers.tsx +++ b/src/react-redux&middleware/redux/reducers/controlReducers.tsx @@ -22,6 +22,7 @@ const initialControlState : ControlStatus = { showMFCC: false, showSpeaker: false, showIntent: false, + micNoAudio: false, } export function ControlReducer(state = initialControlState, action) { @@ -41,6 +42,8 @@ export function ControlReducer(state = initialControlState, action) { return { ...state, showIntent: !state.showIntent }; case 'FLIP_RECORDING_PHRASE': return { ...state, listening: action.payload}; + case 'SET_MIC_INACTIVITY': + return { ...state, micNoAudio: action.payload }; case 'SET_SPEECH_LANGUAGE': return { ...state, diff --git a/src/react-redux&middleware/redux/types/controlStatus.tsx b/src/react-redux&middleware/redux/types/controlStatus.tsx index 52197a4e..e736446a 100644 --- a/src/react-redux&middleware/redux/types/controlStatus.tsx +++ b/src/react-redux&middleware/redux/types/controlStatus.tsx @@ -16,4 +16,6 @@ export type ControlStatus = { showMFCC: boolean showSpeaker: boolean showIntent: boolean + // true when microphone is on but no audio chunks have been received for a short time + micNoAudio?: boolean } diff --git a/src/store.tsx b/src/store.tsx index 8b1f8a3c..f90c72f6 100644 --- a/src/store.tsx +++ b/src/store.tsx @@ -37,3 +37,5 @@ export const store = configureStore({ }); export type RootState = ReturnType +// Expose store on window for runtime helpers (used by mic inactivity monitor) +(window as any).store = store; From f68f25720727ff9e29653783df89cd6e5d529d26 Mon Sep 17 00:00:00 2001 From: mukherja04 <84088896+mukherja04@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:45:27 -0600 Subject: [PATCH 2/5] Clean up comments in scribearRecognizer.tsx Removed comments related to audio chunk timestamps and inactivity checks. --- src/components/api/scribearServer/scribearRecognizer.tsx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index 971d223a..06777734 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -31,9 +31,7 @@ export class ScribearRecognizer implements Recognizer { private language: string private recorder?: RecordRTC; private kSampleRate = 16000; - // last time we received an audio chunk (ms since epoch) private lastAudioTimestamp: number | null = null; - // interval id for inactivity checks private inactivityInterval: any = null; urlParams = new URLSearchParams(window.location.search); @@ -82,8 +80,8 @@ export class ScribearRecognizer implements Recognizer { this.recorder.startRecording(); - // start inactivity monitor: if mic is on but we haven't received audio for threshold -> set micNoAudio - const thresholdMs = 3000; // consider no audio if no chunks in 3s + // start inactivity monitor + const thresholdMs = 3000; if (this.inactivityInterval == null) { this.inactivityInterval = setInterval(() => { try { @@ -101,7 +99,6 @@ export class ScribearRecognizer implements Recognizer { } } } else { - // not listening: ensure flag is cleared if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); } } catch (e) { @@ -223,7 +220,6 @@ export class ScribearRecognizer implements Recognizer { if (!this.socket) { return; } this.socket.close(); this.socket = null; - // clear inactivity interval and reset mic inactivity flag if (this.inactivityInterval) { clearInterval(this.inactivityInterval); this.inactivityInterval = null; From 55a61008596ae7f7fc48c4d3250d834b01f4aa7b Mon Sep 17 00:00:00 2001 From: mukherja04 <84088896+mukherja04@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:48:09 -0600 Subject: [PATCH 3/5] Clean up comments in whisperRecognizer.tsx Removed commented-out code related to mic activity tracking and inactivity monitoring. --- src/components/api/whisper/whisperRecognizer.tsx | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/components/api/whisper/whisperRecognizer.tsx b/src/components/api/whisper/whisperRecognizer.tsx index c33a57d2..4bc866ed 100644 --- a/src/components/api/whisper/whisperRecognizer.tsx +++ b/src/components/api/whisper/whisperRecognizer.tsx @@ -48,7 +48,6 @@ export class WhisperRecognizer implements Recognizer { private num_threads: number; private transcribed_callback: ((newFinalBlocks: Array, newInProgressBlock: TranscriptBlock) => void) | null = null; - // mic activity tracking private lastAudioTimestamp: number | null = null; private inactivityInterval: any = null; @@ -143,8 +142,6 @@ export class WhisperRecognizer implements Recognizer { try { (window as any).__lastAudioTimestamp = this.lastAudioTimestamp; } catch (e) {} try { (window as any).__hasReceivedAudio = true; if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} try { - // clear micNoAudio if previously set - // avoid importing store at top; use global store via require to prevent circular import issues const { store } = require('../../../store'); const controlState = (store.getState() as any).ControlReducer; if (controlState?.micNoAudio === true) { @@ -168,7 +165,7 @@ export class WhisperRecognizer implements Recognizer { this.recorder.startRecording(); console.log("Whisper: Done setting up audio context"); - // start inactivity monitor + const thresholdMs = 3000; if (this.inactivityInterval == null) { const { store } = require('../../../store'); @@ -299,7 +296,7 @@ export class WhisperRecognizer implements Recognizer { this.whisper.set_status("paused"); this.context.suspend(); this.recorder?.stopRecording(); - // clear inactivity interval and reset mic inactivity flag + if (this.inactivityInterval) { clearInterval(this.inactivityInterval); this.inactivityInterval = null; From 012e619ac05bb26cf70cce9f20009d2c35fa9b94 Mon Sep 17 00:00:00 2001 From: mukherja04 <84088896+mukherja04@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:49:58 -0600 Subject: [PATCH 4/5] Remove comment on micNoAudio property Removed comment about microphone audio status. --- src/react-redux&middleware/redux/types/controlStatus.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/react-redux&middleware/redux/types/controlStatus.tsx b/src/react-redux&middleware/redux/types/controlStatus.tsx index e736446a..2279af24 100644 --- a/src/react-redux&middleware/redux/types/controlStatus.tsx +++ b/src/react-redux&middleware/redux/types/controlStatus.tsx @@ -16,6 +16,5 @@ export type ControlStatus = { showMFCC: boolean showSpeaker: boolean showIntent: boolean - // true when microphone is on but no audio chunks have been received for a short time micNoAudio?: boolean } From c057765c417cfb66ddb0b69193b424e5b4facc99 Mon Sep 17 00:00:00 2001 From: mukherja04 <84088896+mukherja04@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:50:45 -0600 Subject: [PATCH 5/5] Refactor store configuration in store.tsx --- src/store.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store.tsx b/src/store.tsx index f90c72f6..6e0e883f 100644 --- a/src/store.tsx +++ b/src/store.tsx @@ -37,5 +37,5 @@ export const store = configureStore({ }); export type RootState = ReturnType -// Expose store on window for runtime helpers (used by mic inactivity monitor) (window as any).store = store; +