From 158acb9ab3acca55c9d0612607906281cae4cbe3 Mon Sep 17 00:00:00 2001 From: Bennett Wu <57691028+bennettrwu@users.noreply.github.com> Date: Mon, 5 May 2025 11:16:08 -0500 Subject: [PATCH 1/4] add option to configure model from frontend --- src/components/api/recogComponent.tsx | 4 +- src/components/api/returnAPI.tsx | 27 +++-- .../api/scribearServer/scribearRecognizer.tsx | 114 +++++++++++------- src/components/navbars/sidebar/model/menu.tsx | 56 +++++++++ src/components/navbars/sidebar/sidebar.tsx | 7 +- .../topbar/api/ScribearServerSettings.tsx | 23 ++++ src/muiImports.tsx | 3 +- .../redux/reducers/apiReducers.tsx | 3 +- .../redux/reducers/modelSelectionReducers.tsx | 33 +++++ .../redux/types/apiTypes.tsx | 3 +- .../redux/types/modelSelection.tsx | 18 +++ .../redux/typesImports.tsx | 14 ++- src/store.tsx | 10 +- 13 files changed, 246 insertions(+), 69 deletions(-) create mode 100644 src/components/navbars/sidebar/model/menu.tsx create mode 100644 src/react-redux&middleware/redux/reducers/modelSelectionReducers.tsx create mode 100644 src/react-redux&middleware/redux/types/modelSelection.tsx diff --git a/src/components/api/recogComponent.tsx b/src/components/api/recogComponent.tsx index f927cd0b..e4b2ce9c 100644 --- a/src/components/api/recogComponent.tsx +++ b/src/components/api/recogComponent.tsx @@ -17,6 +17,7 @@ import { STTRenderer } from '../sttRenderer'; import { PlaybackStatus, StreamTextStatus } from '../../react-redux&middleware/redux/types/apiTypes'; import Swal from 'sweetalert2'; import { useRecognition } from './returnAPI'; +import { selectSelectedModel } from '../../react-redux&middleware/redux/reducers/modelSelectionReducers'; export const RecogComponent: React.FC = (props) => { @@ -82,9 +83,10 @@ export const RecogComponent: React.FC = (props) => { const sRecog = useSelector((state: RootState) => { return state.SRecognitionReducer as SRecognition; }) + const selectedModelOption = useSelector(selectSelectedModel); // if else for whisper transcript, apiStatus for 4=whisper and control status for listening - const transcript = useRecognition(sRecog, apiStatus, controlStatus, azureStatus, streamTextStatus, scribearServerStatus, playbackStatus); + const transcript = useRecognition(sRecog, apiStatus, controlStatus, azureStatus, streamTextStatus, scribearServerStatus, selectedModelOption, playbackStatus); // console.log("Recog component received new transcript: ", transcript) return STTRenderer(transcript); }; diff --git a/src/components/api/returnAPI.tsx b/src/components/api/returnAPI.tsx index 141783fa..6a1a6080 100644 --- a/src/components/api/returnAPI.tsx +++ b/src/components/api/returnAPI.tsx @@ -1,6 +1,6 @@ // import * as sdk from 'microsoft-cognitiveservices-speech-sdk' import installCOIServiceWorker from './coi-serviceworker' -import { API, PlaybackStatus} from '../../react-redux&middleware/redux/typesImports'; +import { API, PlaybackStatus } from '../../react-redux&middleware/redux/typesImports'; import { ApiStatus, AzureStatus, @@ -9,7 +9,7 @@ import { StreamTextStatus, ScribearServerStatus } from '../../react-redux&middleware/redux/typesImports'; -import {useEffect, useState } from 'react'; +import { useEffect, useState } from 'react'; import { batch, useDispatch, useSelector } from 'react-redux'; import { AzureRecognizer } from './azure/azureRecognizer'; @@ -23,6 +23,7 @@ import { WhisperRecognizer } from './whisper/whisperRecognizer'; import { PlaybackRecognizer } from './playback/playbackRecognizer'; import { ScribearRecognizer } from './scribearServer/scribearRecognizer'; +import type { SelectedOption } from '../../react-redux&middleware/redux/types/modelSelection'; // import { PlaybackReducer } from '../../react-redux&middleware/redux/reducers/apiReducers'; // controls what api to send and what to do when error handling. @@ -55,9 +56,9 @@ export const returnRecogAPI = (api : ApiStatus, control : ControlStatus, azure : */ -const createRecognizer = (currentApi: number, control: ControlStatus, azure: AzureStatus, streamTextConfig: StreamTextStatus, scribearServerStatus: ScribearServerStatus,playbackStatus:PlaybackStatus): Recognizer => { +const createRecognizer = (currentApi: number, control: ControlStatus, azure: AzureStatus, streamTextConfig: StreamTextStatus, scribearServerStatus: ScribearServerStatus, selectedModelOption: SelectedOption, playbackStatus: PlaybackStatus): Recognizer => { if (currentApi === API.SCRIBEAR_SERVER) { - return new ScribearRecognizer(scribearServerStatus, control.speechLanguage.CountryCode); + return new ScribearRecognizer(scribearServerStatus, selectedModelOption, control.speechLanguage.CountryCode); } else if (currentApi === API.PLAYBACK) { return new PlaybackRecognizer(playbackStatus); } @@ -65,7 +66,7 @@ const createRecognizer = (currentApi: number, control: ControlStatus, azure: Azu return new WebSpeechRecognizer(null, control.speechLanguage.CountryCode); } else if (currentApi === API.AZURE_TRANSLATION) { return new AzureRecognizer(null, control.speechLanguage.CountryCode, azure); - } + } else if (currentApi === API.AZURE_CONVERSATION) { throw new Error("Not implemented"); } @@ -92,9 +93,9 @@ const updateTranscript = (dispatch: Dispatch) => (newFinalBlocks: Array { for (const block of newFinalBlocks) { - dispatch({type: "transcript/new_final_block", payload: block}); + dispatch({ type: "transcript/new_final_block", payload: block }); } - dispatch({type: 'transcript/update_in_progress_block', payload: newInProgressBlock}); + dispatch({ type: 'transcript/update_in_progress_block', payload: newInProgressBlock }); }) } @@ -111,8 +112,8 @@ const updateTranscript = (dispatch: Dispatch) => (newFinalBlocks: Array { +export const useRecognition = (sRecog: SRecognition, api: ApiStatus, control: ControlStatus, + azure: AzureStatus, streamTextConfig: StreamTextStatus, scribearServerStatus, selectedModelOption: SelectedOption, playbackStatus: PlaybackStatus) => { const [recognizer, setRecognizer] = useState(); // TODO: Add a reset button to utitlize resetTranscript @@ -129,9 +130,9 @@ export const useRecognition = (sRecog : SRecognition, api : ApiStatus, control : console.log("UseRecognition, switching to new recognizer: ", api.currentApi); let newRecognizer: Recognizer | null; - try{ + try { // Create new recognizer, and subscribe to its events - newRecognizer = createRecognizer(api.currentApi, control, azure, streamTextConfig, scribearServerStatus, playbackStatus); + newRecognizer = createRecognizer(api.currentApi, control, azure, streamTextConfig, scribearServerStatus, selectedModelOption, playbackStatus); newRecognizer.onTranscribed(updateTranscript(dispatch)); setRecognizer(newRecognizer) @@ -148,7 +149,7 @@ export const useRecognition = (sRecog : SRecognition, api : ApiStatus, control : // Stop current recognizer when switching to another one, if possible newRecognizer?.stop(); } - }, [api.currentApi, azure, control, streamTextConfig, playbackStatus, scribearServerStatus]); + }, [api.currentApi, azure, control, streamTextConfig, playbackStatus, scribearServerStatus, selectedModelOption]); // Start / stop recognizer, if listening toggled useEffect(() => { @@ -173,7 +174,7 @@ export const useRecognition = (sRecog : SRecognition, api : ApiStatus, control : }, [azure.phrases]); // TODO: whisper's transcript is not in redux store but only in sessionStorage at the moment. - let transcript : string = useSelector((state: RootState) => { + let transcript: string = useSelector((state: RootState) => { return state.TranscriptReducer.transcripts[0].toString() }); // if (api.currentApi === API.WHISPER) { diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index 53096819..fc1dff29 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -1,7 +1,10 @@ import { Recognizer } from '../recognizer'; import { TranscriptBlock } from '../../../react-redux&middleware/redux/types/TranscriptTypes'; import { ScribearServerStatus } from '../../../react-redux&middleware/redux/typesImports'; -import RecordRTC, {StereoAudioRecorder} from 'recordrtc'; +import RecordRTC, { StereoAudioRecorder } from 'recordrtc'; +import { store } from '../../../store' +import { setModelOptions, setSelectedModel } from '../../../react-redux&middleware/redux/reducers/modelSelectionReducers'; +import type { SelectedOption } from '../../../react-redux&middleware/redux/types/modelSelection'; enum BackendTranscriptBlockType { @@ -19,11 +22,12 @@ type BackendTranscriptBlock = { export class ScribearRecognizer implements Recognizer { - private scribearServerStatus : ScribearServerStatus - private socket : WebSocket | null = null - private transcribedCallback : any + private scribearServerStatus: ScribearServerStatus + private selectedModelOption: SelectedOption + private socket: WebSocket | null = null + private transcribedCallback: any private errorCallback?: (e: Error) => void; - private language : string + private language: string private recorder?: RecordRTC; private kSampleRate = 16000; @@ -34,14 +38,16 @@ export class ScribearRecognizer implements Recognizer { * @param audioSource Not implemented yet * @param language Expected language of the speech to be transcribed */ - constructor(scribearServerStatus: ScribearServerStatus, language:string) { + constructor(scribearServerStatus: ScribearServerStatus, selectedModelOption: SelectedOption, language: string) { console.log("ScribearRecognizer, new recognizer being created!") + this.language = language; + this.selectedModelOption = selectedModelOption; this.scribearServerStatus = scribearServerStatus; } private async _startRecording() { - let mic_stream = await navigator.mediaDevices.getUserMedia({audio: true, video: false}); + let mic_stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); this.recorder = new RecordRTC(mic_stream, { type: 'audio', @@ -53,7 +59,7 @@ export class ScribearRecognizer implements Recognizer { }, recorderType: StereoAudioRecorder, numberOfAudioChannels: 1, - }); + }); this.recorder.startRecording(); } @@ -64,50 +70,72 @@ export class ScribearRecognizer implements Recognizer { */ start() { console.log("ScribearRecognizer.start()"); - if(this.socket) {return;} + if (this.socket) { return; } const scribearURL = new URL(this.scribearServerStatus.scribearServerAddress) - if (scribearURL.pathname != '/sink') { + if (scribearURL.pathname !== '/sink') { this._startRecording(); } - this.socket = new WebSocket(this.scribearServerStatus.scribearServerAddress); + const socket = new WebSocket(this.scribearServerStatus.scribearServerAddress); - // this.socket.onopen = (e)=> {...} - const inProgressBlock = new TranscriptBlock(); - - this.socket.onmessage = (event)=> { - const server_block: BackendTranscriptBlock = JSON.parse(event.data); + socket.onopen = (event) => { + socket.send(JSON.stringify({ + api_key: this.scribearServerStatus.scribearServerKey, + sourceToken: this.scribearServerStatus.scribearServerKey + })); + } - // Todo: extract type of message (inprogress v final) and the text from the message - const inProgress = server_block.type === BackendTranscriptBlockType.InProgress; - const text = server_block.text; + socket.onmessage = (event) => { + const options = JSON.parse(event.data); + store.dispatch(setModelOptions(options)); - if(inProgress) { - inProgressBlock.text = text; // replace text - this.transcribedCallback([], inProgressBlock); + if (this.selectedModelOption) { + socket.send(JSON.stringify(this.selectedModelOption)); } else { - inProgressBlock.text = "" //reset in progress - const finalBlock = new TranscriptBlock(); - finalBlock.text = text - this.transcribedCallback([finalBlock], inProgressBlock) + store.dispatch(setSelectedModel(options[0])); + return; } - }; - - this.socket.onerror = (event) => { - const error = new Error("WebSocket error"); - console.error("WebSocket error event:", event); - this.errorCallback?.(error); - }; - - this.socket.onclose = (event) => { - console.warn(`WebSocket closed: code=${event.code}, reason=${event.reason}`); - this.socket = null; - if (event.code !== 1000) { // 1000 = normal closure - const error = new Error(`WebSocket closed unexpectedly: code=${event.code}`); + + this.socket = socket; + + const inProgressBlock = new TranscriptBlock(); + + this.socket.onmessage = (event) => { + const server_block: BackendTranscriptBlock = JSON.parse(event.data); + + // Todo: extract type of message (inprogress v final) and the text from the message + const inProgress = server_block.type === BackendTranscriptBlockType.InProgress; + const text = server_block.text; + + if (inProgress) { + inProgressBlock.text = text; // replace text + this.transcribedCallback([], inProgressBlock); + } else { + inProgressBlock.text = "" //reset in progress + const finalBlock = new TranscriptBlock(); + finalBlock.text = text + this.transcribedCallback([finalBlock], inProgressBlock) + } + }; + + this.socket.onerror = (event) => { + const error = new Error("WebSocket error"); + console.error("WebSocket error event:", event); this.errorCallback?.(error); - } - }; + }; + + this.socket.onclose = (event) => { + console.warn(`WebSocket closed: code=${event.code}, reason=${event.reason}`); + this.socket = null; + if (event.code !== 1000) { // 1000 = normal closure + const error = new Error(`WebSocket closed unexpectedly: code=${event.code}`); + this.errorCallback?.(error); + } + }; + } + + } /** @@ -117,7 +145,7 @@ export class ScribearRecognizer implements Recognizer { stop() { console.log("ScribearRecognizer.stop()"); this.recorder?.stopRecording(); - if(! this.socket) {return;} + if (!this.socket) { return; } this.socket.close(); this.socket = null; } @@ -130,7 +158,7 @@ export class ScribearRecognizer implements Recognizer { onTranscribed(callback: (newFinalBlocks: Array, newInProgressBlock: TranscriptBlock) => void) { console.log("ScribearRecognizer.onTranscribed()"); // "recognizing" event signals that the in-progress block has been updated - this.transcribedCallback = callback; + this.transcribedCallback = callback; } /** diff --git a/src/components/navbars/sidebar/model/menu.tsx b/src/components/navbars/sidebar/model/menu.tsx new file mode 100644 index 00000000..72809abc --- /dev/null +++ b/src/components/navbars/sidebar/model/menu.tsx @@ -0,0 +1,56 @@ +import React from 'react'; + +import { List, ListItemText, Collapse, ListItem, MemoryIcon, Autocomplete, TextField, Tooltip } from '../../../../muiImports'; +import { useSelector } from 'react-redux'; +import type { RootState } from '../../../../store'; +import { API, type ApiStatus } from '../../../../react-redux&middleware/redux/typesImports'; +import { useDispatch } from 'react-redux'; +import { selectModelOptions, selectSelectedModel, setSelectedModel } from '../../../../react-redux&middleware/redux/reducers/modelSelectionReducers'; + +export default function ModelMenu(props) { + const dispatch = useDispatch(); + const APIStatus = useSelector((state: RootState) => { + return state.APIStatusReducer as ApiStatus; + }); + const modelOptions = useSelector(selectModelOptions); + const selected = useSelector(selectSelectedModel); + const modelSelectEnable = APIStatus.currentApi !== API.SCRIBEAR_SERVER; + + return ( +
+ {props.listItemHeader("Model", "model", MemoryIcon)} + + + + + + + + + { + dispatch(setSelectedModel(val)) + }} + defaultValue={selected} + getOptionLabel={(v) => v.display_name} + isOptionEqualToValue={(a, b) => a.model_key === b.model_key} + renderInput={(params) => } + renderOption={(props, option) => { + return + + {option.display_name} + + + }} + /> + + + + +
+ ); +} \ No newline at end of file diff --git a/src/components/navbars/sidebar/sidebar.tsx b/src/components/navbars/sidebar/sidebar.tsx index 6df9b4a2..9caa0fb4 100644 --- a/src/components/navbars/sidebar/sidebar.tsx +++ b/src/components/navbars/sidebar/sidebar.tsx @@ -6,10 +6,12 @@ import LangMenu from './language/menu'; import PhraseMenu from './phrase/menu'; import VisualizationMenu from './audioVisBar/menu'; import CaptionsMenu from './captions/menu'; +import ModelMenu from './model/menu'; export default function SideBar(props) { const [state, setState] = React.useState({ + model: false, display: false, lang: true, visualization: false, @@ -48,7 +50,10 @@ export default function SideBar(props) { - + + + + diff --git a/src/components/navbars/topbar/api/ScribearServerSettings.tsx b/src/components/navbars/topbar/api/ScribearServerSettings.tsx index 90685816..f0d7beda 100644 --- a/src/components/navbars/topbar/api/ScribearServerSettings.tsx +++ b/src/components/navbars/topbar/api/ScribearServerSettings.tsx @@ -88,6 +88,29 @@ export default function ScribearServerSettings(props) { }} style={{ width: '100%' }} /> + + + + :not(style)': { pr: '1vw', width: '15vw' }, + }} + noValidate + autoComplete="off" + onSubmit={closePopup} // Prevent default submission behavior of refreshing page + > + diff --git a/src/muiImports.tsx b/src/muiImports.tsx index cc0ae89b..a08a7b18 100644 --- a/src/muiImports.tsx +++ b/src/muiImports.tsx @@ -54,7 +54,7 @@ import SubtitlesIcon from '@mui/icons-material/Subtitles'; import FormatColorTextIcon from '@mui/icons-material/FormatColorText'; import Autocomplete from '@mui/material/Autocomplete'; import { Switch } from '@mui/material'; - +import MemoryIcon from '@mui/icons-material/Memory'; export { createTheme, Autocomplete, @@ -114,4 +114,5 @@ export { FormatColorTextIcon, ArchitectureIcon, Switch, + MemoryIcon } \ No newline at end of file diff --git a/src/react-redux&middleware/redux/reducers/apiReducers.tsx b/src/react-redux&middleware/redux/reducers/apiReducers.tsx index 38b01dab..66aa625a 100644 --- a/src/react-redux&middleware/redux/reducers/apiReducers.tsx +++ b/src/react-redux&middleware/redux/reducers/apiReducers.tsx @@ -47,7 +47,8 @@ const initialPlaybackStatus: PlaybackStatus = { } const initialScribearServerState: ScribearServerStatus = { - scribearServerAddress: 'ws://localhost:1234' + scribearServerAddress: 'ws://localhost:8080/sourcesink', + scribearServerKey: '', } const saveLocally = (varName: string, value: any) => { diff --git a/src/react-redux&middleware/redux/reducers/modelSelectionReducers.tsx b/src/react-redux&middleware/redux/reducers/modelSelectionReducers.tsx new file mode 100644 index 00000000..faccd37a --- /dev/null +++ b/src/react-redux&middleware/redux/reducers/modelSelectionReducers.tsx @@ -0,0 +1,33 @@ +import type { ModelOptions, ModelSelection, SelectedOption } from '../types/modelSelection' +import type { RootState } from '../typesImports' + +const initialState: ModelSelection = { options: [], selected: null } + +export const ModelSelectionReducer = ( + state = initialState, + action +) => { + switch (action.type) { + case 'SET_MODEL_OPTIONS': + return { + ...state, + options: action.payload as ModelOptions + } + case 'SET_SELECTED_MODEL': + return { + ...state, + selected: action.payload as SelectedOption + } + default: + return state + } +} + +export const selectModelOptions = (state: RootState) => state.ModelSelectionReducer.options; +export const setModelOptions = (options: ModelOptions) => { + return { type: 'SET_MODEL_OPTIONS', payload: options } +} +export const selectSelectedModel = (state: RootState) => state.ModelSelectionReducer.selected; +export function setSelectedModel(selected: SelectedOption) { + return { type: 'SET_SELECTED_MODEL', payload: selected } +} \ No newline at end of file diff --git a/src/react-redux&middleware/redux/types/apiTypes.tsx b/src/react-redux&middleware/redux/types/apiTypes.tsx index 1c6c7558..3380452e 100644 --- a/src/react-redux&middleware/redux/types/apiTypes.tsx +++ b/src/react-redux&middleware/redux/types/apiTypes.tsx @@ -65,6 +65,7 @@ export type PlaybackStatus = { } export type ScribearServerStatus = { - scribearServerAddress : string + scribearServerAddress: string + scribearServerKey: string // Many IP (or hostname) and port in the future } \ No newline at end of file diff --git a/src/react-redux&middleware/redux/types/modelSelection.tsx b/src/react-redux&middleware/redux/types/modelSelection.tsx new file mode 100644 index 00000000..8a4603fa --- /dev/null +++ b/src/react-redux&middleware/redux/types/modelSelection.tsx @@ -0,0 +1,18 @@ +export type ModelOptions = Array<{ + model_key: string; + display_name: string; + description: string; + available_features: string; +}>; + +export type SelectedOption = { + model_key: string; + display_name: string; + description: string; + available_features: string; +} | null; + +export type ModelSelection = { + options: ModelOptions, + selected: SelectedOption +}; diff --git a/src/react-redux&middleware/redux/typesImports.tsx b/src/react-redux&middleware/redux/typesImports.tsx index f55c8cdd..b06f5f39 100644 --- a/src/react-redux&middleware/redux/typesImports.tsx +++ b/src/react-redux&middleware/redux/typesImports.tsx @@ -1,12 +1,15 @@ import { API, ApiType, STATUS, StatusType } from './types/apiEnums'; -import { ApiStatus, AzureStatus, PhraseList, PhraseListStatus, - StreamTextStatus, WhisperStatus, ScribearServerStatus, PlaybackStatus } from './types/apiTypes'; +import { + ApiStatus, AzureStatus, PhraseList, PhraseListStatus, + StreamTextStatus, WhisperStatus, ScribearServerStatus, PlaybackStatus +} from './types/apiTypes'; import { ControlStatus, LanguageList } from "./types/controlStatus"; import { SRecognition, ScribeHandler, ScribeRecognizer } from "./types/sRecognition"; import { Sentence, Word } from "./types/TranscriptTypes"; import { DisplayStatus } from "./types/displayStatus"; import { MainStream } from "./types/bucketStreamTypes"; +import type { ModelSelection } from './types/modelSelection'; export type { ApiStatus, @@ -44,9 +47,10 @@ export type RootState = { AzureReducer: AzureStatus ControlReducer: ControlStatus PhraseListReducer: PhraseListStatus - initialStreams : MainStream + initialStreams: MainStream WhisperReducer: WhisperStatus StreamTextReducer: StreamTextStatus - ScribearServerReducer : ScribearServerStatus - PlaybackReducer : PlaybackStatus + ScribearServerReducer: ScribearServerStatus + PlaybackReducer: PlaybackStatus, + ModelSelectionReducer: ModelSelection } \ No newline at end of file diff --git a/src/store.tsx b/src/store.tsx index 4a2f1eab..8b1f8a3c 100644 --- a/src/store.tsx +++ b/src/store.tsx @@ -1,5 +1,7 @@ -import { APIStatusReducer, AzureReducer, PhraseListReducer, StreamTextReducer, - WhisperReducer,ScribearServerReducer,PlaybackReducer } from './react-redux&middleware/redux/reducers/apiReducers' +import { + APIStatusReducer, AzureReducer, PhraseListReducer, StreamTextReducer, + WhisperReducer, ScribearServerReducer, PlaybackReducer +} from './react-redux&middleware/redux/reducers/apiReducers' import { combineReducers } from 'redux' import { BucketStreamReducer } from './react-redux&middleware/redux/reducers/bucketStreamReducers' @@ -10,6 +12,7 @@ import { TranscriptReducer } from './react-redux&middleware/redux/reducers/trans import { configureStore } from '@reduxjs/toolkit' import thunkMiddleware from 'redux-thunk' +import { ModelSelectionReducer } from './react-redux&middleware/redux/reducers/modelSelectionReducers' const rootReducer = combineReducers({ DisplayReducer, @@ -23,7 +26,8 @@ const rootReducer = combineReducers({ TranscriptReducer, SRecognitionReducer, ScribearServerReducer, - PlaybackReducer + PlaybackReducer, + ModelSelectionReducer }) // export const store = createStore(rootReducer) From 624418f2f0a67b44bf01cc73cb2d24f6185ec0e6 Mon Sep 17 00:00:00 2001 From: Bennett Wu <57691028+bennettrwu@users.noreply.github.com> Date: Mon, 5 May 2025 14:36:54 -0500 Subject: [PATCH 2/4] update qr code logic to support new authentication style --- .../api/scribearServer/scribearRecognizer.tsx | 11 +++-- .../topbar/api/ScribearServerSettings.tsx | 4 +- src/components/navbars/topbar/apiDropdown.tsx | 49 ++++++++++--------- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index fc1dff29..44ab2e12 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -82,18 +82,21 @@ export class ScribearRecognizer implements Recognizer { socket.onopen = (event) => { socket.send(JSON.stringify({ api_key: this.scribearServerStatus.scribearServerKey, - sourceToken: this.scribearServerStatus.scribearServerKey + sourceToken: this.scribearServerStatus.scribearServerKey, + sessionToken: this.scribearServerStatus.scribearServerKey })); } socket.onmessage = (event) => { - const options = JSON.parse(event.data); - store.dispatch(setModelOptions(options)); + const message = JSON.parse(event.data); + console.log(message); + if (message['error'] || !Array.isArray(message)) return; + + store.dispatch(setModelOptions(message)); if (this.selectedModelOption) { socket.send(JSON.stringify(this.selectedModelOption)); } else { - store.dispatch(setSelectedModel(options[0])); return; } diff --git a/src/components/navbars/topbar/api/ScribearServerSettings.tsx b/src/components/navbars/topbar/api/ScribearServerSettings.tsx index f0d7beda..79c34efe 100644 --- a/src/components/navbars/topbar/api/ScribearServerSettings.tsx +++ b/src/components/navbars/topbar/api/ScribearServerSettings.tsx @@ -104,10 +104,10 @@ export default function ScribearServerSettings(props) { onChange={updateReact} value={scribearServerStatusBuf.scribearServerKey} id="scribearServerKey" - label="ScribeAR Server API Key / Source Token" + label="ScribeAR Server API Key / Token" variant="outlined" inputProps={{ - placeholder: 'Enter ScribeAR API Key or Source Token', + placeholder: 'Enter ScribeAR API Key or Token', }} style={{ width: '100%' }} /> diff --git a/src/components/navbars/topbar/apiDropdown.tsx b/src/components/navbars/topbar/apiDropdown.tsx index 76111c2e..28f99eab 100644 --- a/src/components/navbars/topbar/apiDropdown.tsx +++ b/src/components/navbars/topbar/apiDropdown.tsx @@ -14,11 +14,11 @@ import { useDispatch, useSelector } from "react-redux"; import { ScribearServerStatus } from '../../../react-redux&middleware/redux/typesImports'; const currTheme = createTheme({ - palette: { - primary: { - main: '#ffffff' - } - }, + palette: { + primary: { + main: '#ffffff' + } + }, }); export default function ApiDropdown(props) { @@ -40,28 +40,29 @@ export default function ApiDropdown(props) { // Automatically use scribear server as sink when in student mode or as sourcesink if in kiosk mode useEffect(() => { - function switchToScribeARServer(scribearServerAddress: string) { + function switchToScribeARServer(scribearServerAddress: string, token: string) { // Set new scribear server address let copyScribearServerStatus = Object.assign({}, scribearServerStatus); copyScribearServerStatus.scribearServerAddress = scribearServerAddress - - dispatch({type: 'CHANGE_SCRIBEAR_SERVER_ADDRESS', payload: {scribearServerAddress}}); - - // Switch to scribear server - let copyStatus = Object.assign({}, apiStatus); - copyStatus.currentApi = API.SCRIBEAR_SERVER; - copyStatus.webspeechStatus = STATUS.AVAILABLE; - copyStatus.azureConvoStatus = STATUS.AVAILABLE; - copyStatus.whisperStatus = STATUS.AVAILABLE; - copyStatus.streamTextStatus = STATUS.AVAILABLE; - copyStatus.playbackStatus = STATUS.AVAILABLE; - copyStatus.scribearServerStatus = STATUS.TRANSCRIBING; + copyScribearServerStatus.scribearServerKey = token + + dispatch({ type: 'CHANGE_SCRIBEAR_SERVER_ADDRESS', payload: copyScribearServerStatus }); - dispatch({type: 'CHANGE_API_STATUS', payload: copyStatus}); - } + // Switch to scribear server + let copyStatus = Object.assign({}, apiStatus); + copyStatus.currentApi = API.SCRIBEAR_SERVER; + copyStatus.webspeechStatus = STATUS.AVAILABLE; + copyStatus.azureConvoStatus = STATUS.AVAILABLE; + copyStatus.whisperStatus = STATUS.AVAILABLE; + copyStatus.streamTextStatus = STATUS.AVAILABLE; + copyStatus.playbackStatus = STATUS.AVAILABLE; + copyStatus.scribearServerStatus = STATUS.TRANSCRIBING; + + dispatch({ type: 'CHANGE_API_STATUS', payload: copyStatus }); + } if (mode === 'kiosk') { - switchToScribeARServer(`ws://${kioskServerAddress}/sourcesink?sourceToken=${sourceToken}`); + switchToScribeARServer(`ws://${kioskServerAddress}/sourcesink`, sourceToken as string); } else if (mode === 'student') { console.log("Sending startSession POST with accessToken:", accessToken); fetch(`http://${serverAddress}/startSession`, { @@ -77,7 +78,7 @@ export default function ApiDropdown(props) { const scribearServerAddress = `ws://${serverAddress}/sink?sessionToken=${encodeURIComponent(data.sessionToken)}`; - switchToScribeARServer(scribearServerAddress); + switchToScribeARServer(scribearServerAddress, data.sessionToken); }) .catch(error => { console.error('Error starting session:', error); @@ -97,7 +98,7 @@ export default function ApiDropdown(props) { // Make this a dropdown menu with the current api as the menu title return ( - + {props.apiDisplayName} @@ -141,7 +142,7 @@ export default function ApiDropdown(props) { transformOrigin={{ horizontal: 'center', vertical: 'top' }} anchorOrigin={{ horizontal: 'center', vertical: 'bottom' }} > - + ); From b541b0145e3b6b26b93d7f508bcea05bd16e29b6 Mon Sep 17 00:00:00 2001 From: Bennett Wu <57691028+bennettrwu@users.noreply.github.com> Date: Mon, 5 May 2025 14:48:42 -0500 Subject: [PATCH 3/4] fix some issues with qr code and new model selection protocol --- .../api/scribearServer/scribearRecognizer.tsx | 20 ++++---- src/components/navbars/topbar/apiDropdown.tsx | 9 ++-- .../redux/reducers/apiReducers.tsx | 47 +++++++++++-------- .../redux/types/apiTypes.tsx | 1 + 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index 44ab2e12..b8ce2023 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -83,21 +83,23 @@ export class ScribearRecognizer implements Recognizer { socket.send(JSON.stringify({ api_key: this.scribearServerStatus.scribearServerKey, sourceToken: this.scribearServerStatus.scribearServerKey, - sessionToken: this.scribearServerStatus.scribearServerKey + sessionToken: this.scribearServerStatus.scribearServerSessionToken, })); } socket.onmessage = (event) => { - const message = JSON.parse(event.data); - console.log(message); - if (message['error'] || !Array.isArray(message)) return; + if (!this.scribearServerStatus.scribearServerSessionToken) { + const message = JSON.parse(event.data); + console.log(message); + if (message['error'] || !Array.isArray(message)) return; - store.dispatch(setModelOptions(message)); + store.dispatch(setModelOptions(message)); - if (this.selectedModelOption) { - socket.send(JSON.stringify(this.selectedModelOption)); - } else { - return; + if (this.selectedModelOption) { + socket.send(JSON.stringify(this.selectedModelOption)); + } else { + return; + } } this.socket = socket; diff --git a/src/components/navbars/topbar/apiDropdown.tsx b/src/components/navbars/topbar/apiDropdown.tsx index 28f99eab..b5d1e605 100644 --- a/src/components/navbars/topbar/apiDropdown.tsx +++ b/src/components/navbars/topbar/apiDropdown.tsx @@ -40,11 +40,12 @@ export default function ApiDropdown(props) { // Automatically use scribear server as sink when in student mode or as sourcesink if in kiosk mode useEffect(() => { - function switchToScribeARServer(scribearServerAddress: string, token: string) { + function switchToScribeARServer(scribearServerAddress: string, token: string | undefined) { // Set new scribear server address let copyScribearServerStatus = Object.assign({}, scribearServerStatus); copyScribearServerStatus.scribearServerAddress = scribearServerAddress - copyScribearServerStatus.scribearServerKey = token + copyScribearServerStatus.scribearServerKey = sourceToken as string; + copyScribearServerStatus.scribearServerSessionToken = token dispatch({ type: 'CHANGE_SCRIBEAR_SERVER_ADDRESS', payload: copyScribearServerStatus }); @@ -62,7 +63,7 @@ export default function ApiDropdown(props) { } if (mode === 'kiosk') { - switchToScribeARServer(`ws://${kioskServerAddress}/sourcesink`, sourceToken as string); + switchToScribeARServer(`ws://${kioskServerAddress}/sourcesink`, undefined); } else if (mode === 'student') { console.log("Sending startSession POST with accessToken:", accessToken); fetch(`http://${serverAddress}/startSession`, { @@ -76,7 +77,7 @@ export default function ApiDropdown(props) { .then(data => { console.log('Session token:', data.sessionToken); - const scribearServerAddress = `ws://${serverAddress}/sink?sessionToken=${encodeURIComponent(data.sessionToken)}`; + const scribearServerAddress = `ws://${serverAddress}/sink`; switchToScribeARServer(scribearServerAddress, data.sessionToken); }) diff --git a/src/react-redux&middleware/redux/reducers/apiReducers.tsx b/src/react-redux&middleware/redux/reducers/apiReducers.tsx index 66aa625a..23abe2bb 100644 --- a/src/react-redux&middleware/redux/reducers/apiReducers.tsx +++ b/src/react-redux&middleware/redux/reducers/apiReducers.tsx @@ -1,6 +1,6 @@ import { API, STATUS } from '../types/apiEnums'; import { ApiStatus, AzureStatus, PhraseList, PhraseListStatus } from "../typesImports"; -import { StreamTextStatus, WhisperStatus,ScribearServerStatus, PlaybackStatus } from "../types/apiTypes"; +import { StreamTextStatus, WhisperStatus, ScribearServerStatus, PlaybackStatus } from "../types/apiTypes"; const initialAPIStatusState: ApiStatus = { currentApi: API.WEBSPEECH, @@ -9,8 +9,8 @@ const initialAPIStatusState: ApiStatus = { azureConvoStatus: STATUS.AVAILABLE, whisperStatus: STATUS.AVAILABLE, streamTextStatus: STATUS.AVAILABLE, - scribearServerStatus : STATUS.AVAILABLE, - playbackStatus : STATUS.AVAILABLE + scribearServerStatus: STATUS.AVAILABLE, + playbackStatus: STATUS.AVAILABLE } const initialPhraseList: PhraseList = { @@ -49,6 +49,7 @@ const initialPlaybackStatus: PlaybackStatus = { const initialScribearServerState: ScribearServerStatus = { scribearServerAddress: 'ws://localhost:8080/sourcesink', scribearServerKey: '', + scribearServerSessionToken: undefined, } const saveLocally = (varName: string, value: any) => { @@ -89,7 +90,7 @@ const getLocalState = (name: string) => { } else if (name === "scribearServerStatus") { return saveLocally("scribearServerStatus", initialScribearServerState); } - return {} ; + return {}; }; export const APIStatusReducer = (state = getLocalState("apiStatus") as ApiStatus, action) => { @@ -106,15 +107,17 @@ export const APIStatusReducer = (state = getLocalState("apiStatus") as ApiStatus } } -export const AzureReducer = (state = getLocalState("azureStatus") , action) => { +export const AzureReducer = (state = getLocalState("azureStatus"), action) => { switch (action.type) { case 'CHANGE_AZURE_LOGIN': return { ...state, ...action.payload }; case 'CHANGE_AZURE_STATUS': return { ...state, ...action.payload }; case 'CHANGE_LIST': - return { ...state, - phrases: action.payload } + return { + ...state, + phrases: action.payload + } default: return state; } @@ -125,10 +128,10 @@ export const WhisperReducer = (state = getLocalState("whisperStatus"), action) = } export const StreamTextReducer = (state = getLocalState("streamTextStatus"), action) => { - switch(action.type) { + switch (action.type) { case 'CHANGE_STREAMTEXT_STATUS': return { ...state, ...action.payload }; - + default: return state; } @@ -136,22 +139,22 @@ export const StreamTextReducer = (state = getLocalState("streamTextStatus"), act export const ScribearServerReducer = (state = getLocalState("scribearServerStatus"), action) => { switch (action.type) { case 'CHANGE_SCRIBEAR_SERVER_ADDRESS': - const newState = { ...state, ...action.payload }; + const newState = { ...state, ...action.payload }; saveLocally('scribearServerStatus', newState); return newState; - + default: return state; } } export const PlaybackReducer = (state = getLocalState("playbackStatus"), action) => { - - switch(action.type) { + + switch (action.type) { case 'CHANGE_PLAYBACK_STATUS': console.log("PlaybackReducer CHANGE_PLAYBACK_STATUS: status & action", state, action); return { ...state, ...action.payload }; - + default: return state; } @@ -196,13 +199,17 @@ export const PhraseListReducer = (state = initialPhraseListState, action) => { ...state, } } - case 'EDIT_PHRASE_LIST': - return { ...state, - currentPhraseList: action.payload} + case 'EDIT_PHRASE_LIST': + return { + ...state, + currentPhraseList: action.payload + } // existing cases - case 'SET_FILE_CONTENT': - return { ...state, - fileContent: action.payload} + case 'SET_FILE_CONTENT': + return { + ...state, + fileContent: action.payload + } case 'SET_PHRASE_OPTION_TO_CUSTOM': { const newPhraseListMap = new Map(state.phraseListMap); const phraseData: PhraseList = newPhraseListMap.get(action.payload.phraseName) || { diff --git a/src/react-redux&middleware/redux/types/apiTypes.tsx b/src/react-redux&middleware/redux/types/apiTypes.tsx index 3380452e..63c6dbfc 100644 --- a/src/react-redux&middleware/redux/types/apiTypes.tsx +++ b/src/react-redux&middleware/redux/types/apiTypes.tsx @@ -67,5 +67,6 @@ export type PlaybackStatus = { export type ScribearServerStatus = { scribearServerAddress: string scribearServerKey: string + scribearServerSessionToken: string | undefined // Many IP (or hostname) and port in the future } \ No newline at end of file From 6125f7a4099f63c1c95282e3f1ac6c1a510c2611 Mon Sep 17 00:00:00 2001 From: Bennett Wu <57691028+bennettrwu@users.noreply.github.com> Date: Mon, 5 May 2025 15:14:51 -0500 Subject: [PATCH 4/4] clean up websocket to fix inconsistent connection behavior --- .../api/scribearServer/scribearRecognizer.tsx | 85 +++++++++---------- .../redux/reducers/modelSelectionReducers.tsx | 25 +++++- 2 files changed, 63 insertions(+), 47 deletions(-) diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index b8ce2023..0ae45087 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -25,12 +25,15 @@ export class ScribearRecognizer implements Recognizer { private scribearServerStatus: ScribearServerStatus private selectedModelOption: SelectedOption private socket: WebSocket | null = null + private ready = false; private transcribedCallback: any private errorCallback?: (e: Error) => void; private language: string private recorder?: RecordRTC; private kSampleRate = 16000; + urlParams = new URLSearchParams(window.location.search); + mode = this.urlParams.get('mode'); /** * Creates an Azure recognizer instance that listens to the default microphone @@ -77,18 +80,20 @@ export class ScribearRecognizer implements Recognizer { this._startRecording(); } - const socket = new WebSocket(this.scribearServerStatus.scribearServerAddress); + this.socket = new WebSocket(this.scribearServerStatus.scribearServerAddress); - socket.onopen = (event) => { - socket.send(JSON.stringify({ + this.socket.onopen = (event) => { + this.socket?.send(JSON.stringify({ api_key: this.scribearServerStatus.scribearServerKey, sourceToken: this.scribearServerStatus.scribearServerKey, sessionToken: this.scribearServerStatus.scribearServerSessionToken, })); } - socket.onmessage = (event) => { - if (!this.scribearServerStatus.scribearServerSessionToken) { + const inProgressBlock = new TranscriptBlock(); + + this.socket.onmessage = (event) => { + if (!this.ready && this.mode !== 'student') { const message = JSON.parse(event.data); console.log(message); if (message['error'] || !Array.isArray(message)) return; @@ -96,51 +101,43 @@ export class ScribearRecognizer implements Recognizer { store.dispatch(setModelOptions(message)); if (this.selectedModelOption) { - socket.send(JSON.stringify(this.selectedModelOption)); - } else { - return; + this.socket?.send(JSON.stringify(this.selectedModelOption)); + this.ready = true; } + return; } - this.socket = socket; - - const inProgressBlock = new TranscriptBlock(); - - this.socket.onmessage = (event) => { - const server_block: BackendTranscriptBlock = JSON.parse(event.data); + const server_block: BackendTranscriptBlock = JSON.parse(event.data); - // Todo: extract type of message (inprogress v final) and the text from the message - const inProgress = server_block.type === BackendTranscriptBlockType.InProgress; - const text = server_block.text; + // Todo: extract type of message (inprogress v final) and the text from the message + const inProgress = server_block.type === BackendTranscriptBlockType.InProgress; + const text = server_block.text; - if (inProgress) { - inProgressBlock.text = text; // replace text - this.transcribedCallback([], inProgressBlock); - } else { - inProgressBlock.text = "" //reset in progress - const finalBlock = new TranscriptBlock(); - finalBlock.text = text - this.transcribedCallback([finalBlock], inProgressBlock) - } - }; - - this.socket.onerror = (event) => { - const error = new Error("WebSocket error"); - console.error("WebSocket error event:", event); + if (inProgress) { + inProgressBlock.text = text; // replace text + this.transcribedCallback([], inProgressBlock); + } else { + inProgressBlock.text = "" //reset in progress + const finalBlock = new TranscriptBlock(); + finalBlock.text = text + this.transcribedCallback([finalBlock], inProgressBlock) + } + }; + + this.socket.onerror = (event) => { + const error = new Error("WebSocket error"); + console.error("WebSocket error event:", event); + this.errorCallback?.(error); + }; + + this.socket.onclose = (event) => { + console.warn(`WebSocket closed: code=${event.code}, reason=${event.reason}`); + this.socket = null; + if (event.code !== 1000) { // 1000 = normal closure + const error = new Error(`WebSocket closed unexpectedly: code=${event.code}`); this.errorCallback?.(error); - }; - - this.socket.onclose = (event) => { - console.warn(`WebSocket closed: code=${event.code}, reason=${event.reason}`); - this.socket = null; - if (event.code !== 1000) { // 1000 = normal closure - const error = new Error(`WebSocket closed unexpectedly: code=${event.code}`); - this.errorCallback?.(error); - } - }; - } - - + } + }; } /** diff --git a/src/react-redux&middleware/redux/reducers/modelSelectionReducers.tsx b/src/react-redux&middleware/redux/reducers/modelSelectionReducers.tsx index faccd37a..20c1f005 100644 --- a/src/react-redux&middleware/redux/reducers/modelSelectionReducers.tsx +++ b/src/react-redux&middleware/redux/reducers/modelSelectionReducers.tsx @@ -3,21 +3,40 @@ import type { RootState } from '../typesImports' const initialState: ModelSelection = { options: [], selected: null } + +const saveLocally = (key: string, value: any) => { + localStorage.setItem(key, JSON.stringify(value)); +} +const getLocalState = (key: string) => { + const localState = localStorage.getItem(key); + + if (localState) { + return Object.assign(initialState, JSON.parse(localState)); + } + + return initialState; +} + export const ModelSelectionReducer = ( - state = initialState, + state = getLocalState('modelSelection'), action ) => { + let newState; switch (action.type) { case 'SET_MODEL_OPTIONS': - return { + newState = { ...state, options: action.payload as ModelOptions } + saveLocally('modelSelection', newState) + return newState; case 'SET_SELECTED_MODEL': - return { + newState = { ...state, selected: action.payload as SelectedOption } + saveLocally('modelSelection', newState) + return newState; default: return state }